<reponame>rolker/noaadata
#!/usr/bin/env python
"""Tools to generate python code to serialize/deserialize messages
between python and ais binary.  Trying to be as inline as possible, so
no XML on the fly like in ais-py.

serialize: python to ais binary
deserialize: ais binary to python

The generated code uses translators.py, binary.py, and aisstring.py
which should be packaged with the resulting files.

@license: Apache 2.0

 TODO(schwehr):add a link to generated doc string to bring up the html for the pretty version
 TODO(schwehr):write a separate validation script that distinguishes standard messages and bin messages
 TODO(schwehr):make sure binary is only used in AIS ITU messages and not within the binary messages!
 TODO(schwehr):arrays
 TODO(schwehr):handle entry ranges in lookup tables
 TODO(schwehr):arraylength of -1 for messages 12 and 14
 TODO(schwehr):in the table html/kml form output, if no LUT and value is the unknown case, say so in the converted column!
 TODO(schwehr):stop using sqlhelp.  Can have trouble with corrupted data.
@bug: NOT complete
@note: This compiler assumes that all locations are in WGS 84 (EPSG 4326)
 TODO(schwehr):add type checking of command line parameters for crafting AIS messages
"""

import datetime
import os
import sys
from decimal import Decimal
from lxml import etree


def suggestType(name,curType,printout=True):
    '''Try to suggest a type name if one did not work.

    @param printout: if true, write a suggestion to stdout.

    >>> suggestType('myFieldName', 'unsigned int')
    Recommend switching "unsigned int" to "uint" for field "myFieldName"
    'uint'

    >>> suggestType('JohnWarfon', 'yoyodyne')
    Sorry!  No recommendation available for bad type "yoyodyne" for field "JohnWarfon"
    '''
    newType = None
    if curType.lower()=='unsigned int':
        newType = 'uint'
    elif curType.lower()=='unsigned decimal':
        newType = 'udecimal'

    if printout:
        if newType:
            print 'Recommend switching "'+curType+'" to "'+newType+'" for field "'+name+'"'
        else:
            print 'Sorry!  No recommendation available for bad type "'+curType+'" for field "'+name+'"'
    return newType


def hasSubTag(et,subtag):
    '''
    @return: true if the tag a sub tag with name subtag
    '''
    if 0<len(et.xpath(subtag)): return True
    return False


def writeBeginning(o):
    '''Write the doc string header for the message file.

    @param o: Open output file to write code to.
    Must be pre-expanded with the expandais.py command.
    '''

    d = datetime.datetime.utcnow()
    dateStr = str(d.year)+'-'+("%02d" %d.month)+'-'+("%02d"%d.day)

    # Need to pass in info about the source file, etc.

    # Minor trickery to get svn to ignore the keywords in the next few lines
    o.write('''#!/usr/bin/env python

\"\"\"Autogenerated python functions to serialize/deserialize binary messages.

Generated by: ''' + __file__ + '''

Need to then wrap these functions with the outer AIS packet and then
convert the whole binary blob to a NMEA string.  Those functions are
not currently provided in this file.

serialize: python to ais binary
deserialize: ais binary to python

The generated code uses translators.py, binary.py, and aisstring.py
which should be packaged with the resulting files.

''')

    o.write('''
 TODO(schwehr):FIX: put in a description of the message here with fields and types.
\"\"\"
import doctest
import sys
from decimal import Decimal
import unittest

from aisutils.BitVector import BitVector

from aisutils import aisstring
from aisutils import binary
from aisutils import sqlhelp
from aisutils import uscg

# FIX: check to see if these will be needed
TrueBV  = BitVector(bitstring="1")
"Why always rebuild the True bit?  This should speed things up a bunch"
FalseBV = BitVector(bitstring="0")
"Why always rebuild the False bit?  This should speed things up a bunch"


''')
    return

def generatePython(infile,outfile, prefixName=False,verbose=False):
    '''
    @param infile: xml ais binary message definition file
    @param outfile: where to dump the python code
    '''

    aisMsgsET = etree.parse(infile).getroot()

    o = file(outfile,'w')
    os.chmod(outfile,0755)

    writeBeginning(o)

    for msgET in aisMsgsET:
        if msgET.tag != 'message': continue
        print msgET.tag, msgET.attrib['name']

        if len(msgET.xpath('include-struct')) > 0:
            sys.exit("ERROR: cannot handle xml that still has include-struct tags.\n  Please use expandais.py.")
        buildHelpers(o,msgET,prefixName=prefixName,verbose=verbose)
        buildEncode(o,msgET,prefixName=prefixName,verbose=verbose)
        buildDecode(o,msgET,prefixName=prefixName)
        buildDecodeParts(o,msgET,prefixName=prefixName) # functions that only decode one field
        buildPrint(o,msgET,prefixName=prefixName)
        buildLUT(o,msgET,prefixName=prefixName)
        buildSQL(o,msgET,prefixName=prefixName)
        buildLaTeX(o,msgET,prefixName=prefixName,verbose=verbose)
        buildTextDef(o,msgET,prefixName=prefixName)

    o.write('\n\n######################################################################\n')
    o.write('# UNIT TESTING\n')
    o.write('######################################################################\n')

    for msgET in aisMsgsET:
        if msgET.tag != 'message': continue
        print 'Building unit tests for message ...', msgET.attrib['name']

        buildUnitTest(o,msgET, prefixName=prefixName)

    for msgET in aisMsgsET:
        if msgET.tag != 'message': continue
        buildMain(o,msgET, prefixName=prefixName)

    return

######################################################################
# Build Helpers
######################################################################

def buildHelpers(o,msgET, verbose=False, prefixName=False):
    '''
    emit the fieldList and other things???

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node

     TODO(schwehr):for lookuptable/entry values, make it also print the decoded value.
     TODO(schwehr):use a different name for message and field
     TODO(schwehr):put in comments with the python and sql data type for each field
    '''

    if verbose:
        msgname = msgET.attrib['name']
        print 'Building helpers ...',msgname

    if prefixName:
        o.write(prefixName,'FieldList = (\n')
    else:
        o.write('fieldList = (\n')

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        o.write('    \''+name+'\',\n')

    o.write(')\n\n')

    # FIX: add some documentation to the fieldList
    # FIX: like that it is used as the default csv list

    if prefixName:
        o.write(prefixName,'FieldListPostgres = (\n')
    else:
        o.write('fieldListPostgres = (\n')
    finished=[] # postgis names
    for field in msgET.xpath('field'):
        name = field.attrib['name']
        if 'postgisName' in field.attrib:
            name = field.attrib['postgisName']
            if name in finished: continue # already done, so skip
            finished.append(name)
            o.write('    \''+name+'\',    # PostGIS data type\n')
        else:
            o.write('    \''+name+'\',\n')

    o.write(')\n\n')

    del finished



    if prefixName:
        o.write(prefixName,'ToPgFields = {\n')
    else:
        o.write('toPgFields = {\n')
    for field in msgET.xpath('field'):
        if 'postgisName' not in field.attrib: continue
        name = field.attrib['name']
        pgName = field.attrib['postgisName']
        o.write('    \''+name+'\':\''+pgName+'\',\n')
    o.write('}\n')
    o.write('''\"\"\"
Go to the Postgis field names from the straight field name
\"\"\"

''')


    if prefixName:
        o.write(prefixName,'FromPgFields = {\n')
    else:
        o.write('fromPgFields = {\n')
    finished=[] # postgis names
    for field in msgET.xpath('field'):
        if 'postgisName' not in field.attrib: continue
        name = field.attrib['name']
        pgName = field.attrib['postgisName']

        if pgName in finished: continue # already done, so skip
        finished.append(pgName)
        o.write('    \''+pgName+'\':(')
        xp = 'field[@postgisName=\''+pgName+'\']'
        parts = ['\''+part.attrib['name']+'\'' for part in msgET.xpath(xp)]
        o.write(','.join(parts)+',),\n')
    o.write('}\n')
    o.write('''\"\"\"
Go from the Postgis field names to the straight field name
\"\"\"

''')


    if prefixName:
        o.write(prefixName,'PgTypes = {\n')
    else:
        o.write('pgTypes = {\n')
    finished=[] # postgis names
    for field in msgET.xpath('field'):
        if 'postgisName' not in field.attrib: continue
        pgType = field.attrib['postgisType']
        pgName = field.attrib['postgisName']
        if pgName in finished: continue # already done, so skip
        finished.append(pgName)
        o.write('    \''+pgName+'\':\''+pgType+'\',\n')
    o.write('}\n')
    o.write('''\"\"\"
Lookup table for each postgis field name to get its type.
\"\"\"

''')







######################################################################
# SIMPLE PRINT
######################################################################

def getMaxFieldNameLen(msgET):
    '''Get the maximum string length of any field name'''
    maxStrLen=0
    for field in msgET.xpath('field'):
        fieldLen = len(field.attrib['name'])
        if fieldLen>maxStrLen: maxStrLen = fieldLen
    return maxStrLen

def padStrRight(aStr,strlen):
    '''Pad a string out to the length requested with spaces out to the right'''
    return aStr + ' '*(strlen-len(aStr))

def haveLocatableMessage(msgET):
    '''Make sure this message has both long/x and lat/y fields.
    @rtype: bool
    '''
    #if getLongitudeFieldName(msgET) and getLatitudeFieldName(msgET): return True

    if 0==len(msgET.xpath('field[contains(@name, "longitude")]')): return False
    if 0==len(msgET.xpath('field[contains(@name, "latitude")]')): return False
    return True

def getLongitudeFieldName(msgET):
    '''
    Dig up the first field name that include longitude and return it
     TODO(schwehr):might want to allow a search for a special tag to mark this
    '''
    return msgET.xpath('field[contains(@name, "longitude")]')[0].attrib['name']

def getLatitudeFieldName(msgET):
    '''
    Dig up the first field name that include longitude and return it
     TODO(schwehr):might want to allow a search for a special tag to mark this
    '''
    return msgET.xpath('field[contains(@name, "latitude")]')[0].attrib['name']

def buildPrint(o,msgET, verbose=False, prefixName=False):
    '''
    Write a simple in order print for the resulting dictionary.

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node

     TODO(schwehr):for lookuptable/entry values, make it also print the decoded value.
     TODO(schwehr):use a different name for message and field
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    #msgName = msgname = name  # FIX: make these all msgName

    print 'Generating print ...',msgName # FIX: verbose?
    # +1 for the ':'
    maxFieldLen = 1 + getMaxFieldNameLen(msgET)


    ##############################
    # Html builder
    ##############################

    printHtmlName = 'printHtml'
    if prefixName: printHtmlName = name+'printHtml'




    ####################
    ####### HTML format
    ####################
    #o.write('    elif \'html\'==format:\n')
    o.write('\n')
    o.write('def '+printHtmlName+'(params, out=sys.stdout):\n')
    o.write('        out.write("<h3>'+msgName+'</h3>\\n")\n')
    o.write('        out.write("<table border=\\"1\\">\\n")\n')
    #o.write('        out.write("<tr bgcolor=\\"#9acd32\\">\\n")\n')
    o.write('        out.write("<tr bgcolor=\\"orange\\">\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Field Name</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Type</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Value</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Value in Lookup Table</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Units</th>\\n")\n')
    o.write('        out.write("</tr>\\n")\n')


    for field in msgET.xpath('field'):
        o.write('        out.write("\\n")\n')
        o.write('        out.write("<tr>\\n")\n')
        fieldname = field.attrib['name']
        fieldtype = field.attrib['type']
        o.write('        out.write("<td>'+fieldname+'</td>\\n")\n')
        o.write('        out.write("<td>'+fieldtype+'</td>\\n")\n')

        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:  arraylen=int(field.attrib['arraylength'])

        if 1==arraylen or fieldtype=='aisstr6':
            o.write('        if \''+fieldname+'\' in params:\n            out.write("    <td>"+str(params[\''+fieldname+'\'])+"</td>\\n")\n')
            if not hasSubTag(field,'lookuptable'):
                # Just duplicate the value through
                o.write('            out.write("    <td>"+str(params[\''+fieldname+'\'])+"</td>\\n")\n')
            else:
                lutName = fieldname+'DecodeLut'
                if prefixName: lutName = msgname+fieldname.capitalize()+'DecodeLut'

                o.write('            if str(params[\''+fieldname+'\']) in '+lutName+':\n')
                o.write('                out.write("<td>"+'+lutName+'[str(params[\''+fieldname+'\'])]+"</td>")\n')
                o.write('            else:\n')
                o.write('                out.write("<td><i>Missing LUT entry</i></td>")\n')
        else: sys.exit ('FIX: handle arrays in the buildPrint func')

        if hasSubTag(field,'units'):
            o.write('        out.write("<td>'+field.xpath('units')[0].text+'</td>\\n")\n')

        o.write('        out.write("</tr>\\n")\n')

    o.write('        out.write("</table>\\n")\n')
    o.write('\n')

    ##############################
    ####### KML GoogleEarth
    ##############################

    printKmlName = 'printKml'
    if prefixName: printKmlName = msgName+'printKml'
    if not haveLocatableMessage(msgET): printKmlName=None
    else:
        # FIX: use the first postgis print tagged lat and lon field
        o.write('\n')
        o.write('def printKml(params, out=sys.stdout):\n')
        o.write('    \"\"\"KML (Keyhole Markup Language) for Google Earth, but without the header/footer\"\"\"\n')
        o.write('    out.write("    <Placemark>\\n")\n')
        o.write('    out.write("        <name>"+str(params[\''+msgET.attrib['titlefield']+'\'])+"</name>\\n")\n')
        o.write('    out.write("        <description>\\n")\n')

        o.write('    import StringIO\n')
        o.write('    buf = StringIO.StringIO()\n')
        o.write('    printHtml(params,buf)\n')
        o.write('    import cgi\n')
        o.write('    out.write(cgi.escape(buf.getvalue()))\n')

        o.write('    out.write("        </description>\\n")\n')
        o.write('    out.write("        <styleUrl>#m_ylw-pushpin_copy0</styleUrl>\\n")\n')
        o.write('    out.write("        <Point>\\n")\n')
        o.write('    out.write("            <coordinates>")\n')
        o.write('    out.write(str(params[\''+getLongitudeFieldName(msgET)+'\']))\n')
        o.write('    out.write(\',\')\n')
        o.write('    out.write(str(params[\''+getLatitudeFieldName(msgET)+'\']))\n')
        o.write('    out.write(",0</coordinates>\\n")\n')
        o.write('    out.write("        </Point>\\n")\n')
        o.write('    out.write("    </Placemark>\\n")\n')
        o.write('\n')


    ##############################
    ##### Main print dispatch
    ##############################

    funcName = 'printFields'
    if prefixName: funcName = msgName+'PrintFields'

    o.write('def '+funcName+'(params, out=sys.stdout, format=\'std\', fieldList=None, dbType=\'postgres\'):\n')

    ########################################
    # doc string
    o.write("    '''Print a "+msgName+" message to stdout.\n\n")
    o.write('    Fields in params:\n')
    for field in msgET.xpath('field'):
        desc = field[0].text.replace('\n',' ') # get ride of new lines
        o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description
        if len(field.xpath("required")) == 1:
            o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')

        o.write('\n')
    o.write('    @param params: Dictionary of field names/values.\n')
    o.write('    @param out: File like object to write to.\n')

    o.write("    @rtype: stdout\n")
    o.write("    @return: text to out\n")
    o.write("    '''\n\n")

    ########################################
    # Actually build the code

    o.write('    if \'std\'==format:\n')
    o.write('        out.write("'+msgName+':\\n")\n')

    if verbose: print 'number of fields = ', len(msgET.xpath('field'))


    for field in msgET.xpath('field'):
        fieldname = field.attrib['name']
        fieldtype = field.attrib['type']
        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',fieldname,'('+fieldtype+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'
        else:
            if verbose: print 'Processing field ...',fieldname,'(',fieldtype+' ',numbits,')'

        if 1==arraylen or fieldtype=='aisstr6':
            o.write('        if \''+fieldname+'\' in params: out.write("    '+padStrRight(fieldname+':',maxFieldLen)+'  "+str(params[\''+fieldname+'\'])+"\\n")\n')
            # FIX: elif aisstr6 strip @@@ and then print
        else:
            print '\n','ERROR:',fieldname,fieldtype
            sys.exit ('ERROR: FIX: handle arrays in the buildPrint func')

    ####################
    ####### Comma separated values (csv)
    ####################
    o.write('''        elif 'csv'==format:
                if None == options.fieldList:
                        options.fieldList = fieldList
                needComma = False;
                for field in fieldList:
                        if needComma: out.write(',')
                        needComma = True
                        if field in params:
                                out.write(str(params[field]))
                        # else: leave it empty
                out.write("\\n")
''')

    ####################
    ####### Other dispatchers
    ####################
    o.write('    elif \'html\'==format:\n')
    o.write('        '+printHtmlName+'(params,out)\n')

    o.write('''    elif 'sql'==format:
                sqlInsertStr(params,out,dbType=dbType)
''')

    if haveLocatableMessage(msgET):
        # Has a lon/lat pair somewhere in there.
        o.write('    elif \'kml\'==format:\n')
        o.write('        '+printKmlName+'(params,out)\n')

        o.write('    elif \'kml-full\'==format:\n')

        o.write('        out.write(\"<?xml version=\\"1.0\\" encoding=\\"UTF-8\\"?>\\n")\n')
        o.write('        out.write("<kml xmlns=\\"http://earth.google.com/kml/2.1\\">\\n")\n')
        o.write('        out.write("<Document>\\n")\n')
#        o.write('        out.write("    <name>Position</name>\\n")\n')
        o.write('        out.write("    <name>'+msgName+'</name>\\n")\n')

        o.write('        '+printKmlName+'(params,out)\n')

        o.write('        out.write("</Document>\\n")\n')
        o.write('        out.write("</kml>\\n")\n')



    ####################
    ####### Make safety check
    ####################
    o.write('    else:\n')
    o.write('        print "ERROR: unknown format:",format\n')
    o.write('        assert False\n')

    o.write('\n    return # Nothing to return\n\n')


######################################################################
# Build Lookup tables for enumerated int or uint fields
######################################################################


######################################################################
# DELIMITED TEXT FILE
######################################################################

def buildTextDef(o,msgET, verbose=False, prefixName=False):
    '''
    Write functions for text definition output

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    @param verbose: talk lots in the process
    @param prefixName: set to a string to have the commands prefixed by that character.

     TODO(schwehr):should this be a style sheet thing instead?
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    print 'Generating text table definition ...', msgName

    createFuncName = 'textDefinitionTable'
    if prefixName: createFuncName = msgName+'textDefinitionTable'

    o.write('''\n######################################################################
# Text Definition
######################################################################

''')

    # Should this really default to true for the uscg fields?
    o.write('def '+createFuncName+'''(outfile=sys.stdout ,delim='    '):
    \"\"\"Return the text definition table for this message type

    @param outfile: file like object to print to.
    @type outfile: file obj
    @return: text table string via the outfile
    @rtype: str

    \"\"\"
    o = outfile
''')

    # NOTE: using / to mean one  in the output compiled code
    o.write('    o.write(\'Parameter\'+delim+\'Number of bits\'+delim+\"\"\"Description\n')

    lines = []
    totalBits = 0
    for field in msgET.xpath('field'):
        fieldName = field.attrib['name']
        fieldType = field.attrib['type']
        fieldBits = field.attrib['numberofbits']
        if 'arraylength' in field.attrib:
            fieldBits = str(int(fieldBits) * int(field.attrib['arraylength']))
        totalBits += int(fieldBits)
        # FIX: maybe cut down to the first sentence?
        fieldDescription = field.xpath('description')[0].text.replace('\n',' ')
        lines.append(fieldName+'\"\"\"+delim+\''+fieldBits+'\'+delim+\"\"\"'+fieldDescription)

    o.write('\n'.join(lines))
    # 168 in the first slot and following slots get to use the whole 256??
    # FIX: is this right?
    from math import ceil
    numSlots = 1 + int( ceil((totalBits-168)/256.) )
    emptySlotBits = ( 168 + (numSlots-1)*256 ) - totalBits
    s = ''
    if numSlots>1: s = 's'
    o.write(('\nTotal bits\"\"\"+delim+\"\"\"'+str(totalBits)+'\"\"\"+delim+\"\"\"Appears to take '+str(numSlots)+' slot'+s))
    if emptySlotBits > 0:
        o.write(' with '+str(emptySlotBits)+' pad bits to fill the last slot')

    o.write( ('''\"\"\")
'''))

    # FIX: should it return a copy of outfile?


######################################################################
def buildLaTeX(o,msgET, verbose=False, prefixName=False):
    '''
    Write functions for LaTeX output

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    @param verbose: talk lots in the process
    @param prefixName: set to a string to have the commands prefixed by that character.

     TODO(schwehr):should this be a style sheet thing instead?
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    print 'Generating LaTeX definition table ...', msgName

    createFuncName = 'latexDefinitionTable'
    if prefixName: createFuncName = msgName+'LaTeXDefinitionTable'

    o.write('''\n######################################################################
# LATEX SUPPORT
######################################################################

''')

    # Should this really default to true for the uscg fields?
    o.write('def '+createFuncName+'''(outfile=sys.stdout
                ):
        \"\"\"
        Return the LaTeX definition table for this message type
        @param outfile: file like object to print to.
        @type outfile: file obj
        @return: LaTeX table string via the outfile
        @rtype: str

        \"\"\"
        o = outfile
''')

    # NOTE: using / to mean one  in the output compiled code
    o.write('''
        o.write(\"\"\"
/begin{table}%[htb]
/centering
/begin{tabular}{|l|c|l|}
/hline
Parameter & Number of bits & Description
//  /hline/hline
'''.replace('/','\\\\'))

    lines = []
    totalBits = 0
    for field in msgET.xpath('field'):
        fieldName = field.attrib['name'].replace('_','\_')  # _ means subscript to latex
        fieldType = field.attrib['type']
        fieldBits = field.attrib['numberofbits']
        if 'arraylength' in field.attrib:
            fieldBits = str(int(fieldBits) * int(field.attrib['arraylength']))
        totalBits += int(fieldBits)
        # FIX: maybe cut down to the first sentence?
        fieldDescription = field.xpath('description')[0].text.replace('\n',' ')
        lines.append(fieldName+' & '+fieldBits+' & '+fieldDescription)

    o.write(' \\\\\\\\ \\hline\n'.join(lines))
    # 168 in the first slot and following slots get to use the whole 256??  FIX: is this right?
    from math import ceil
    numSlots = 1 + int( ceil((totalBits-168)/256.) )
    emptySlotBits = ( 168 + (numSlots-1)*256 ) - totalBits
    s = ''
    if numSlots>1: s = 's'
    o.write(('// /hline /hline\nTotal bits & '+str(totalBits)+' & Appears to take '+str(numSlots)+' slot'+s).replace('/','\\\\'))
    if emptySlotBits > 0:
        o.write(' with '+str(emptySlotBits)+' pad bits to fill the last slot')

#    if verbose:
#        sys.stderr.write (' msgnum:'+msgET.attrib['aismsgnum'] +' - ')
#        sys.stderr.write (' desc: '+ str(msgET.xpath('description')[0])+' - ')
#        sys.stderr.write (' name: '+ str(msgET.attrib['name'])+'\n')
        #sys.stderr.write (' :'+ str()+'\n')

    o.write( (''' // /hline
/end{tabular}
/caption{AIS message number '''+msgET.attrib['aismsgnum']+': '+msgET.xpath('description')[0].text.replace('\n',' ')+'''}
/label{tab:'''+msgET.attrib['name']+'''}
/end{table}
\"\"\")
''').replace('/','\\\\'))

    # FIX: should it return a copy of outfile?


######################################################################

def buildSQL(o,msgET, verbose=False, prefixName=False):
    '''
    Write SQL code

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    @param verbose: talk lots in the process
    @param prefixName: set to a string to have the commands prefixed by that character.
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    print 'Generating SQL commands ...', msgName

    createFuncName = 'sqlCreate'
    if prefixName: createFuncName = msgName+'SqlCreate'
    insertFuncName = 'sqlInsert'
    if prefixName: insertFuncName = msgName+'SqlInsert'

    o.write('''######################################################################
# SQL SUPPORT
######################################################################

''')

    o.write('dbTableName=\''+msgName+'\'\n')
    o.write('\'Database table name\'\n\n')

    # Should this really default to true for the uscg fields?
    o.write('def '+createFuncName+'''Str(outfile=sys.stdout, fields=None, extraFields=None
                ,addCoastGuardFields=True
                ,dbType='postgres'
                ):
        \"\"\"
        Return the SQL CREATE command for this message type
        @param outfile: file like object to print to.
        @param fields: which fields to put in the create.  Defaults to all.
        @param extraFields: A sequence of tuples containing (name,sql type) for additional fields
        @param addCoastGuardFields: Add the extra fields that come after the NMEA check some from the USCG N-AIS format
        @param dbType: Which flavor of database we are using so that the create is tailored ('sqlite' or 'postgres')
        @type addCoastGuardFields: bool
        @return: sql create string
        @rtype: str

        @see: sqlCreate
        \"\"\"
        # FIX: should this sqlCreate be the same as in LaTeX (createFuncName) rather than hard coded?
        outfile.write(str(sqlCreate(fields,extraFields,addCoastGuardFields,dbType=dbType)))

''')

    o.write('def '+createFuncName+'''(fields=None, extraFields=None, addCoastGuardFields=True, dbType='postgres'):
    \"\"\"Return the sqlhelp object to create the table.

    @param fields: which fields to put in the create.  Defaults to all.
    @param extraFields: A sequence of tuples containing (name,sql type) for additional fields
    @param addCoastGuardFields: Add the extra fields that come after the NMEA check some from the USCG N-AIS format
    @type addCoastGuardFields: bool
    @param dbType: Which flavor of database we are using so that the create is tailored ('sqlite' or 'postgres')
    @return: An object that can be used to generate a return
    @rtype: sqlhelp.create
    \"\"\"
    if fields is None:
        fields = fieldList
''')

    o.write('    c = sqlhelp.create(\''+msgName+'\',dbType=dbType)\n')
    o.write('    c.addPrimaryKey()\n');

    for field in msgET.xpath('field'):
        fieldName = field.attrib['name']
        fieldType = field.attrib['type']

        postgis = False
        if 'postgisType' in field.attrib:
            postgis=True
            o.write('    if dbType != \'postgres\':\n    ')
        o.write('    if \''+fieldName+'\' in fields: c.add')
        # FIX: add the ADD command here.
        if   fieldType in ('int','uint'): o.write('Int (\''+fieldName+'\')')
        elif fieldType in ('float'     ): o.write('Real(\''+fieldName+'\')')
        elif fieldType == 'bool':         o.write('Bool(\''+fieldName+'\')')
        elif fieldType in ('decimal','udecimal'):
            if not hasSubTag(field,'decimalplaces'):
                print '\n ** ERROR: missing decimalplaces field for ',fieldName
                print
                assert (False)
            scaleSQL = int(field.xpath('decimalplaces')[0].text)  # number of decimal places
            numBits = int(field.attrib['numberofbits'])
            if not hasSubTag(field,'scale'):
                print 'ERROR: missing required <scale> for',fieldName,'of type',fieldType
            scaleVal = float(field.xpath('scale')[0].text)
            precision = scaleSQL + len(str(int((2**numBits)/scaleVal)))
            #precision += 1 # FIX: do I really need this safety factor
            o.write('Decimal(\''+fieldName+'\','+str(precision)+','+str(scaleSQL)+')')
        elif fieldType == 'binary':
            numBits = int(field.attrib['numberofbits'])
            if -1 == numBits: numBits=1024 # FIX: what is a good maximum for AIS?
            o.write('BitVarying(\''+fieldName+'\','+str(numBits)+')')
        elif fieldType == 'aisstr6':
            arrayLength = int(field.attrib['arraylength'])
            o.write('VarChar(\''+fieldName+'\','+str(arrayLength)+')')
        else:
            print '\n\n *** Unknown type in SQL code generation for field',fieldName+':',fieldType,'\n\n'
            assert(False)
        o.write('\n')

    o.write('''
    if addCoastGuardFields:
        # c.addInt('cg_s_rssi')  # Relative signal strength indicator
        # c.addInt('cg_d_strength')  # dBm receive strength
        # c.addVarChar('cg_x',10)  # Idonno
        c.addInt('cg_t_arrival')  # Receive timestamp from the AIS equipment 'T'
        c.addInt('cg_s_slotnum')  # Slot received in
        c.addVarChar('cg_r',15)  # Receiver station ID  -  should usually be an MMSI, but sometimes is a string
        c.addInt('cg_sec')  # UTC seconds since the epoch

        c.addTimestamp('cg_timestamp') # UTC decoded cg_sec - not actually in the data stream
''')


    #
    # Postgres with PostGIS added
    #
    postgisFields = msgET.xpath('field[@postgisType]')
    if len(postgisFields)<1:
        if verbose: print 'No postgis fields'
    else:
        o.write('\n    if dbType == \'postgres\':\n')
        finished=[] # Should be a set?
        print 'processing postgis fields for create:'
        for field in postgisFields:
            #print '  ',field.attrib['name'],field.attrib['postgisName']
            pgName = field.attrib['postgisName']
            if pgName in finished:
                print 'already handled',pgName
                continue
            finished.append(pgName)
            pgType = field.attrib['postgisType']
            #print '   doing',pgName,pgType
            components = msgET.xpath('field[@postgisName=\''+pgName+'\']')
            #print '    all components'
            #for c in components:
            #    print '     ',c.attrib['postgisName']
            o.write('        #--- EPSG 4326 : WGS 84\n')

            o.write('''        #INSERT INTO "spatial_ref_sys" ("srid","auth_name","auth_srid","srtext","proj4text") VALUES (4326,'EPSG',4326,'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]','+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs ');\n''')

            o.write('        c.addPostGIS(\''+pgName+'\',\''+pgType+'\','+str(len(components))+',SRID=4326);\n')

    o.write('\n    return c\n\n')


    # FIX: function name
    o.write('''def '''+insertFuncName+'''Str(params, outfile=sys.stdout, extraParams=None, dbType='postgres'):
        \"\"\"
        Return the SQL INSERT command for this message type
        @param params: dictionary of values keyed by field name
        @param outfile: file like object to print to.
        @param extraParams: A sequence of tuples containing (name,sql type) for additional fields
        @return: sql create string
        @rtype: str

        @see: sqlCreate
        \"\"\"
        outfile.write(str(sqlInsert(params,extraParams,dbType=dbType)))


''')

    # FIX: function name
    o.write('def ' + insertFuncName + """(params,extraParams=None,dbType='postgres'):
        \"\"\"
        Give the SQL INSERT statement
        @param params: dict keyed by field name of values
        @param extraParams: any extra fields that you have created beyond the normal ais message fields
        @rtype: sqlhelp.insert
        @return: insert class instance
         TODO(schwehr):allow optional type checking of params?
        @warning: this will take invalid keys happily and do what???
        \"\"\"

        i = sqlhelp.insert('""" + msgName + """',dbType=dbType)

        if dbType=='postgres':
                finished = []
                for key in params:
                        if key in finished:
                                continue

                        if key not in toPgFields and key not in fromPgFields:
                                if type(params[key])==Decimal: i.add(key,float(params[key]))
                                else: i.add(key,params[key])
                        else:
                                if key in fromPgFields:
                                        val = params[key]
                                        # Had better be a WKT type like POINT(-88.1 30.321)
                                        i.addPostGIS(key,val)
                                        finished.append(key)
                                else:
                                        # Need to construct the type.
                                        pgName = toPgFields[key]
                                        #valStr='GeomFromText(\\''+pgTypes[pgName]+'('
                                        valStr=pgTypes[pgName]+'('
                                        vals = []
                                        for nonPgKey in fromPgFields[pgName]:
                                                vals.append(str(params[nonPgKey]))
                                                finished.append(nonPgKey)
                                        valStr+=' '.join(vals)+')'
                                        i.addPostGIS(pgName,valStr)
        else:
                for key in params:
                        if type(params[key])==Decimal: i.add(key,float(params[key]))
                        else: i.add(key,params[key])

        if None != extraParams:
                for key in extraParams:
                        i.add(key,extraParams[key])

        return i
""")



######################################################################
# Build Lookup tables for enumerated int or uint fields
######################################################################


def buildLUT(o,msgET, verbose=False, prefixName=False):
    '''
    Write lookup tables for enumerated types (uint or int, maybe bool too).

     TODO(schwehr):FIX: what to do about multiple entries with the same text?  Need to ban that kind of thing
     TODO(schwehr):Make doc strings for each LUT.

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    '''
    assert(msgET.tag=='message')
    msgname = msgET.attrib['name']

    print 'Generating lookup tables ...',msgname # FIX: verbose?

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        if not hasSubTag(field,'lookuptable'): continue
        lut = field.xpath('lookuptable')[0]

        lutName = name
        if prefixName: lutName = msgname+name.capitalize()

        o.write(lutName+'EncodeLut = {\n')
        for entry in lut.xpath('entry'):
            o.write('    \''+entry.text+'\':\''+entry.attrib['key']+'\',\n')
        o.write('    } #'+lutName+'EncodeLut\n')
        o.write('\n')

        # FIX: make doc string for LUT here

        o.write(lutName+'DecodeLut = {\n')
        for entry in lut.xpath('entry'):
            o.write('    \''+entry.attrib['key']+'\':\''+entry.text+'\',\n')
        o.write('    } # '+lutName+'EncodeLut\n')
        o.write('\n')

        # FIX: make doc string for LUT here




######################################################################
# ENCODERS
######################################################################
def encodeBool(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for boolean variables
    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: bool, etc.
    @type numbits: int = 1
    @param numbits: How many bits per unit datum (must be 1 for bools)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many bools will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''

    if verbose: print 'bool encode',name,': unvail=',unavailable

    assert type.lower()=='bool'
    assert numbits==1
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type=bool)\n')
    if None != required:
        assert type(required) == bool
        if required: o.write('        bvList.append(TrueBV)\n')
        else: o.write('        bvList.append(FalseBV)\n')
        if verbose: o.write('\n')
        return

    if not unavailable:
        o.write('    if params["' + name + '"]: bvList.append(TrueBV)\n')
        o.write('    else: bvList.append(FalseBV)\n')
    else: # Have a default value that can be filled in
        assert type(unavailable) == bool
        o.write("    if '" + name + "' in params:\n")
        o.write('        if params["' + name + '"]: bvList.append(TrueBV)\n')
        o.write('        else: bvList.append(FalseBV)\n')
        o.write('    else:\n')
        if unavailable: o.write('        bvList.append(TrueBV)\n')
        else: o.write('        bvList.append(FalseBV)\n')
    if verbose: o.write('\n')

def encodeUInt(o, name, type, numbits, required=None, arraylen=1,
               unavailable=None, verbose=False):
    '''Build the encoder for unsigned integer variables.

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many unsigned ints will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose:
        print '  encodeUInt:', name, type, numbits, 'Req:',required,
        print 'alen:', arraylen, unavailable

    assert type == 'uint'
    assert numbits >= 1 and numbits <= 32
    if arraylen != 1:
        assert False # FIX... handle arrays
    if verbose:
        o.write('    ### FIELD: '+name+' (type='+type+')\n')

    if None != required:
        if verbose:
            print '  required:', required
        required=int(required)
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal='+str(required)+'),'+str(numbits)+'))\n')
        if verbose:
            o.write('\n')
        return

    if not unavailable:
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal=params[\''+name+'\']),'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        int(unavailable) # Make sure unavailable is a number object
        o.write("    if '" + name + "' in params:\n")
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal=params[\''+name+'\']'+'),'+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal='+str(unavailable)+'),'+str(numbits)+'))\n')

    if verbose: o.write('\n')


def encodeFloat(o,name,fieldType,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for IEEE float variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type fieldType: str
    @param fieldType: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many unsigned ints will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeUInt:',name,fieldType,numbits,'Req:',required,'alen:',arraylen,unavailable

    assert numbits==32 # Force by the IEEE spec
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+fieldType+')\n')

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(binary.float2bitvec('+str(required)+'))\n')
        if verbose: o.write('\n')
        return

    if not unavailable:
        o.write('    bvList.append(binary.float2bitvec(params[\''+name+'\']))\n')
    else: # Have a default value that can be filled in
        int(unavailable) # Make sure unavailable is a number object
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.float2bitvec(params[\''+name+'\']'+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.float2bitvec('+str(unavailable)+'))\n')

    if verbose: o.write('\n')


def encodeAisstr6(o,name,fieldType,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for aisstr6 variables.  Generally are arrays.
    @bug: do we need to optionally check for a valid string?

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type fieldType: str
    @param fieldType: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many unsigned ints will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeUInt:',name,fieldType,numbits,'Req:',required,'alen:',arraylen,unavailable
    totLen = str(numbits*arraylen)
    assert numbits==6 # Each character must be 6 bits
    if verbose: o.write('    ### FIELD: '+name+' (type='+fieldType+')\n')

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(aisstring.encode(\''+str(required)+'\','+totLen+'))\n')
        if verbose: o.write('\n')
        return

    if not unavailable:
        o.write('    bvList.append(aisstring.encode(params[\''+name+'\'],'+totLen+'))\n')
    else: # Have a default value that can be filled in
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(aisstring.encode(params[\''+name+'\'],'+totLen+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(aisstring.encode(\''+str(unavailable)+'\','+totLen+'))\n')

    if verbose: o.write('\n')


def encodeInt(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for signed integer variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many signed ints will there be?  FIX: handle variable
    @type unavailable: number or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeInt:',name,type,numbits,'Req:',required,'alen:',arraylen,unavailable

    assert numbits>=1 and numbits<=32
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(binary.bvFromSignedInt('+str(required)+','+str(numbits)+'))\n')
        if verbose: o.write('\n')
        return


    if not unavailable:
        o.write('    bvList.append(binary.bvFromSignedInt(params[\''+name+'\'],'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        #assert type(unavailable)==
        int(unavailable) # Make sure unavailable is a number object
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.bvFromSignedInt(params[\''+name+'\']'+','+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.bvFromSignedInt('+str(unavailable)+','+str(numbits)+'))\n')

    if verbose: o.write('\n')



# FIX: Ummm... why am I passing the type?  I guess it makes the one print statement easier
def encodeDecimal(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False, scale=None, offset=None):
    '''
    Build the encoder for signed decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: decimal
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many decimals will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeDecimal:',name,type,numbits,'Req:',required,'alen:',arraylen,unavailable

    assert numbits>=1 and numbits<=32
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    # FIX: optimize to not emit the scaling when it is not needed... or tell the user to switch to an int!
    if None == scale:
        print 'WARNING: if you are not scaling, then you probably want to use an int instead!'
        print 'Beware canadians bearing travel videos'
        scale='1'

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(binary.bvFromSignedInt('+str(int(Decimal(required)*Decimal(scale)))+','+str(numbits)+'))\n')
        if verbose: o.write('\n')
        return

    offsetStr=''
    if None != offset:
        offsetStr='-('+offset+')'

    # FIX: can I get rid of the Decimal around params?
    if not unavailable:
        o.write('    bvList.append(binary.bvFromSignedInt(int(Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')),'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.bvFromSignedInt(int(Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')),'+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.bvFromSignedInt('+str(int(Decimal(unavailable)*Decimal(scale)))+','+str(numbits)+'))\n')

    if verbose: o.write('\n')



def encodeUDecimal(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False, scale=None, offset=None):
    '''
    Build the encoder for signed decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: decimal
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many decimals will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeDecimal:',name,type,numbits,'Req:',required,'alen:',arraylen,unavailable
    assert type=='udecimal'
    assert numbits>=1 and numbits<=32
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    # FIX: optimize to not emit the scaling when it is not needed... or tell the user to switch to an int!
    if None == scale:
        print 'WARNING: if you are not scaling, then you probably want to use an int instead!'
        print 'Beware canadians bearing travel videos'
        scale='1'

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        assert(0<=required)
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal='+str(int(Decimal(required)*Decimal(scale)))+'),'+str(numbits)+'))\n')
        if verbose: o.write('\n')
        return

    offsetStr=''
    if None != offset:
        offsetStr='-('+offset+')'

    # FIX: can I get rid of the Decimal around params?
    if not unavailable:
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal=int((Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')))),'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal=int((Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')))),'+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal=int('+str(int(Decimal(unavailable)*Decimal(scale)))+')),'+str(numbits)+'))\n')

    if verbose: o.write('\n')



def encodeBinary(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False, scale=None):
    '''
    Build the encoder for binary variables.  This is just a pass through.
    This is used for the ais binary message wrapper (e.g. msg 8).  Do
    not use it within binary messages.

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: binary
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..1024 or so)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many decimals will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encode'+name+':',type,numbits,'Req:',required,'alen:',arraylen,unavailable
    assert type=='binary'
    assert (numbits>=1 and numbits<=1024) or numbits==-1
    assert (None == required)  # don't allow this
    assert (None == unavailable) # don't allow this

    if arraylen != 1: assert False # Do not handle arrays.  Arrays of bits is just not necessary.
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    # FIX: can I get rid of the Decimal around params?
    o.write('    bvList.append(params[\''+name+'\'])\n')  # Just pass it through

    if verbose: o.write('\n')


######################################################################
# DECODERS
######################################################################

def decodeBool(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for boolean variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, bool, etc.
    @type startindex: int
    @param startindex: bit that begins the bool(s)
    @type numbits: int = 1
    @param numbits: How many bits per unit datum (must be 1 for bools)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many bools will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert(type=='bool')
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    #int(startindex); int(numbits)  # Make sure it is a number
    assert numbits==1
    assert arraylen == 1 # FIX... handle arrays

    if None != required:
        assert type(required)==bool
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        if required: o.write('True\n')
        else: o.write('False\n')
        if not decodeOnly: o.write('\n')
        return int(startindex)+int(numbits)

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('bool(int('+bv+'['+str(startindex)+':'+str(startindex+int(numbits)*int(arraylen))+']))')
    if not decodeOnly: o.write('\n')

    return int(startindex)+int(numbits)


def decodeUInt(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for unsigned integer variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, etc.
    @type startindex: int
    @param startindex: bit that begins the uint(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: int or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: int or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if arraylen is None:
        arraylen = 1
    assert arraylen == 1 # FIX... handle arrays
    assert numbits >= 1
    if not decodeOnly:
        verbose=False

    if required is not None:
        int(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(required))
        if not decodeOnly: o.write('\n')
        return startindex+numbits

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('int('+bv+'['+str(startindex)+':'+str(startindex+int(numbits)*int(arraylen))+'])')
    if not decodeOnly: o.write('\n')
    if verbose: o.write('\n')

    return startindex+numbits


def decodeInt(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for unsigned integer variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: int
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: int or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: int or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='int'
    if verbose:
        print type, 'decode', name, ': unvail=', unavailable,
        print '  numbits:', numbits, '  startindex=', startindex
    if arraylen is None:
        arraylen=1
    end = startindex+int(numbits) * int(arraylen)
    assert arraylen == 1  # FIX: handle arrays
    assert numbits >= 1

    if None != required:
        int(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(required))
        if not decodeOnly: o.write('\n')
        return end

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('binary.signedIntFromBV('+bv+'['+str(startindex)+':'+str(end)+'])')
    if not decodeOnly: o.write('\n')
    if verbose: o.write('\n')

    return end

def decodeFloat(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for IEEE float variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: int
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: float or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: float or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='float'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert numbits>=1

    if None != required:
        float(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(required))
        if not decodeOnly: o.write('\n')
        if verbose: o.write('\n')
        return end

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('binary.bitvec2float('+bv+'['+str(startindex)+':'+str(end)+'])')
    if not decodeOnly: o.write('\n')

    return end


def decodeAisstr6(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
                  bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for aisstr6 variables.  Generally arrays.
    @bug: FIX: validate strings??
    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'aisstr6'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: restricted str or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: restricted str or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='aisstr6'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen >= 1 # FIX... handle arrays
    assert numbits>=1

    if None != required:
        float(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(required)
        if not decodeOnly: o.write('\n')
        return end

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('aisstring.decode('+bv+'['+str(startindex)+':'+str(end)+'])')
    if not decodeOnly: o.write('\n')

    return end


def decodeDecimal(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False,scale=None, decodeOnly=False,offset=None):
    '''
    Build the decoder for signed decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'decimal'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: Decimal or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='decimal'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert numbits>=1 and numbits <= 32

    if None == scale: scale='1' # Warning about this was in the encode section

    if None != required:
        Decimal(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(Decimal(required))+'/Decimal(\''+scale+'\')')
        if not decodeOnly: o.write('\n')
        return end

    offsetStr=''
    if offset != None:
        offsetStr += '+Decimal(\''+offset+'\')'

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('Decimal(binary.signedIntFromBV('+bv+'['+str(startindex)+':'+str(end)+']))/Decimal(\''+scale+'\')'+offsetStr+'')
    if not decodeOnly: o.write('\n')

    return end




def decodeUDecimal(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False,scale=None, decodeOnly=False,offset=None):
    '''
    Build the decoder for unsigned decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'udecimal'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: Decimal or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='udecimal'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert numbits>=1 and numbits <= 32

    if None == scale: scale='1' # Warning about this was in the encode section

    if None != required:
        assert (Decimal(required)>=0.) # Make sure required is a number and not negative
        if not decodeOnly:
            o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(Decimal(required))+'/Decimal(\''+scale+'\')')
        if not decodeOnly: o.write('\n')
        return end

    offsetStr=''
    if offset != None:
        offsetStr += '+Decimal(\''+offset+'\')'

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('Decimal(int('+bv+'['+str(startindex)+':'+str(end)+']))/Decimal(\''+scale+'\')'+offsetStr+'')
    if not decodeOnly: o.write('\n')

    return end


def decodeBinary(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False,scale=None, decodeOnly=False):
    '''
    Build the decoder for unsigned decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'udecimal'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum.  If -1, then read to the end of the message
    @type required: Decimal or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='binary'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert (numbits>=1 and numbits <= 1024) or -1==numbits # What is good max?
    # FIX: assert not required and not an array an not unavailable


    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write(bv+'['+str(startindex)+':')
    if int(numbits) != -1: o.write(str(end))  # -1 means go to the end of the message
    o.write(']')
    if not decodeOnly: o.write('\n')

    return end



######################################################################
# THE REST
######################################################################



def buildTestParamFunc(o,msgET, verbose=False, prefixName=False):
    '''Scrape the testvalues to make a basic param

    @bug: FIX: make this create a dictionary that sits in the overall namespace and spit out deep copies?
    '''
    name = msgET.attrib['name']

    if prefixName: o.write('def '+name+'TestParams():\n')
    else: o.write('def testParams():\n')
    o.write("    '''Return a params file base on the testvalue tags.\n    @rtype: dict\n    @return: params based on testvalue tags\n    '''\n")
    o.write('    params = {}\n')
    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        if verbose: print 'buildTestParamFunc ...',name,type
        val = None
        if hasSubTag(field,'testvalue') and hasSubTag(field,'required'):
            print 'ERROR: can not have both test value and required tags in the same field'
            assert(False)
        if hasSubTag(field,'testvalue'):
            val = field.xpath('testvalue')[0].text
        else:
            if not hasSubTag(field,'required'):
                sys.exit("ERROR: missing required or testvalue for field: "+name)
            val = field.xpath('required')[0].text
        if verbose: print 'buildTestParamFunc for field '+name+' ...',type,val
        o.write('    params[\''+name+'\'] = ')
        if type=='bool':
            if val=='1' or val.lower=='true': val = 'True'
            else: val = 'False'
            o.write(val)
        elif type in ('uint','int','float'):
            o.write(val)
        elif type in ('decimal','udecimal'):
            o.write('Decimal(\''+val+'\')')

        elif type in ('aisstr6'):
            o.write('\''+val+'\'')
        elif type in ('binary'):
            o.write('BitVector(bitstring=\''+val+'\')')
        else:
            print 'ERROR: type not handled ...',type,'  (found in the ',name,' field).  Time to buy more coffee'
            suggestType(name,type)
            assert(False)

        o.write('\n')


    o.write('\n    return params\n\n')

def buildUnitTest(o,msgET, verbose=False, prefixName=False):
    '''
    Write the unittests for a message

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    '''
    assert(msgET.tag=='message')
    name = msgET.attrib['name']

    buildTestParamFunc(o,msgET, prefixName=prefixName)

    o.write('class Test'+name+'(unittest.TestCase):\n')
    o.write("    '''Use testvalue tag text from each type to build test case the "+name+" message'''\n")
    o.write('    def testEncodeDecode(self):\n\n')
    if prefixName:
        o.write('        params = '+name+'TestParams()\n')
        o.write('        bits   = '+name+'Encode(params)\n')
        o.write('        r      = '+name+'Decode(bits)\n\n')
    else:
        o.write('        params = testParams()\n')
        o.write('        bits   = encode(params)\n')
        o.write('        r      = decode(bits)\n\n')

    o.write('        # Check that each parameter came through ok.\n')
    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        if type in ('bool','uint','int','aisstr6','binary'):
            o.write('        self.failUnlessEqual(r[\''+name+'\'],params[\''+name+'\'])\n')
        else:
            # float, decimal, udecimal
            # FIX: look up the decimal places if decimal
            places = '3'
            if hasSubTag(field,'decimalplaces'): places = field.xpath('decimalplaces')[0].text
            o.write('        self.failUnlessAlmostEqual(r[\''+name+'\'],params[\''+name+'\'],'+places+')\n')



def buildEncode(o,msgET, verbose=False, prefixName=False):
    '''
    Write the encoder/decoder for a message

    http://jaynes.colorado.edu/PythonIdioms.html

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
     TODO(schwehr):handle ais message 20 optional.  very troublesome
    '''
    assert(msgET.tag=='message')
    name = msgET.attrib['name'] # fix rename this variable to avoid hiding later on by field name

    print 'Generating encoder ...',name # FIX: verbose?
    funcName = 'encode'
    if prefixName: funcName = name+'Encode'
    o.write('def '+funcName+'(params, validate=False):\n')

    ########################################
    # doc string
    o.write("    '''Create a "+name+" binary message payload to pack into an AIS Msg "+name+".\n\n")
    o.write('    Fields in params:\n')
    for field in msgET.xpath('field'):
        if verbose:
            print field.tag,field.attrib['name']
            print field.attrib
        desc = field[0].text.replace('\n',' ') # get ride of new lines
        o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description
        if len(field.xpath("required")) == 1:
            o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')

        o.write('\n')
    o.write('    @param params: Dictionary of field names/values.  Throws a ValueError exception if required is missing\n')
    o.write('    @param validate: Set to true to cause checking to occur.  Runs slower.  FIX: not implemented.\n')

    o.write("    @rtype: BitVector\n")
    o.write("    @return: encoded binary message (for binary messages, this needs to be wrapped in a msg 8\n")
    o.write("    @note: The returned bits may not be 6 bit aligned.  It is up to you to pad out the bits.\n")
    o.write("    '''\n\n")

    ########################################
    # Actually build the code

    o.write('    bvList = []\n')

    if verbose: print 'number of fields = ', len(msgET.xpath('field'))

    dynamicArrays = False # Set to true when positioning must be calculated

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',name,'('+type+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'
        else:
            if verbose: print 'Processing field ...',name,'(',type+' ',numbits,')'

        if   type=='bool'   : encodeBool   (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='uint'   : encodeUInt   (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='int'    : encodeInt    (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='float'  : encodeFloat  (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='aisstr6': encodeAisstr6(o,name,type,numbits,required,arraylen,unavailable)
        elif type=='decimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            encodeDecimal(o,name,type,numbits,required,arraylen,unavailable,scale=scale, offset=offset)
        elif type=='udecimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'):
                offset = field.xpath('offset')[0].text
            encodeUDecimal(o,name,type,numbits,required,arraylen,unavailable,scale=scale,offset=offset)
        elif type=='binary': encodeBinary(o,name,type,numbits,required,arraylen,unavailable)
        else:
            print 'WARNING: In buildEncode - Unhandled field type for',name,'...',type
            suggestType (name,type)
            assert False

#    o.write('\n    bv=binary.joinBV(bvList)\n')
#    o.write('\n    bvLen=len(bv)\n')
#    o.write('\n    if bvLen%6!=0:\n')
#    o.write('\n        bv = bv + BitVector(size=bvLen%6)\n')
#    o.write('\n    return bv\n\n')
    o.write('\n    return binary.joinBV(bvList)\n\n')


######################################################################
# DECODER ONE PART AT A TIME - if only using a small part, save some cycles!


#, msgDict=None):
def buildDecodeParts(o,msgET, verbose=False, prefixName=False):

    '''
    Write the decoder for a message

    @param o: open file where resulting code will be written
    @type msgET: elementtree
    @param prefixName: if True, put the name of the message on the functions.
    @param msgET: Element Tree starting at a message node
    @return: None

     TODO(schwehr):FIX: doc strings for each decode!
     TODO(schwehr):FIX: check for a dac,fid, or efid.  If exists, then this is an AIS Msg 8 payload
     TODO(schwehr):May want to take a dictionary of already decoded fields to speed things that need prior info
    for things like variable length arrays
    '''

    assert(msgET.tag=='message')
    name = msgET.attrib['name']

    print 'Generating partial decode functions ...',name

    baseName = name+'Decode'
    if not prefixName: baseName = 'decode'

    startindex = 0 # Where we are in the bitvector... FIX: what about variable length jobs?

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']

        o.write('def '+baseName+name+'(bv, validate=False):\n')
        # Follow the same convention of decoding into a dict so that code is the same
        #o.write('    r={};')
        o.write('    return ')

        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',name,'('+type+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'

        assert None!=startindex
        if verbose: print 'startindex',startindex
        if   type=='bool'   : startindex = decodeBool   (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='uint'   : startindex = decodeUInt   (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='int'    : startindex = decodeInt    (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='float'  : startindex = decodeFloat  (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='aisstr6': startindex = decodeAisstr6(o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='binary' : startindex = decodeBinary (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)

        elif type=='decimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,decodeOnly=True,offset=offset)
        elif type=='udecimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeUDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,decodeOnly=True,offset=offset)

        else:
            print 'WARNING: In buildDecode - Unhandled field type for',name,'...',type
            suggestType (name,type)
            assert False

        #o.write('    return(r[\''+name+'\'])\n\n')
        o.write('\n\n')


######################################################################
# DECODER RING

def buildDecode(o,msgET, verbose=False, prefixName=False):
    '''
    Write the decoder for a message

    @param o: open file where resulting code will be written
    @type msgET: elementtree
    @param msgET: Element Tree starting at a message node
    @return: None

     TODO(schwehr):FIX: check for a dac,fid, or efid.  If exists, then this is an AIS Msg 8 payload
    '''

    assert(msgET.tag=='message')
    name = msgET.attrib['name']

    print 'Generating decoder ...',name
    funcName = 'decode'
    if prefixName: funcName = name+'Decode'
    o.write('def '+funcName+'(bv, validate=False):\n')

    ########################################
    # doc string
    o.write("    '''Unpack a "+name+" message.\n\n")
    o.write('    Fields in params:\n')
    for field in msgET.xpath('field'):
        desc = field[0].text.replace('\n',' ') # get ride of new lines
        o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description
        if len(field.xpath("required")) == 1:
            o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')

        o.write('\n')
    o.write('    @type bv: BitVector\n')
    o.write('    @param bv: Bits defining a message\n')
    o.write('    @param validate: Set to true to cause checking to occur.  Runs slower.  FIX: not implemented.\n')

    o.write("    @rtype: dict\n")
    o.write("    @return: params\n")
    o.write("    '''\n\n")


    o.write('    #Would be nice to check the bit count here..\n')
    o.write('    #if validate:\n')
    o.write('    #    assert (len(bv)==FIX: SOME NUMBER)\n')


    ########################################
    # Actually build the code

    o.write('    r = {}\n')


    if verbose: print 'number of fields = ', len(msgET.xpath('field'))

    dynamicArrays = False # Set to true when positioning must be calculated

    startindex = 0 # Where we are in the bitvector... FIX: what about variable length jobs?

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',name,'('+type+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'
        else:
            if verbose: print 'Processing field ...',name,'(',type+' ',numbits,')'

        assert None!=startindex
        if verbose: print 'startindex',startindex

        if hasSubTag(field,'optional'):
            # NoReturn means that this field is not a point that needs a check to see if we are done
            # FIX: need to do something more efficient
            if msgET.attrib['aismsgnum']!='20':
                sys.exit('optional is only allow on AIS msg 20.  Do NOT use it on new messages!!!')
            text = field.xpath('optional')[0].text
            if None==field.xpath('optional')[0].text: # Found a NoReturn
                # Message 20 is a pain!
                pad = 8-(startindex%8)
                print 'not found: noreturn.  pad=',pad
                assert(pad<=6)
                o.write('    if len(bv)<='+str(startindex+pad)+': return r; # All fields below are optional\n')
            elif text != 'NoReturn':
                sys.exit ('ERROR: optional text must be NoReturn or empty')

        if   type=='bool'   : startindex = decodeBool   (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='uint'   : startindex = decodeUInt   (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='int'    : startindex = decodeInt    (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='float'  : startindex = decodeFloat  (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='aisstr6': startindex = decodeAisstr6(o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='binary' : startindex = decodeBinary (o,name,type,startindex,numbits,required,arraylen,unavailable)

        elif type=='decimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,offset=offset)

        elif type=='udecimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeUDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,offset=offset)

        else:
            print 'WARNING: In buildDecode - Unhandled field type for',name,'...',type
            suggestType (name,type)
            assert False


        if None==startindex: print 'FIX: here. drat.  treat me right'
        assert None!=startindex


    o.write('    return r\n\n')



#def getPythonType(aType):
#    '''
#    Translate a
#    @rtype: str
#    @return: name of the python type
#    '''

aisType2pythonType={
'bool':'Bool',
'uint':'int',
'int':'int',
'udecimal':'Decimal',
'decimal':'Decimal',
'aisstr6':'str',
'binary':'str',
'float':'Float',
}

import copy
aisType2optParseType=copy.deepcopy(aisType2pythonType)
aisType2optParseType['bool']='int' # FIX: make these appear as a flag variable
aisType2optParseType['aisstr6']='string'
aisType2optParseType['aisstr6']='string'
aisType2optParseType['binary']='string'
aisType2optParseType['udecimal']='string'  # FIX: Is this a good choice?
aisType2optParseType['decimal']='string'
aisType2optParseType['float']='float'


def buildOptParse(o,msgET, prefixName=False):
    '''Create a function that adds the options to a parse object'''

    assert None != msgET
    assert msgET.tag=='message'
    msgName = msgET.attrib['name']

    prefix=''
    if prefixName: prefix=msgName

    funcName = 'addMsgOptions'
    if prefixName: funcName = msgName + 'AddMsgOptions'
    o.write('\ndef '+funcName+'(parser):')

    # FIX: write out a doc string

    o.write('''
    parser.add_option('-d','--decode',dest='doDecode',default=False,action='store_true',
                help='decode a "'''+msgName+'''" AIS message')
    parser.add_option('-e','--encode',dest='doEncode',default=False,action='store_true',
                help='encode a "'''+msgName+'''" AIS message')
''')

    #print 'here...', msgName, prefixName
    for field in msgET.xpath('field'):
        name      = field.attrib['name']
        fieldType = field.attrib['type']
        if hasSubTag(field,'required'):
            print 'skipping required field ...',name,fieldType
            continue
        #print 'there',name,fieldType
        o.write('    parser.add_option(\'--')
        if prefixName: o.write(msgName+'-')
        o.write(name+'-field\', dest=\''+name+'Field\'')
        if hasSubTag(field,'unavailable'):
            val = field.xpath('unavailable')[0].text
            o.write(',default=')
            if fieldType in ('uint','int','float'):
                o.write(val)
            elif fieldType in ('decimal','udecimal'):
                o.write('Decimal(\''+val+'\')')
            elif fieldType in ('aisstr6','bitvector'):
                o.write('\''+val+'\'')
        o.write(',metavar=\''+fieldType+'\',type=\''+aisType2optParseType[fieldType]+'\'')
        o.write('\n        ,help=\'Field parameter value [default: %default]\')\n')



def buildMain(o, msgET, prefixName=False):
    assert None != msgET
    assert msgET.tag=='message'
    msgName = msgET.attrib['name']

    prefix=''
    if prefixName: prefix=msgName

    buildOptParse(o, msgET, prefixName)


    o.write('''
def main():
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options]")

    parser.add_option('--doc-test',dest='doctest',default=False,action='store_true',
        help='run the documentation tests')
    parser.add_option('--unit-test',dest='unittest',default=False,action='store_true',
        help='run the unit tests')
    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',
        help='Make the test output verbose')

    # FIX: remove nmea from binary messages.  No way to build the whole packet?
    # FIX: or build the surrounding msg 8 for a broadcast?
    typeChoices = ('binary','nmeapayload','nmea') # FIX: what about a USCG type message?
    parser.add_option('-t', '--type', choices=typeChoices, type='choice',
        dest='ioType', default='nmeapayload',
        help='What kind of string to write for encoding ('+', '.join(typeChoices)+') [default: %default]')


    outputChoices = ('std','html','csv','sql' ''')
#    'xml' -  FIX: need to add xml output
    if haveLocatableMessage(msgET): o.write(', \'kml\',\'kml-full\'')
    o.write(''')
    parser.add_option('-T', '--output-type', choices=outputChoices,
        type='choice', dest='outputType', default='std',
        help='What kind of string to output ('+', '.join(outputChoices)+') '
        '[default: %default]')

    parser.add_option('-o','--output',dest='outputFileName',default=None,
        help='Name of the python file to write [default: stdout]')

    parser.add_option('-f', '--fields', dest='fieldList', default=None,
        action='append', choices=fieldList,
        help='Which fields to include in the output.  Currently only for csv '
        'output [default: all]')

    parser.add_option('-p', '--print-csv-field-list', dest='printCsvfieldList',
        default=False,action='store_true',
        help='Print the field name for csv')

    parser.add_option('-c', '--sql-create', dest='sqlCreate', default=False,
        action='store_true',
        help='Print out an sql create command for the table.')

    parser.add_option('--latex-table', dest='latexDefinitionTable',
        default=False,action='store_true',
        help='Print a LaTeX table of the type')

    parser.add_option('--text-table', dest='textDefinitionTable', default=False,
        action='store_true',
        help='Print delimited table of the type (for Word table importing)')

    parser.add_option('--delimt-text-table', dest='delimTextDefinitionTable',
        default='    ',
        help='Delimiter for text table [default: \\\'%default\\\'] '
        '(for Word table importing)')

    dbChoices = ('sqlite','postgres')
    parser.add_option('-D', '--db-type', dest='dbType', default='postgres',
        choices=dbChoices,type='choice',
        help='What kind of database ('+', '.join(dbChoices)+') '
        '[default: %default]')
''')

    o.write('''
    addMsgOptions(parser)

    (options,args) = parser.parse_args()
    success = True

    if options.doctest:
            import os; print os.path.basename(sys.argv[0]), 'doctests ...',
            sys.argv = [sys.argv[0]]
            if options.verbose:
              sys.argv.append('-v')

            numfail, numtests = doctest.testmod()
            if not numfail:
                print 'ok'
            else:
                print 'FAILED'
                success = False

    if not success: sys.exit('Something Failed')
    del success # Hide success from epydoc

    if options.unittest:
            sys.argv = [sys.argv[0]]
            if options.verbose: sys.argv.append('-v')
            unittest.main()

    outfile = sys.stdout
    if None!=options.outputFileName:
            outfile = file(options.outputFileName,'w')

''')



    ##############################
    # encode
    ##############################
    # FIX: make everything tabs.  Grr.
    o.write('\n    if options.doEncode:\n')
    o.write('        # Make sure all non required options are specified.\n')
    for field in msgET.xpath('field'):
        name      = field.attrib['name']
        fieldType = field.attrib['type']
        varName = prefix + name + 'Field'
        if not hasSubTag(field,'required'):
            o.write('        if None==options.' + varName +
                    ': parser.error("missing value for ' + varName + '")\n')

    # Build dict
    o.write('    msgDict = {\n')
    for field in msgET.xpath('field'):
        name    = field.attrib['name']
        varName = prefix+name+'Field'
        if hasSubTag(field,'required'):
            o.write('        \''+name+'\': \''+field.xpath('required')[0].text+'\',\n')
        else:
            o.write('        \''+name+'\': options.'+varName+',\n')
    o.write('    }\n')

    encodeFunction = 'encode'
    if prefixName:
        encodeFunction = msgName + 'Encode'
    o.write('''
    bits = '''+encodeFunction+'''(msgDict)
    if 'binary' == options.ioType:
        print str(bits)
    elif 'nmeapayload'==options.ioType:
        # FIX: figure out if this might be necessary at compile time
        bitLen=len(bits)
        if bitLen % 6 != 0:
            bits = bits + BitVector(size=(6 - (bitLen%6)))  # Pad out to multiple of 6
        print binary.bitvectoais6(bits)[0]

    # FIX: Do not emit this option for the binary message payloads.  Does not make sense.
    elif 'nmea' == options.ioType:
        nmea = uscg.create_nmea(bits)
        print nmea
    else:
        sys.exit('ERROR: unknown ioType.  Help!')
''')


    ##############################
    # decode all
    ##############################
    decodeFunction = 'decode'
    printFields='printFields'
    if prefixName:
        decodeFunction = msgName+'Decode'
        printFields    = msgName+'PrintFields'
    o.write('''

        if options.sqlCreate:
                sqlCreateStr(outfile,options.fieldList,dbType=options.dbType)

        if options.latexDefinitionTable:
                latexDefinitionTable(outfile)

        # For conversion to word tables
        if options.textDefinitionTable:
                textDefinitionTable(outfile,options.delimTextDefinitionTable)

        if options.printCsvfieldList:
                # Make a csv separated list of fields that will be displayed for csv
                if None == options.fieldList: options.fieldList = fieldList
                import StringIO
                buf = StringIO.StringIO()
                for field in options.fieldList:
                        buf.write(field+',')
                result = buf.getvalue()
                if result[-1] == ',': print result[:-1]
                else: print result

        if options.doDecode:
                if len(args)==0: args = sys.stdin
                for msg in args:
                        bv = None

                        if msg[0] in ('$','!') and msg[3:6] in ('VDM','VDO'):
                                # Found nmea
                                # FIX: do checksum
                                bv = binary.ais6tobitvec(msg.split(',')[5])
                        else: # either binary or nmeapayload... expect mostly nmeapayloads
                                # assumes that an all 0 and 1 string can not be a nmeapayload
                                binaryMsg=True
                                for c in msg:
                                        if c not in ('0','1'):
                                                binaryMsg=False
                                                break
                                if binaryMsg:
                                        bv = BitVector(bitstring=msg)
                                else: # nmeapayload
                                        bv = binary.ais6tobitvec(msg)

                        '''+printFields+'''('''+decodeFunction+'''(bv)
                                    ,out=outfile
                                    ,format=options.outputType
                                    ,fieldList=options.fieldList
                                    ,dbType=options.dbType
                                    )

############################################################
if __name__=='__main__':
    main()
''')


######################################################################
if __name__=='__main__':
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options]")

    parser.add_option('-o','--output',dest='outputFileName',default=None,
                        help='Name of the python file to write')

    parser.add_option('-i','-x','--xml-definition',dest='xmlFileName',default=None,
                        help='XML definition file for the msg to use')

    parser.add_option('--doc-test',dest='doctest',default=False,action='store_true',
                        help='run the documentation tests')

    parser.add_option('-p','--prefix',dest='prefix',default=False,action='store_true',
                        help='put the field name in front of all function names.'
                        +'  Allows multiple messages in one file')

    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',
                        help='run the tests run in verbose mode')

    (options,args) = parser.parse_args()

    success=True

    if options.doctest:
        import os; print os.path.basename(sys.argv[0]), 'doctests ...',
        argvOrig = sys.argv
        sys.argv= [sys.argv[0]]
        if options.verbose: sys.argv.append('-v')
        import doctest
        numfail,numtests=doctest.testmod()
        if numfail==0: print 'ok'
        else:
            print 'FAILED'
            success=False
        sys.argv = argvOrig # Restore the original args
        del argvOrig # hide from epydoc
        sys.exit() # FIX: Will this exit success?

    #infile=sys.stdin
    #if options.xmlFileName: infile = file(options.xmlFileName,'r')

    #outfile=sys.stdout
    #if options.outputFile: outfile = file(options.xmlDefinition,'w')

    # FIX: down the road, it would be good to allow either filenames of std{in/out}

    if None==options.xmlFileName:
        sys.exit('ERROR: must specify an xml definition file.')
    if None==options.outputFileName:
        sys.exit('ERROR: must specify an python file to write to.')
    generatePython(options.xmlFileName,options.outputFileName,prefixName=options.prefix, verbose=options.verbose)

    print '\nrecommend running pychecker like this:'
    print '  pychecker -q',options.outputFileName

#!/usr/bin/env python
"""Tools to generate python code to serialize/deserialize messages
between python and ais binary.  Trying to be as inline as possible, so
no XML on the fly like in ais-py.

serialize: python to ais binary
deserialize: ais binary to python

The generated code uses translators.py, binary.py, and aisstring.py
which should be packaged with the resulting files.

@license: Apache 2.0

 TODO(schwehr):add a link to generated doc string to bring up the html for the pretty version
 TODO(schwehr):write a separate validation script that distinguishes standard messages and bin messages
 TODO(schwehr):make sure binary is only used in AIS ITU messages and not within the binary messages!
 TODO(schwehr):arrays
 TODO(schwehr):handle entry ranges in lookup tables
 TODO(schwehr):arraylength of -1 for messages 12 and 14
 TODO(schwehr):in the table html/kml form output, if no LUT and value is the unknown case, say so in the converted column!
 TODO(schwehr):stop using sqlhelp.  Can have trouble with corrupted data.
@bug: NOT complete
@note: This compiler assumes that all locations are in WGS 84 (EPSG 4326)
 TODO(schwehr):add type checking of command line parameters for crafting AIS messages
"""

import datetime
import os
import sys
from decimal import Decimal
from lxml import etree


def suggestType(name,curType,printout=True):
    '''Try to suggest a type name if one did not work.

    @param printout: if true, write a suggestion to stdout.

    >>> suggestType('myFieldName', 'unsigned int')
    Recommend switching "unsigned int" to "uint" for field "myFieldName"
    'uint'

    >>> suggestType('JohnWarfon', 'yoyodyne')
    Sorry!  No recommendation available for bad type "yoyodyne" for field "JohnWarfon"
    '''
    newType = None
    if curType.lower()=='unsigned int':
        newType = 'uint'
    elif curType.lower()=='unsigned decimal':
        newType = 'udecimal'

    if printout:
        if newType:
            print 'Recommend switching "'+curType+'" to "'+newType+'" for field "'+name+'"'
        else:
            print 'Sorry!  No recommendation available for bad type "'+curType+'" for field "'+name+'"'
    return newType


def hasSubTag(et,subtag):
    '''
    @return: true if the tag a sub tag with name subtag
    '''
    if 0<len(et.xpath(subtag)): return True
    return False


def writeBeginning(o):
    '''Write the doc string header for the message file.

    @param o: Open output file to write code to.
    Must be pre-expanded with the expandais.py command.
    '''

    d = datetime.datetime.utcnow()
    dateStr = str(d.year)+'-'+("%02d" %d.month)+'-'+("%02d"%d.day)

    # Need to pass in info about the source file, etc.

    # Minor trickery to get svn to ignore the keywords in the next few lines
    o.write('''#!/usr/bin/env python

\"\"\"Autogenerated python functions to serialize/deserialize binary messages.

Generated by: ''' + __file__ + '''

Need to then wrap these functions with the outer AIS packet and then
convert the whole binary blob to a NMEA string.  Those functions are
not currently provided in this file.

serialize: python to ais binary
deserialize: ais binary to python

The generated code uses translators.py, binary.py, and aisstring.py
which should be packaged with the resulting files.

''')

    o.write('''
 TODO(schwehr):FIX: put in a description of the message here with fields and types.
\"\"\"
import doctest
import sys
from decimal import Decimal
import unittest

from aisutils.BitVector import BitVector

from aisutils import aisstring
from aisutils import binary
from aisutils import sqlhelp
from aisutils import uscg

# FIX: check to see if these will be needed
TrueBV  = BitVector(bitstring="1")
"Why always rebuild the True bit?  This should speed things up a bunch"
FalseBV = BitVector(bitstring="0")
"Why always rebuild the False bit?  This should speed things up a bunch"


''')
    return

def generatePython(infile,outfile, prefixName=False,verbose=False):
    '''
    @param infile: xml ais binary message definition file
    @param outfile: where to dump the python code
    '''

    aisMsgsET = etree.parse(infile).getroot()

    o = file(outfile,'w')
    os.chmod(outfile,0755)

    writeBeginning(o)

    for msgET in aisMsgsET:
        if msgET.tag != 'message': continue
        print msgET.tag, msgET.attrib['name']

        if len(msgET.xpath('include-struct')) > 0:
            sys.exit("ERROR: cannot handle xml that still has include-struct tags.\n  Please use expandais.py.")
        buildHelpers(o,msgET,prefixName=prefixName,verbose=verbose)
        buildEncode(o,msgET,prefixName=prefixName,verbose=verbose)
        buildDecode(o,msgET,prefixName=prefixName)
        buildDecodeParts(o,msgET,prefixName=prefixName) # functions that only decode one field
        buildPrint(o,msgET,prefixName=prefixName)
        buildLUT(o,msgET,prefixName=prefixName)
        buildSQL(o,msgET,prefixName=prefixName)
        buildLaTeX(o,msgET,prefixName=prefixName,verbose=verbose)
        buildTextDef(o,msgET,prefixName=prefixName)

    o.write('\n\n######################################################################\n')
    o.write('# UNIT TESTING\n')
    o.write('######################################################################\n')

    for msgET in aisMsgsET:
        if msgET.tag != 'message': continue
        print 'Building unit tests for message ...', msgET.attrib['name']

        buildUnitTest(o,msgET, prefixName=prefixName)

    for msgET in aisMsgsET:
        if msgET.tag != 'message': continue
        buildMain(o,msgET, prefixName=prefixName)

    return

######################################################################
# Build Helpers
######################################################################

def buildHelpers(o,msgET, verbose=False, prefixName=False):
    '''
    emit the fieldList and other things???

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node

     TODO(schwehr):for lookuptable/entry values, make it also print the decoded value.
     TODO(schwehr):use a different name for message and field
     TODO(schwehr):put in comments with the python and sql data type for each field
    '''

    if verbose:
        msgname = msgET.attrib['name']
        print 'Building helpers ...',msgname

    if prefixName:
        o.write(prefixName,'FieldList = (\n')
    else:
        o.write('fieldList = (\n')

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        o.write('    \''+name+'\',\n')

    o.write(')\n\n')

    # FIX: add some documentation to the fieldList
    # FIX: like that it is used as the default csv list

    if prefixName:
        o.write(prefixName,'FieldListPostgres = (\n')
    else:
        o.write('fieldListPostgres = (\n')
    finished=[] # postgis names
    for field in msgET.xpath('field'):
        name = field.attrib['name']
        if 'postgisName' in field.attrib:
            name = field.attrib['postgisName']
            if name in finished: continue # already done, so skip
            finished.append(name)
            o.write('    \''+name+'\',    # PostGIS data type\n')
        else:
            o.write('    \''+name+'\',\n')

    o.write(')\n\n')

    del finished



    if prefixName:
        o.write(prefixName,'ToPgFields = {\n')
    else:
        o.write('toPgFields = {\n')
    for field in msgET.xpath('field'):
        if 'postgisName' not in field.attrib: continue
        name = field.attrib['name']
        pgName = field.attrib['postgisName']
        o.write('    \''+name+'\':\''+pgName+'\',\n')
    o.write('}\n')
    o.write('''\"\"\"
Go to the Postgis field names from the straight field name
\"\"\"

''')


    if prefixName:
        o.write(prefixName,'FromPgFields = {\n')
    else:
        o.write('fromPgFields = {\n')
    finished=[] # postgis names
    for field in msgET.xpath('field'):
        if 'postgisName' not in field.attrib: continue
        name = field.attrib['name']
        pgName = field.attrib['postgisName']

        if pgName in finished: continue # already done, so skip
        finished.append(pgName)
        o.write('    \''+pgName+'\':(')
        xp = 'field[@postgisName=\''+pgName+'\']'
        parts = ['\''+part.attrib['name']+'\'' for part in msgET.xpath(xp)]
        o.write(','.join(parts)+',),\n')
    o.write('}\n')
    o.write('''\"\"\"
Go from the Postgis field names to the straight field name
\"\"\"

''')


    if prefixName:
        o.write(prefixName,'PgTypes = {\n')
    else:
        o.write('pgTypes = {\n')
    finished=[] # postgis names
    for field in msgET.xpath('field'):
        if 'postgisName' not in field.attrib: continue
        pgType = field.attrib['postgisType']
        pgName = field.attrib['postgisName']
        if pgName in finished: continue # already done, so skip
        finished.append(pgName)
        o.write('    \''+pgName+'\':\''+pgType+'\',\n')
    o.write('}\n')
    o.write('''\"\"\"
Lookup table for each postgis field name to get its type.
\"\"\"

''')







######################################################################
# SIMPLE PRINT
######################################################################

def getMaxFieldNameLen(msgET):
    '''Get the maximum string length of any field name'''
    maxStrLen=0
    for field in msgET.xpath('field'):
        fieldLen = len(field.attrib['name'])
        if fieldLen>maxStrLen: maxStrLen = fieldLen
    return maxStrLen

def padStrRight(aStr,strlen):
    '''Pad a string out to the length requested with spaces out to the right'''
    return aStr + ' '*(strlen-len(aStr))

def haveLocatableMessage(msgET):
    '''Make sure this message has both long/x and lat/y fields.
    @rtype: bool
    '''
    #if getLongitudeFieldName(msgET) and getLatitudeFieldName(msgET): return True

    if 0==len(msgET.xpath('field[contains(@name, "longitude")]')): return False
    if 0==len(msgET.xpath('field[contains(@name, "latitude")]')): return False
    return True

def getLongitudeFieldName(msgET):
    '''
    Dig up the first field name that include longitude and return it
     TODO(schwehr):might want to allow a search for a special tag to mark this
    '''
    return msgET.xpath('field[contains(@name, "longitude")]')[0].attrib['name']

def getLatitudeFieldName(msgET):
    '''
    Dig up the first field name that include longitude and return it
     TODO(schwehr):might want to allow a search for a special tag to mark this
    '''
    return msgET.xpath('field[contains(@name, "latitude")]')[0].attrib['name']

def buildPrint(o,msgET, verbose=False, prefixName=False):
    '''
    Write a simple in order print for the resulting dictionary.

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node

     TODO(schwehr):for lookuptable/entry values, make it also print the decoded value.
     TODO(schwehr):use a different name for message and field
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    #msgName = msgname = name  # FIX: make these all msgName

    print 'Generating print ...',msgName # FIX: verbose?
    # +1 for the ':'
    maxFieldLen = 1 + getMaxFieldNameLen(msgET)


    ##############################
    # Html builder
    ##############################

    printHtmlName = 'printHtml'
    if prefixName: printHtmlName = name+'printHtml'




    ####################
    ####### HTML format
    ####################
    #o.write('    elif \'html\'==format:\n')
    o.write('\n')
    o.write('def '+printHtmlName+'(params, out=sys.stdout):\n')
    o.write('        out.write("<h3>'+msgName+'</h3>\\n")\n')
    o.write('        out.write("<table border=\\"1\\">\\n")\n')
    #o.write('        out.write("<tr bgcolor=\\"#9acd32\\">\\n")\n')
    o.write('        out.write("<tr bgcolor=\\"orange\\">\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Field Name</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Type</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Value</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Value in Lookup Table</th>\\n")\n')
    o.write('        out.write("<th align=\\"left\\">Units</th>\\n")\n')
    o.write('        out.write("</tr>\\n")\n')


    for field in msgET.xpath('field'):
        o.write('        out.write("\\n")\n')
        o.write('        out.write("<tr>\\n")\n')
        fieldname = field.attrib['name']
        fieldtype = field.attrib['type']
        o.write('        out.write("<td>'+fieldname+'</td>\\n")\n')
        o.write('        out.write("<td>'+fieldtype+'</td>\\n")\n')

        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:  arraylen=int(field.attrib['arraylength'])

        if 1==arraylen or fieldtype=='aisstr6':
            o.write('        if \''+fieldname+'\' in params:\n            out.write("    <td>"+str(params[\''+fieldname+'\'])+"</td>\\n")\n')
            if not hasSubTag(field,'lookuptable'):
                # Just duplicate the value through
                o.write('            out.write("    <td>"+str(params[\''+fieldname+'\'])+"</td>\\n")\n')
            else:
                lutName = fieldname+'DecodeLut'
                if prefixName: lutName = msgname+fieldname.capitalize()+'DecodeLut'

                o.write('            if str(params[\''+fieldname+'\']) in '+lutName+':\n')
                o.write('                out.write("<td>"+'+lutName+'[str(params[\''+fieldname+'\'])]+"</td>")\n')
                o.write('            else:\n')
                o.write('                out.write("<td><i>Missing LUT entry</i></td>")\n')
        else: sys.exit ('FIX: handle arrays in the buildPrint func')

        if hasSubTag(field,'units'):
            o.write('        out.write("<td>'+field.xpath('units')[0].text+'</td>\\n")\n')

        o.write('        out.write("</tr>\\n")\n')

    o.write('        out.write("</table>\\n")\n')
    o.write('\n')

    ##############################
    ####### KML GoogleEarth
    ##############################

    printKmlName = 'printKml'
    if prefixName: printKmlName = msgName+'printKml'
    if not haveLocatableMessage(msgET): printKmlName=None
    else:
        # FIX: use the first postgis print tagged lat and lon field
        o.write('\n')
        o.write('def printKml(params, out=sys.stdout):\n')
        o.write('    \"\"\"KML (Keyhole Markup Language) for Google Earth, but without the header/footer\"\"\"\n')
        o.write('    out.write("    <Placemark>\\n")\n')
        o.write('    out.write("        <name>"+str(params[\''+msgET.attrib['titlefield']+'\'])+"</name>\\n")\n')
        o.write('    out.write("        <description>\\n")\n')

        o.write('    import StringIO\n')
        o.write('    buf = StringIO.StringIO()\n')
        o.write('    printHtml(params,buf)\n')
        o.write('    import cgi\n')
        o.write('    out.write(cgi.escape(buf.getvalue()))\n')

        o.write('    out.write("        </description>\\n")\n')
        o.write('    out.write("        <styleUrl>#m_ylw-pushpin_copy0</styleUrl>\\n")\n')
        o.write('    out.write("        <Point>\\n")\n')
        o.write('    out.write("            <coordinates>")\n')
        o.write('    out.write(str(params[\''+getLongitudeFieldName(msgET)+'\']))\n')
        o.write('    out.write(\',\')\n')
        o.write('    out.write(str(params[\''+getLatitudeFieldName(msgET)+'\']))\n')
        o.write('    out.write(",0</coordinates>\\n")\n')
        o.write('    out.write("        </Point>\\n")\n')
        o.write('    out.write("    </Placemark>\\n")\n')
        o.write('\n')


    ##############################
    ##### Main print dispatch
    ##############################

    funcName = 'printFields'
    if prefixName: funcName = msgName+'PrintFields'

    o.write('def '+funcName+'(params, out=sys.stdout, format=\'std\', fieldList=None, dbType=\'postgres\'):\n')

    ########################################
    # doc string
    o.write("    '''Print a "+msgName+" message to stdout.\n\n")
    o.write('    Fields in params:\n')
    for field in msgET.xpath('field'):
        desc = field[0].text.replace('\n',' ') # get ride of new lines
        o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description
        if len(field.xpath("required")) == 1:
            o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')

        o.write('\n')
    o.write('    @param params: Dictionary of field names/values.\n')
    o.write('    @param out: File like object to write to.\n')

    o.write("    @rtype: stdout\n")
    o.write("    @return: text to out\n")
    o.write("    '''\n\n")

    ########################################
    # Actually build the code

    o.write('    if \'std\'==format:\n')
    o.write('        out.write("'+msgName+':\\n")\n')

    if verbose: print 'number of fields = ', len(msgET.xpath('field'))


    for field in msgET.xpath('field'):
        fieldname = field.attrib['name']
        fieldtype = field.attrib['type']
        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',fieldname,'('+fieldtype+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'
        else:
            if verbose: print 'Processing field ...',fieldname,'(',fieldtype+' ',numbits,')'

        if 1==arraylen or fieldtype=='aisstr6':
            o.write('        if \''+fieldname+'\' in params: out.write("    '+padStrRight(fieldname+':',maxFieldLen)+'  "+str(params[\''+fieldname+'\'])+"\\n")\n')
            # FIX: elif aisstr6 strip @@@ and then print
        else:
            print '\n','ERROR:',fieldname,fieldtype
            sys.exit ('ERROR: FIX: handle arrays in the buildPrint func')

    ####################
    ####### Comma separated values (csv)
    ####################
    o.write('''        elif 'csv'==format:
                if None == options.fieldList:
                        options.fieldList = fieldList
                needComma = False;
                for field in fieldList:
                        if needComma: out.write(',')
                        needComma = True
                        if field in params:
                                out.write(str(params[field]))
                        # else: leave it empty
                out.write("\\n")
''')

    ####################
    ####### Other dispatchers
    ####################
    o.write('    elif \'html\'==format:\n')
    o.write('        '+printHtmlName+'(params,out)\n')

    o.write('''    elif 'sql'==format:
                sqlInsertStr(params,out,dbType=dbType)
''')

    if haveLocatableMessage(msgET):
        # Has a lon/lat pair somewhere in there.
        o.write('    elif \'kml\'==format:\n')
        o.write('        '+printKmlName+'(params,out)\n')

        o.write('    elif \'kml-full\'==format:\n')

        o.write('        out.write(\"<?xml version=\\"1.0\\" encoding=\\"UTF-8\\"?>\\n")\n')
        o.write('        out.write("<kml xmlns=\\"http://earth.google.com/kml/2.1\\">\\n")\n')
        o.write('        out.write("<Document>\\n")\n')
#        o.write('        out.write("    <name>Position</name>\\n")\n')
        o.write('        out.write("    <name>'+msgName+'</name>\\n")\n')

        o.write('        '+printKmlName+'(params,out)\n')

        o.write('        out.write("</Document>\\n")\n')
        o.write('        out.write("</kml>\\n")\n')



    ####################
    ####### Make safety check
    ####################
    o.write('    else:\n')
    o.write('        print "ERROR: unknown format:",format\n')
    o.write('        assert False\n')

    o.write('\n    return # Nothing to return\n\n')


######################################################################
# Build Lookup tables for enumerated int or uint fields
######################################################################


######################################################################
# DELIMITED TEXT FILE
######################################################################

def buildTextDef(o,msgET, verbose=False, prefixName=False):
    '''
    Write functions for text definition output

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    @param verbose: talk lots in the process
    @param prefixName: set to a string to have the commands prefixed by that character.

     TODO(schwehr):should this be a style sheet thing instead?
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    print 'Generating text table definition ...', msgName

    createFuncName = 'textDefinitionTable'
    if prefixName: createFuncName = msgName+'textDefinitionTable'

    o.write('''\n######################################################################
# Text Definition
######################################################################

''')

    # Should this really default to true for the uscg fields?
    o.write('def '+createFuncName+'''(outfile=sys.stdout ,delim='    '):
    \"\"\"Return the text definition table for this message type

    @param outfile: file like object to print to.
    @type outfile: file obj
    @return: text table string via the outfile
    @rtype: str

    \"\"\"
    o = outfile
''')

    # NOTE: using / to mean one  in the output compiled code
    o.write('    o.write(\'Parameter\'+delim+\'Number of bits\'+delim+\"\"\"Description\n')

    lines = []
    totalBits = 0
    for field in msgET.xpath('field'):
        fieldName = field.attrib['name']
        fieldType = field.attrib['type']
        fieldBits = field.attrib['numberofbits']
        if 'arraylength' in field.attrib:
            fieldBits = str(int(fieldBits) * int(field.attrib['arraylength']))
        totalBits += int(fieldBits)
        # FIX: maybe cut down to the first sentence?
        fieldDescription = field.xpath('description')[0].text.replace('\n',' ')
        lines.append(fieldName+'\"\"\"+delim+\''+fieldBits+'\'+delim+\"\"\"'+fieldDescription)

    o.write('\n'.join(lines))
    # 168 in the first slot and following slots get to use the whole 256??
    # FIX: is this right?
    from math import ceil
    numSlots = 1 + int( ceil((totalBits-168)/256.) )
    emptySlotBits = ( 168 + (numSlots-1)*256 ) - totalBits
    s = ''
    if numSlots>1: s = 's'
    o.write(('\nTotal bits\"\"\"+delim+\"\"\"'+str(totalBits)+'\"\"\"+delim+\"\"\"Appears to take '+str(numSlots)+' slot'+s))
    if emptySlotBits > 0:
        o.write(' with '+str(emptySlotBits)+' pad bits to fill the last slot')

    o.write( ('''\"\"\")
'''))

    # FIX: should it return a copy of outfile?


######################################################################
def buildLaTeX(o,msgET, verbose=False, prefixName=False):
    '''
    Write functions for LaTeX output

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    @param verbose: talk lots in the process
    @param prefixName: set to a string to have the commands prefixed by that character.

     TODO(schwehr):should this be a style sheet thing instead?
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    print 'Generating LaTeX definition table ...', msgName

    createFuncName = 'latexDefinitionTable'
    if prefixName: createFuncName = msgName+'LaTeXDefinitionTable'

    o.write('''\n######################################################################
# LATEX SUPPORT
######################################################################

''')

    # Should this really default to true for the uscg fields?
    o.write('def '+createFuncName+'''(outfile=sys.stdout
                ):
        \"\"\"
        Return the LaTeX definition table for this message type
        @param outfile: file like object to print to.
        @type outfile: file obj
        @return: LaTeX table string via the outfile
        @rtype: str

        \"\"\"
        o = outfile
''')

    # NOTE: using / to mean one  in the output compiled code
    o.write('''
        o.write(\"\"\"
/begin{table}%[htb]
/centering
/begin{tabular}{|l|c|l|}
/hline
Parameter & Number of bits & Description
//  /hline/hline
'''.replace('/','\\\\'))

    lines = []
    totalBits = 0
    for field in msgET.xpath('field'):
        fieldName = field.attrib['name'].replace('_','\_')  # _ means subscript to latex
        fieldType = field.attrib['type']
        fieldBits = field.attrib['numberofbits']
        if 'arraylength' in field.attrib:
            fieldBits = str(int(fieldBits) * int(field.attrib['arraylength']))
        totalBits += int(fieldBits)
        # FIX: maybe cut down to the first sentence?
        fieldDescription = field.xpath('description')[0].text.replace('\n',' ')
        lines.append(fieldName+' & '+fieldBits+' & '+fieldDescription)

    o.write(' \\\\\\\\ \\hline\n'.join(lines))
    # 168 in the first slot and following slots get to use the whole 256??  FIX: is this right?
    from math import ceil
    numSlots = 1 + int( ceil((totalBits-168)/256.) )
    emptySlotBits = ( 168 + (numSlots-1)*256 ) - totalBits
    s = ''
    if numSlots>1: s = 's'
    o.write(('// /hline /hline\nTotal bits & '+str(totalBits)+' & Appears to take '+str(numSlots)+' slot'+s).replace('/','\\\\'))
    if emptySlotBits > 0:
        o.write(' with '+str(emptySlotBits)+' pad bits to fill the last slot')

#    if verbose:
#        sys.stderr.write (' msgnum:'+msgET.attrib['aismsgnum'] +' - ')
#        sys.stderr.write (' desc: '+ str(msgET.xpath('description')[0])+' - ')
#        sys.stderr.write (' name: '+ str(msgET.attrib['name'])+'\n')
        #sys.stderr.write (' :'+ str()+'\n')

    o.write( (''' // /hline
/end{tabular}
/caption{AIS message number '''+msgET.attrib['aismsgnum']+': '+msgET.xpath('description')[0].text.replace('\n',' ')+'''}
/label{tab:'''+msgET.attrib['name']+'''}
/end{table}
\"\"\")
''').replace('/','\\\\'))

    # FIX: should it return a copy of outfile?


######################################################################

def buildSQL(o,msgET, verbose=False, prefixName=False):
    '''
    Write SQL code

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    @param verbose: talk lots in the process
    @param prefixName: set to a string to have the commands prefixed by that character.
    '''
    assert(msgET.tag=='message')
    msgName = msgET.attrib['name']
    print 'Generating SQL commands ...', msgName

    createFuncName = 'sqlCreate'
    if prefixName: createFuncName = msgName+'SqlCreate'
    insertFuncName = 'sqlInsert'
    if prefixName: insertFuncName = msgName+'SqlInsert'

    o.write('''######################################################################
# SQL SUPPORT
######################################################################

''')

    o.write('dbTableName=\''+msgName+'\'\n')
    o.write('\'Database table name\'\n\n')

    # Should this really default to true for the uscg fields?
    o.write('def '+createFuncName+'''Str(outfile=sys.stdout, fields=None, extraFields=None
                ,addCoastGuardFields=True
                ,dbType='postgres'
                ):
        \"\"\"
        Return the SQL CREATE command for this message type
        @param outfile: file like object to print to.
        @param fields: which fields to put in the create.  Defaults to all.
        @param extraFields: A sequence of tuples containing (name,sql type) for additional fields
        @param addCoastGuardFields: Add the extra fields that come after the NMEA check some from the USCG N-AIS format
        @param dbType: Which flavor of database we are using so that the create is tailored ('sqlite' or 'postgres')
        @type addCoastGuardFields: bool
        @return: sql create string
        @rtype: str

        @see: sqlCreate
        \"\"\"
        # FIX: should this sqlCreate be the same as in LaTeX (createFuncName) rather than hard coded?
        outfile.write(str(sqlCreate(fields,extraFields,addCoastGuardFields,dbType=dbType)))

''')

    o.write('def '+createFuncName+'''(fields=None, extraFields=None, addCoastGuardFields=True, dbType='postgres'):
    \"\"\"Return the sqlhelp object to create the table.

    @param fields: which fields to put in the create.  Defaults to all.
    @param extraFields: A sequence of tuples containing (name,sql type) for additional fields
    @param addCoastGuardFields: Add the extra fields that come after the NMEA check some from the USCG N-AIS format
    @type addCoastGuardFields: bool
    @param dbType: Which flavor of database we are using so that the create is tailored ('sqlite' or 'postgres')
    @return: An object that can be used to generate a return
    @rtype: sqlhelp.create
    \"\"\"
    if fields is None:
        fields = fieldList
''')

    o.write('    c = sqlhelp.create(\''+msgName+'\',dbType=dbType)\n')
    o.write('    c.addPrimaryKey()\n');

    for field in msgET.xpath('field'):
        fieldName = field.attrib['name']
        fieldType = field.attrib['type']

        postgis = False
        if 'postgisType' in field.attrib:
            postgis=True
            o.write('    if dbType != \'postgres\':\n    ')
        o.write('    if \''+fieldName+'\' in fields: c.add')
        # FIX: add the ADD command here.
        if   fieldType in ('int','uint'): o.write('Int (\''+fieldName+'\')')
        elif fieldType in ('float'     ): o.write('Real(\''+fieldName+'\')')
        elif fieldType == 'bool':         o.write('Bool(\''+fieldName+'\')')
        elif fieldType in ('decimal','udecimal'):
            if not hasSubTag(field,'decimalplaces'):
                print '\n ** ERROR: missing decimalplaces field for ',fieldName
                print
                assert (False)
            scaleSQL = int(field.xpath('decimalplaces')[0].text)  # number of decimal places
            numBits = int(field.attrib['numberofbits'])
            if not hasSubTag(field,'scale'):
                print 'ERROR: missing required <scale> for',fieldName,'of type',fieldType
            scaleVal = float(field.xpath('scale')[0].text)
            precision = scaleSQL + len(str(int((2**numBits)/scaleVal)))
            #precision += 1 # FIX: do I really need this safety factor
            o.write('Decimal(\''+fieldName+'\','+str(precision)+','+str(scaleSQL)+')')
        elif fieldType == 'binary':
            numBits = int(field.attrib['numberofbits'])
            if -1 == numBits: numBits=1024 # FIX: what is a good maximum for AIS?
            o.write('BitVarying(\''+fieldName+'\','+str(numBits)+')')
        elif fieldType == 'aisstr6':
            arrayLength = int(field.attrib['arraylength'])
            o.write('VarChar(\''+fieldName+'\','+str(arrayLength)+')')
        else:
            print '\n\n *** Unknown type in SQL code generation for field',fieldName+':',fieldType,'\n\n'
            assert(False)
        o.write('\n')

    o.write('''
    if addCoastGuardFields:
        # c.addInt('cg_s_rssi')  # Relative signal strength indicator
        # c.addInt('cg_d_strength')  # dBm receive strength
        # c.addVarChar('cg_x',10)  # Idonno
        c.addInt('cg_t_arrival')  # Receive timestamp from the AIS equipment 'T'
        c.addInt('cg_s_slotnum')  # Slot received in
        c.addVarChar('cg_r',15)  # Receiver station ID  -  should usually be an MMSI, but sometimes is a string
        c.addInt('cg_sec')  # UTC seconds since the epoch

        c.addTimestamp('cg_timestamp') # UTC decoded cg_sec - not actually in the data stream
''')


    #
    # Postgres with PostGIS added
    #
    postgisFields = msgET.xpath('field[@postgisType]')
    if len(postgisFields)<1:
        if verbose: print 'No postgis fields'
    else:
        o.write('\n    if dbType == \'postgres\':\n')
        finished=[] # Should be a set?
        print 'processing postgis fields for create:'
        for field in postgisFields:
            #print '  ',field.attrib['name'],field.attrib['postgisName']
            pgName = field.attrib['postgisName']
            if pgName in finished:
                print 'already handled',pgName
                continue
            finished.append(pgName)
            pgType = field.attrib['postgisType']
            #print '   doing',pgName,pgType
            components = msgET.xpath('field[@postgisName=\''+pgName+'\']')
            #print '    all components'
            #for c in components:
            #    print '     ',c.attrib['postgisName']
            o.write('        #--- EPSG 4326 : WGS 84\n')

            o.write('''        #INSERT INTO "spatial_ref_sys" ("srid","auth_name","auth_srid","srtext","proj4text") VALUES (4326,'EPSG',4326,'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]','+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs ');\n''')

            o.write('        c.addPostGIS(\''+pgName+'\',\''+pgType+'\','+str(len(components))+',SRID=4326);\n')

    o.write('\n    return c\n\n')


    # FIX: function name
    o.write('''def '''+insertFuncName+'''Str(params, outfile=sys.stdout, extraParams=None, dbType='postgres'):
        \"\"\"
        Return the SQL INSERT command for this message type
        @param params: dictionary of values keyed by field name
        @param outfile: file like object to print to.
        @param extraParams: A sequence of tuples containing (name,sql type) for additional fields
        @return: sql create string
        @rtype: str

        @see: sqlCreate
        \"\"\"
        outfile.write(str(sqlInsert(params,extraParams,dbType=dbType)))


''')

    # FIX: function name
    o.write('def ' + insertFuncName + """(params,extraParams=None,dbType='postgres'):
        \"\"\"
        Give the SQL INSERT statement
        @param params: dict keyed by field name of values
        @param extraParams: any extra fields that you have created beyond the normal ais message fields
        @rtype: sqlhelp.insert
        @return: insert class instance
         TODO(schwehr):allow optional type checking of params?
        @warning: this will take invalid keys happily and do what???
        \"\"\"

        i = sqlhelp.insert('""" + msgName + """',dbType=dbType)

        if dbType=='postgres':
                finished = []
                for key in params:
                        if key in finished:
                                continue

                        if key not in toPgFields and key not in fromPgFields:
                                if type(params[key])==Decimal: i.add(key,float(params[key]))
                                else: i.add(key,params[key])
                        else:
                                if key in fromPgFields:
                                        val = params[key]
                                        # Had better be a WKT type like POINT(-88.1 30.321)
                                        i.addPostGIS(key,val)
                                        finished.append(key)
                                else:
                                        # Need to construct the type.
                                        pgName = toPgFields[key]
                                        #valStr='GeomFromText(\\''+pgTypes[pgName]+'('
                                        valStr=pgTypes[pgName]+'('
                                        vals = []
                                        for nonPgKey in fromPgFields[pgName]:
                                                vals.append(str(params[nonPgKey]))
                                                finished.append(nonPgKey)
                                        valStr+=' '.join(vals)+')'
                                        i.addPostGIS(pgName,valStr)
        else:
                for key in params:
                        if type(params[key])==Decimal: i.add(key,float(params[key]))
                        else: i.add(key,params[key])

        if None != extraParams:
                for key in extraParams:
                        i.add(key,extraParams[key])

        return i
""")



######################################################################
# Build Lookup tables for enumerated int or uint fields
######################################################################


def buildLUT(o,msgET, verbose=False, prefixName=False):
    '''
    Write lookup tables for enumerated types (uint or int, maybe bool too).

     TODO(schwehr):FIX: what to do about multiple entries with the same text?  Need to ban that kind of thing
     TODO(schwehr):Make doc strings for each LUT.

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    '''
    assert(msgET.tag=='message')
    msgname = msgET.attrib['name']

    print 'Generating lookup tables ...',msgname # FIX: verbose?

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        if not hasSubTag(field,'lookuptable'): continue
        lut = field.xpath('lookuptable')[0]

        lutName = name
        if prefixName: lutName = msgname+name.capitalize()

        o.write(lutName+'EncodeLut = {\n')
        for entry in lut.xpath('entry'):
            o.write('    \''+entry.text+'\':\''+entry.attrib['key']+'\',\n')
        o.write('    } #'+lutName+'EncodeLut\n')
        o.write('\n')

        # FIX: make doc string for LUT here

        o.write(lutName+'DecodeLut = {\n')
        for entry in lut.xpath('entry'):
            o.write('    \''+entry.attrib['key']+'\':\''+entry.text+'\',\n')
        o.write('    } # '+lutName+'EncodeLut\n')
        o.write('\n')

        # FIX: make doc string for LUT here




######################################################################
# ENCODERS
######################################################################
def encodeBool(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for boolean variables
    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: bool, etc.
    @type numbits: int = 1
    @param numbits: How many bits per unit datum (must be 1 for bools)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many bools will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''

    if verbose: print 'bool encode',name,': unvail=',unavailable

    assert type.lower()=='bool'
    assert numbits==1
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type=bool)\n')
    if None != required:
        assert type(required) == bool
        if required: o.write('        bvList.append(TrueBV)\n')
        else: o.write('        bvList.append(FalseBV)\n')
        if verbose: o.write('\n')
        return

    if not unavailable:
        o.write('    if params["' + name + '"]: bvList.append(TrueBV)\n')
        o.write('    else: bvList.append(FalseBV)\n')
    else: # Have a default value that can be filled in
        assert type(unavailable) == bool
        o.write("    if '" + name + "' in params:\n")
        o.write('        if params["' + name + '"]: bvList.append(TrueBV)\n')
        o.write('        else: bvList.append(FalseBV)\n')
        o.write('    else:\n')
        if unavailable: o.write('        bvList.append(TrueBV)\n')
        else: o.write('        bvList.append(FalseBV)\n')
    if verbose: o.write('\n')

def encodeUInt(o, name, type, numbits, required=None, arraylen=1,
               unavailable=None, verbose=False):
    '''Build the encoder for unsigned integer variables.

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many unsigned ints will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose:
        print '  encodeUInt:', name, type, numbits, 'Req:',required,
        print 'alen:', arraylen, unavailable

    assert type == 'uint'
    assert numbits >= 1 and numbits <= 32
    if arraylen != 1:
        assert False # FIX... handle arrays
    if verbose:
        o.write('    ### FIELD: '+name+' (type='+type+')\n')

    if None != required:
        if verbose:
            print '  required:', required
        required=int(required)
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal='+str(required)+'),'+str(numbits)+'))\n')
        if verbose:
            o.write('\n')
        return

    if not unavailable:
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal=params[\''+name+'\']),'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        int(unavailable) # Make sure unavailable is a number object
        o.write("    if '" + name + "' in params:\n")
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal=params[\''+name+'\']'+'),'+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal='+str(unavailable)+'),'+str(numbits)+'))\n')

    if verbose: o.write('\n')


def encodeFloat(o,name,fieldType,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for IEEE float variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type fieldType: str
    @param fieldType: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many unsigned ints will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeUInt:',name,fieldType,numbits,'Req:',required,'alen:',arraylen,unavailable

    assert numbits==32 # Force by the IEEE spec
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+fieldType+')\n')

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(binary.float2bitvec('+str(required)+'))\n')
        if verbose: o.write('\n')
        return

    if not unavailable:
        o.write('    bvList.append(binary.float2bitvec(params[\''+name+'\']))\n')
    else: # Have a default value that can be filled in
        int(unavailable) # Make sure unavailable is a number object
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.float2bitvec(params[\''+name+'\']'+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.float2bitvec('+str(unavailable)+'))\n')

    if verbose: o.write('\n')


def encodeAisstr6(o,name,fieldType,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for aisstr6 variables.  Generally are arrays.
    @bug: do we need to optionally check for a valid string?

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type fieldType: str
    @param fieldType: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many unsigned ints will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeUInt:',name,fieldType,numbits,'Req:',required,'alen:',arraylen,unavailable
    totLen = str(numbits*arraylen)
    assert numbits==6 # Each character must be 6 bits
    if verbose: o.write('    ### FIELD: '+name+' (type='+fieldType+')\n')

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(aisstring.encode(\''+str(required)+'\','+totLen+'))\n')
        if verbose: o.write('\n')
        return

    if not unavailable:
        o.write('    bvList.append(aisstring.encode(params[\''+name+'\'],'+totLen+'))\n')
    else: # Have a default value that can be filled in
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(aisstring.encode(params[\''+name+'\'],'+totLen+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(aisstring.encode(\''+str(unavailable)+'\','+totLen+'))\n')

    if verbose: o.write('\n')


def encodeInt(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False):
    '''
    Build the encoder for signed integer variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, bool, etc.
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many signed ints will there be?  FIX: handle variable
    @type unavailable: number or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeInt:',name,type,numbits,'Req:',required,'alen:',arraylen,unavailable

    assert numbits>=1 and numbits<=32
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(binary.bvFromSignedInt('+str(required)+','+str(numbits)+'))\n')
        if verbose: o.write('\n')
        return


    if not unavailable:
        o.write('    bvList.append(binary.bvFromSignedInt(params[\''+name+'\'],'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        #assert type(unavailable)==
        int(unavailable) # Make sure unavailable is a number object
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.bvFromSignedInt(params[\''+name+'\']'+','+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.bvFromSignedInt('+str(unavailable)+','+str(numbits)+'))\n')

    if verbose: o.write('\n')



# FIX: Ummm... why am I passing the type?  I guess it makes the one print statement easier
def encodeDecimal(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False, scale=None, offset=None):
    '''
    Build the encoder for signed decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: decimal
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many decimals will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeDecimal:',name,type,numbits,'Req:',required,'alen:',arraylen,unavailable

    assert numbits>=1 and numbits<=32
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    # FIX: optimize to not emit the scaling when it is not needed... or tell the user to switch to an int!
    if None == scale:
        print 'WARNING: if you are not scaling, then you probably want to use an int instead!'
        print 'Beware canadians bearing travel videos'
        scale='1'

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        o.write('    bvList.append(binary.bvFromSignedInt('+str(int(Decimal(required)*Decimal(scale)))+','+str(numbits)+'))\n')
        if verbose: o.write('\n')
        return

    offsetStr=''
    if None != offset:
        offsetStr='-('+offset+')'

    # FIX: can I get rid of the Decimal around params?
    if not unavailable:
        o.write('    bvList.append(binary.bvFromSignedInt(int(Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')),'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.bvFromSignedInt(int(Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')),'+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.bvFromSignedInt('+str(int(Decimal(unavailable)*Decimal(scale)))+','+str(numbits)+'))\n')

    if verbose: o.write('\n')



def encodeUDecimal(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False, scale=None, offset=None):
    '''
    Build the encoder for signed decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: decimal
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..32)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many decimals will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encodeDecimal:',name,type,numbits,'Req:',required,'alen:',arraylen,unavailable
    assert type=='udecimal'
    assert numbits>=1 and numbits<=32
    if arraylen != 1: assert False # FIX... handle arrays
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    # FIX: optimize to not emit the scaling when it is not needed... or tell the user to switch to an int!
    if None == scale:
        print 'WARNING: if you are not scaling, then you probably want to use an int instead!'
        print 'Beware canadians bearing travel videos'
        scale='1'

    if None != required:
        if verbose: print '  required:',required
        required=int(required)
        assert(0<=required)
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal='+str(int(Decimal(required)*Decimal(scale)))+'),'+str(numbits)+'))\n')
        if verbose: o.write('\n')
        return

    offsetStr=''
    if None != offset:
        offsetStr='-('+offset+')'

    # FIX: can I get rid of the Decimal around params?
    if not unavailable:
        o.write('    bvList.append(binary.setBitVectorSize(BitVector(intVal=int((Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')))),'+str(numbits)+'))\n')
    else: # Have a default value that can be filled in
        o.write("    if '"+name+"' in params:\n")
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal=int((Decimal(params[\''+name+'\']'+offsetStr+')*Decimal(\''+scale+'\')))),'+str(numbits)+'))\n')
        o.write('    else:\n')
        o.write('        bvList.append(binary.setBitVectorSize(BitVector(intVal=int('+str(int(Decimal(unavailable)*Decimal(scale)))+')),'+str(numbits)+'))\n')

    if verbose: o.write('\n')



def encodeBinary(o,name,type,numbits,required=None,arraylen=1,unavailable=None, verbose=False, scale=None):
    '''
    Build the encoder for binary variables.  This is just a pass through.
    This is used for the ais binary message wrapper (e.g. msg 8).  Do
    not use it within binary messages.

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: binary
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum (must be 1..1024 or so)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many decimals will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @return: None
    '''
    if verbose: print '  encode'+name+':',type,numbits,'Req:',required,'alen:',arraylen,unavailable
    assert type=='binary'
    assert (numbits>=1 and numbits<=1024) or numbits==-1
    assert (None == required)  # don't allow this
    assert (None == unavailable) # don't allow this

    if arraylen != 1: assert False # Do not handle arrays.  Arrays of bits is just not necessary.
    if verbose: o.write('    ### FIELD: '+name+' (type='+type+')\n')

    # FIX: can I get rid of the Decimal around params?
    o.write('    bvList.append(params[\''+name+'\'])\n')  # Just pass it through

    if verbose: o.write('\n')


######################################################################
# DECODERS
######################################################################

def decodeBool(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for boolean variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, bool, etc.
    @type startindex: int
    @param startindex: bit that begins the bool(s)
    @type numbits: int = 1
    @param numbits: How many bits per unit datum (must be 1 for bools)
    @type required: bool or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many bools will there be?  FIX: handle variable
    @type unavailable: bool or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert(type=='bool')
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    #int(startindex); int(numbits)  # Make sure it is a number
    assert numbits==1
    assert arraylen == 1 # FIX... handle arrays

    if None != required:
        assert type(required)==bool
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        if required: o.write('True\n')
        else: o.write('False\n')
        if not decodeOnly: o.write('\n')
        return int(startindex)+int(numbits)

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('bool(int('+bv+'['+str(startindex)+':'+str(startindex+int(numbits)*int(arraylen))+']))')
    if not decodeOnly: o.write('\n')

    return int(startindex)+int(numbits)


def decodeUInt(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for unsigned integer variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: uint, etc.
    @type startindex: int
    @param startindex: bit that begins the uint(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: int or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: int or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if arraylen is None:
        arraylen = 1
    assert arraylen == 1 # FIX... handle arrays
    assert numbits >= 1
    if not decodeOnly:
        verbose=False

    if required is not None:
        int(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(required))
        if not decodeOnly: o.write('\n')
        return startindex+numbits

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('int('+bv+'['+str(startindex)+':'+str(startindex+int(numbits)*int(arraylen))+'])')
    if not decodeOnly: o.write('\n')
    if verbose: o.write('\n')

    return startindex+numbits


def decodeInt(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for unsigned integer variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: int
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: int or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: int or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='int'
    if verbose:
        print type, 'decode', name, ': unvail=', unavailable,
        print '  numbits:', numbits, '  startindex=', startindex
    if arraylen is None:
        arraylen=1
    end = startindex+int(numbits) * int(arraylen)
    assert arraylen == 1  # FIX: handle arrays
    assert numbits >= 1

    if None != required:
        int(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(required))
        if not decodeOnly: o.write('\n')
        return end

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('binary.signedIntFromBV('+bv+'['+str(startindex)+':'+str(end)+'])')
    if not decodeOnly: o.write('\n')
    if verbose: o.write('\n')

    return end

def decodeFloat(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for IEEE float variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: int
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: float or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: float or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='float'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert numbits>=1

    if None != required:
        float(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(required))
        if not decodeOnly: o.write('\n')
        if verbose: o.write('\n')
        return end

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('binary.bitvec2float('+bv+'['+str(startindex)+':'+str(end)+'])')
    if not decodeOnly: o.write('\n')

    return end


def decodeAisstr6(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
                  bv='bv',dataDict='r',verbose=False, decodeOnly=False):
    '''
    Build the decoder for aisstr6 variables.  Generally arrays.
    @bug: FIX: validate strings??
    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'aisstr6'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: restricted str or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: restricted str or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='aisstr6'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen >= 1 # FIX... handle arrays
    assert numbits>=1

    if None != required:
        float(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(required)
        if not decodeOnly: o.write('\n')
        return end

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('aisstring.decode('+bv+'['+str(startindex)+':'+str(end)+'])')
    if not decodeOnly: o.write('\n')

    return end


def decodeDecimal(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False,scale=None, decodeOnly=False,offset=None):
    '''
    Build the decoder for signed decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'decimal'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: Decimal or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='decimal'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert numbits>=1 and numbits <= 32

    if None == scale: scale='1' # Warning about this was in the encode section

    if None != required:
        Decimal(required) # Make sure required is a number
        if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(Decimal(required))+'/Decimal(\''+scale+'\')')
        if not decodeOnly: o.write('\n')
        return end

    offsetStr=''
    if offset != None:
        offsetStr += '+Decimal(\''+offset+'\')'

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('Decimal(binary.signedIntFromBV('+bv+'['+str(startindex)+':'+str(end)+']))/Decimal(\''+scale+'\')'+offsetStr+'')
    if not decodeOnly: o.write('\n')

    return end




def decodeUDecimal(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False,scale=None, decodeOnly=False,offset=None):
    '''
    Build the decoder for unsigned decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'udecimal'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum
    @type required: Decimal or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='udecimal'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert numbits>=1 and numbits <= 32

    if None == scale: scale='1' # Warning about this was in the encode section

    if None != required:
        assert (Decimal(required)>=0.) # Make sure required is a number and not negative
        if not decodeOnly:
            o.write('    '+dataDict+'[\''+name+'\']=')
        o.write(str(Decimal(required))+'/Decimal(\''+scale+'\')')
        if not decodeOnly: o.write('\n')
        return end

    offsetStr=''
    if offset != None:
        offsetStr += '+Decimal(\''+offset+'\')'

    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write('Decimal(int('+bv+'['+str(startindex)+':'+str(end)+']))/Decimal(\''+scale+'\')'+offsetStr+'')
    if not decodeOnly: o.write('\n')

    return end


def decodeBinary(o,name,type,startindex,numbits,required=None,arraylen=1,unavailable=None,
               bv='bv',dataDict='r',verbose=False,scale=None, decodeOnly=False):
    '''
    Build the decoder for unsigned decimal variables

    @type o: file like obj
    @param o: where write the code
    @type name: str
    @param name: field name
    @type type: str
    @param type: 'udecimal'
    @type startindex: int
    @param startindex: bit that begins the int(s)
    @type numbits: int >= 1
    @param numbits: How many bits per unit datum.  If -1, then read to the end of the message
    @type required: Decimal or None
    @param required: If not None, then the value must be set to this.
    @type arraylen: int >= 1
    @param arraylen: many ints will there be?  FIX: handle variable
    @type unavailable: Decimal or None
    @param unavailable: the default value to use if none given (if not None)
    @type bv: str
    @param bv: BitVector containing the incoming data
    @type dataDict: str
    @param dataDict: dictionary in which to place the results
    @type decodeOnly: bool
    @param decodeOnly: Set to true to only get the code for decoding
    @rtype: int
    @return: index one past the end of where this read
    '''
    assert type=='binary'
    if verbose: print type,'decode',name,': unvail=',unavailable,'  numbits:',numbits, '  startindex=',startindex
    if None==arraylen: arraylen=1
    end = startindex+int(numbits)*int(arraylen)
    assert arraylen == 1 # FIX... handle arrays
    assert (numbits>=1 and numbits <= 1024) or -1==numbits # What is good max?
    # FIX: assert not required and not an array an not unavailable


    if not decodeOnly: o.write('    '+dataDict+'[\''+name+'\']=')
    o.write(bv+'['+str(startindex)+':')
    if int(numbits) != -1: o.write(str(end))  # -1 means go to the end of the message
    o.write(']')
    if not decodeOnly: o.write('\n')

    return end



######################################################################
# THE REST
######################################################################



def buildTestParamFunc(o,msgET, verbose=False, prefixName=False):
    '''Scrape the testvalues to make a basic param

    @bug: FIX: make this create a dictionary that sits in the overall namespace and spit out deep copies?
    '''
    name = msgET.attrib['name']

    if prefixName: o.write('def '+name+'TestParams():\n')
    else: o.write('def testParams():\n')
    o.write("    '''Return a params file base on the testvalue tags.\n    @rtype: dict\n    @return: params based on testvalue tags\n    '''\n")
    o.write('    params = {}\n')
    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        if verbose: print 'buildTestParamFunc ...',name,type
        val = None
        if hasSubTag(field,'testvalue') and hasSubTag(field,'required'):
            print 'ERROR: can not have both test value and required tags in the same field'
            assert(False)
        if hasSubTag(field,'testvalue'):
            val = field.xpath('testvalue')[0].text
        else:
            if not hasSubTag(field,'required'):
                sys.exit("ERROR: missing required or testvalue for field: "+name)
            val = field.xpath('required')[0].text
        if verbose: print 'buildTestParamFunc for field '+name+' ...',type,val
        o.write('    params[\''+name+'\'] = ')
        if type=='bool':
            if val=='1' or val.lower=='true': val = 'True'
            else: val = 'False'
            o.write(val)
        elif type in ('uint','int','float'):
            o.write(val)
        elif type in ('decimal','udecimal'):
            o.write('Decimal(\''+val+'\')')

        elif type in ('aisstr6'):
            o.write('\''+val+'\'')
        elif type in ('binary'):
            o.write('BitVector(bitstring=\''+val+'\')')
        else:
            print 'ERROR: type not handled ...',type,'  (found in the ',name,' field).  Time to buy more coffee'
            suggestType(name,type)
            assert(False)

        o.write('\n')


    o.write('\n    return params\n\n')

def buildUnitTest(o,msgET, verbose=False, prefixName=False):
    '''
    Write the unittests for a message

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
    '''
    assert(msgET.tag=='message')
    name = msgET.attrib['name']

    buildTestParamFunc(o,msgET, prefixName=prefixName)

    o.write('class Test'+name+'(unittest.TestCase):\n')
    o.write("    '''Use testvalue tag text from each type to build test case the "+name+" message'''\n")
    o.write('    def testEncodeDecode(self):\n\n')
    if prefixName:
        o.write('        params = '+name+'TestParams()\n')
        o.write('        bits   = '+name+'Encode(params)\n')
        o.write('        r      = '+name+'Decode(bits)\n\n')
    else:
        o.write('        params = testParams()\n')
        o.write('        bits   = encode(params)\n')
        o.write('        r      = decode(bits)\n\n')

    o.write('        # Check that each parameter came through ok.\n')
    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        if type in ('bool','uint','int','aisstr6','binary'):
            o.write('        self.failUnlessEqual(r[\''+name+'\'],params[\''+name+'\'])\n')
        else:
            # float, decimal, udecimal
            # FIX: look up the decimal places if decimal
            places = '3'
            if hasSubTag(field,'decimalplaces'): places = field.xpath('decimalplaces')[0].text
            o.write('        self.failUnlessAlmostEqual(r[\''+name+'\'],params[\''+name+'\'],'+places+')\n')



def buildEncode(o,msgET, verbose=False, prefixName=False):
    '''
    Write the encoder/decoder for a message

    http://jaynes.colorado.edu/PythonIdioms.html

    @param o: open file where resulting code will be written
    @param msgET: Element Tree starting at a message node
     TODO(schwehr):handle ais message 20 optional.  very troublesome
    '''
    assert(msgET.tag=='message')
    name = msgET.attrib['name'] # fix rename this variable to avoid hiding later on by field name

    print 'Generating encoder ...',name # FIX: verbose?
    funcName = 'encode'
    if prefixName: funcName = name+'Encode'
    o.write('def '+funcName+'(params, validate=False):\n')

    ########################################
    # doc string
    o.write("    '''Create a "+name+" binary message payload to pack into an AIS Msg "+name+".\n\n")
    o.write('    Fields in params:\n')
    for field in msgET.xpath('field'):
        if verbose:
            print field.tag,field.attrib['name']
            print field.attrib
        desc = field[0].text.replace('\n',' ') # get ride of new lines
        o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description
        if len(field.xpath("required")) == 1:
            o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')

        o.write('\n')
    o.write('    @param params: Dictionary of field names/values.  Throws a ValueError exception if required is missing\n')
    o.write('    @param validate: Set to true to cause checking to occur.  Runs slower.  FIX: not implemented.\n')

    o.write("    @rtype: BitVector\n")
    o.write("    @return: encoded binary message (for binary messages, this needs to be wrapped in a msg 8\n")
    o.write("    @note: The returned bits may not be 6 bit aligned.  It is up to you to pad out the bits.\n")
    o.write("    '''\n\n")

    ########################################
    # Actually build the code

    o.write('    bvList = []\n')

    if verbose: print 'number of fields = ', len(msgET.xpath('field'))

    dynamicArrays = False # Set to true when positioning must be calculated

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',name,'('+type+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'
        else:
            if verbose: print 'Processing field ...',name,'(',type+' ',numbits,')'

        if   type=='bool'   : encodeBool   (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='uint'   : encodeUInt   (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='int'    : encodeInt    (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='float'  : encodeFloat  (o,name,type,numbits,required,arraylen,unavailable)
        elif type=='aisstr6': encodeAisstr6(o,name,type,numbits,required,arraylen,unavailable)
        elif type=='decimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            encodeDecimal(o,name,type,numbits,required,arraylen,unavailable,scale=scale, offset=offset)
        elif type=='udecimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'):
                offset = field.xpath('offset')[0].text
            encodeUDecimal(o,name,type,numbits,required,arraylen,unavailable,scale=scale,offset=offset)
        elif type=='binary': encodeBinary(o,name,type,numbits,required,arraylen,unavailable)
        else:
            print 'WARNING: In buildEncode - Unhandled field type for',name,'...',type
            suggestType (name,type)
            assert False

#    o.write('\n    bv=binary.joinBV(bvList)\n')
#    o.write('\n    bvLen=len(bv)\n')
#    o.write('\n    if bvLen%6!=0:\n')
#    o.write('\n        bv = bv + BitVector(size=bvLen%6)\n')
#    o.write('\n    return bv\n\n')
    o.write('\n    return binary.joinBV(bvList)\n\n')


######################################################################
# DECODER ONE PART AT A TIME - if only using a small part, save some cycles!


#, msgDict=None):
def buildDecodeParts(o,msgET, verbose=False, prefixName=False):

    '''
    Write the decoder for a message

    @param o: open file where resulting code will be written
    @type msgET: elementtree
    @param prefixName: if True, put the name of the message on the functions.
    @param msgET: Element Tree starting at a message node
    @return: None

     TODO(schwehr):FIX: doc strings for each decode!
     TODO(schwehr):FIX: check for a dac,fid, or efid.  If exists, then this is an AIS Msg 8 payload
     TODO(schwehr):May want to take a dictionary of already decoded fields to speed things that need prior info
    for things like variable length arrays
    '''

    assert(msgET.tag=='message')
    name = msgET.attrib['name']

    print 'Generating partial decode functions ...',name

    baseName = name+'Decode'
    if not prefixName: baseName = 'decode'

    startindex = 0 # Where we are in the bitvector... FIX: what about variable length jobs?

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']

        o.write('def '+baseName+name+'(bv, validate=False):\n')
        # Follow the same convention of decoding into a dict so that code is the same
        #o.write('    r={};')
        o.write('    return ')

        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',name,'('+type+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'

        assert None!=startindex
        if verbose: print 'startindex',startindex
        if   type=='bool'   : startindex = decodeBool   (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='uint'   : startindex = decodeUInt   (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='int'    : startindex = decodeInt    (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='float'  : startindex = decodeFloat  (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='aisstr6': startindex = decodeAisstr6(o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)
        elif type=='binary' : startindex = decodeBinary (o,name,type,startindex,numbits,required,arraylen,unavailable,decodeOnly=True)

        elif type=='decimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,decodeOnly=True,offset=offset)
        elif type=='udecimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeUDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,decodeOnly=True,offset=offset)

        else:
            print 'WARNING: In buildDecode - Unhandled field type for',name,'...',type
            suggestType (name,type)
            assert False

        #o.write('    return(r[\''+name+'\'])\n\n')
        o.write('\n\n')


######################################################################
# DECODER RING

def buildDecode(o,msgET, verbose=False, prefixName=False):
    '''
    Write the decoder for a message

    @param o: open file where resulting code will be written
    @type msgET: elementtree
    @param msgET: Element Tree starting at a message node
    @return: None

     TODO(schwehr):FIX: check for a dac,fid, or efid.  If exists, then this is an AIS Msg 8 payload
    '''

    assert(msgET.tag=='message')
    name = msgET.attrib['name']

    print 'Generating decoder ...',name
    funcName = 'decode'
    if prefixName: funcName = name+'Decode'
    o.write('def '+funcName+'(bv, validate=False):\n')

    ########################################
    # doc string
    o.write("    '''Unpack a "+name+" message.\n\n")
    o.write('    Fields in params:\n')
    for field in msgET.xpath('field'):
        desc = field[0].text.replace('\n',' ') # get ride of new lines
        o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description
        if len(field.xpath("required")) == 1:
            o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')

        o.write('\n')
    o.write('    @type bv: BitVector\n')
    o.write('    @param bv: Bits defining a message\n')
    o.write('    @param validate: Set to true to cause checking to occur.  Runs slower.  FIX: not implemented.\n')

    o.write("    @rtype: dict\n")
    o.write("    @return: params\n")
    o.write("    '''\n\n")


    o.write('    #Would be nice to check the bit count here..\n')
    o.write('    #if validate:\n')
    o.write('    #    assert (len(bv)==FIX: SOME NUMBER)\n')


    ########################################
    # Actually build the code

    o.write('    r = {}\n')


    if verbose: print 'number of fields = ', len(msgET.xpath('field'))

    dynamicArrays = False # Set to true when positioning must be calculated

    startindex = 0 # Where we are in the bitvector... FIX: what about variable length jobs?

    for field in msgET.xpath('field'):
        name = field.attrib['name']
        type = field.attrib['type']
        numbits = int(field.attrib['numberofbits'])
        required = None;
        if hasSubTag(field,'required'):
            required = field.xpath('required')[0].text
        unavailable=None;
        if hasSubTag(field,'unavailable'): unavailable = field.xpath('unavailable')[0].text
        arraylen=1
        if 'arraylength' in field.attrib:
            arraylen=int(field.attrib['arraylength'])
            if verbose: print 'Processing field ...',name,'('+type+' ',numbits,'*',arraylen,'=',numbits*arraylen,'bits )'
        else:
            if verbose: print 'Processing field ...',name,'(',type+' ',numbits,')'

        assert None!=startindex
        if verbose: print 'startindex',startindex

        if hasSubTag(field,'optional'):
            # NoReturn means that this field is not a point that needs a check to see if we are done
            # FIX: need to do something more efficient
            if msgET.attrib['aismsgnum']!='20':
                sys.exit('optional is only allow on AIS msg 20.  Do NOT use it on new messages!!!')
            text = field.xpath('optional')[0].text
            if None==field.xpath('optional')[0].text: # Found a NoReturn
                # Message 20 is a pain!
                pad = 8-(startindex%8)
                print 'not found: noreturn.  pad=',pad
                assert(pad<=6)
                o.write('    if len(bv)<='+str(startindex+pad)+': return r; # All fields below are optional\n')
            elif text != 'NoReturn':
                sys.exit ('ERROR: optional text must be NoReturn or empty')

        if   type=='bool'   : startindex = decodeBool   (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='uint'   : startindex = decodeUInt   (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='int'    : startindex = decodeInt    (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='float'  : startindex = decodeFloat  (o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='aisstr6': startindex = decodeAisstr6(o,name,type,startindex,numbits,required,arraylen,unavailable)
        elif type=='binary' : startindex = decodeBinary (o,name,type,startindex,numbits,required,arraylen,unavailable)

        elif type=='decimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,offset=offset)

        elif type=='udecimal':
            scale = None
            if hasSubTag(field,'scale'): scale = field.xpath('scale')[0].text
            offset = None
            if hasSubTag(field,'offset'): offset = field.xpath('offset')[0].text
            startindex = decodeUDecimal(o,name,type,startindex, numbits,required,arraylen,unavailable,scale=scale,offset=offset)

        else:
            print 'WARNING: In buildDecode - Unhandled field type for',name,'...',type
            suggestType (name,type)
            assert False


        if None==startindex: print 'FIX: here. drat.  treat me right'
        assert None!=startindex


    o.write('    return r\n\n')



#def getPythonType(aType):
#    '''
#    Translate a
#    @rtype: str
#    @return: name of the python type
#    '''

aisType2pythonType={
'bool':'Bool',
'uint':'int',
'int':'int',
'udecimal':'Decimal',
'decimal':'Decimal',
'aisstr6':'str',
'binary':'str',
'float':'Float',
}

import copy
aisType2optParseType=copy.deepcopy(aisType2pythonType)
aisType2optParseType['bool']='int' # FIX: make these appear as a flag variable
aisType2optParseType['aisstr6']='string'
aisType2optParseType['aisstr6']='string'
aisType2optParseType['binary']='string'
aisType2optParseType['udecimal']='string'  # FIX: Is this a good choice?
aisType2optParseType['decimal']='string'
aisType2optParseType['float']='float'


def buildOptParse(o,msgET, prefixName=False):
    '''Create a function that adds the options to a parse object'''

    assert None != msgET
    assert msgET.tag=='message'
    msgName = msgET.attrib['name']

    prefix=''
    if prefixName: prefix=msgName

    funcName = 'addMsgOptions'
    if prefixName: funcName = msgName + 'AddMsgOptions'
    o.write('\ndef '+funcName+'(parser):')

    # FIX: write out a doc string

    o.write('''
    parser.add_option('-d','--decode',dest='doDecode',default=False,action='store_true',
                help='decode a "'''+msgName+'''" AIS message')
    parser.add_option('-e','--encode',dest='doEncode',default=False,action='store_true',
                help='encode a "'''+msgName+'''" AIS message')
''')

    #print 'here...', msgName, prefixName
    for field in msgET.xpath('field'):
        name      = field.attrib['name']
        fieldType = field.attrib['type']
        if hasSubTag(field,'required'):
            print 'skipping required field ...',name,fieldType
            continue
        #print 'there',name,fieldType
        o.write('    parser.add_option(\'--')
        if prefixName: o.write(msgName+'-')
        o.write(name+'-field\', dest=\''+name+'Field\'')
        if hasSubTag(field,'unavailable'):
            val = field.xpath('unavailable')[0].text
            o.write(',default=')
            if fieldType in ('uint','int','float'):
                o.write(val)
            elif fieldType in ('decimal','udecimal'):
                o.write('Decimal(\''+val+'\')')
            elif fieldType in ('aisstr6','bitvector'):
                o.write('\''+val+'\'')
        o.write(',metavar=\''+fieldType+'\',type=\''+aisType2optParseType[fieldType]+'\'')
        o.write('\n        ,help=\'Field parameter value [default: %default]\')\n')



def buildMain(o, msgET, prefixName=False):
    assert None != msgET
    assert msgET.tag=='message'
    msgName = msgET.attrib['name']

    prefix=''
    if prefixName: prefix=msgName

    buildOptParse(o, msgET, prefixName)


    o.write('''
def main():
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options]")

    parser.add_option('--doc-test',dest='doctest',default=False,action='store_true',
        help='run the documentation tests')
    parser.add_option('--unit-test',dest='unittest',default=False,action='store_true',
        help='run the unit tests')
    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',
        help='Make the test output verbose')

    # FIX: remove nmea from binary messages.  No way to build the whole packet?
    # FIX: or build the surrounding msg 8 for a broadcast?
    typeChoices = ('binary','nmeapayload','nmea') # FIX: what about a USCG type message?
    parser.add_option('-t', '--type', choices=typeChoices, type='choice',
        dest='ioType', default='nmeapayload',
        help='What kind of string to write for encoding ('+', '.join(typeChoices)+') [default: %default]')


    outputChoices = ('std','html','csv','sql' ''')
#    'xml' -  FIX: need to add xml output
    if haveLocatableMessage(msgET): o.write(', \'kml\',\'kml-full\'')
    o.write(''')
    parser.add_option('-T', '--output-type', choices=outputChoices,
        type='choice', dest='outputType', default='std',
        help='What kind of string to output ('+', '.join(outputChoices)+') '
        '[default: %default]')

    parser.add_option('-o','--output',dest='outputFileName',default=None,
        help='Name of the python file to write [default: stdout]')

    parser.add_option('-f', '--fields', dest='fieldList', default=None,
        action='append', choices=fieldList,
        help='Which fields to include in the output.  Currently only for csv '
        'output [default: all]')

    parser.add_option('-p', '--print-csv-field-list', dest='printCsvfieldList',
        default=False,action='store_true',
        help='Print the field name for csv')

    parser.add_option('-c', '--sql-create', dest='sqlCreate', default=False,
        action='store_true',
        help='Print out an sql create command for the table.')

    parser.add_option('--latex-table', dest='latexDefinitionTable',
        default=False,action='store_true',
        help='Print a LaTeX table of the type')

    parser.add_option('--text-table', dest='textDefinitionTable', default=False,
        action='store_true',
        help='Print delimited table of the type (for Word table importing)')

    parser.add_option('--delimt-text-table', dest='delimTextDefinitionTable',
        default='    ',
        help='Delimiter for text table [default: \\\'%default\\\'] '
        '(for Word table importing)')

    dbChoices = ('sqlite','postgres')
    parser.add_option('-D', '--db-type', dest='dbType', default='postgres',
        choices=dbChoices,type='choice',
        help='What kind of database ('+', '.join(dbChoices)+') '
        '[default: %default]')
''')

    o.write('''
    addMsgOptions(parser)

    (options,args) = parser.parse_args()
    success = True

    if options.doctest:
            import os; print os.path.basename(sys.argv[0]), 'doctests ...',
            sys.argv = [sys.argv[0]]
            if options.verbose:
              sys.argv.append('-v')

            numfail, numtests = doctest.testmod()
            if not numfail:
                print 'ok'
            else:
                print 'FAILED'
                success = False

    if not success: sys.exit('Something Failed')
    del success # Hide success from epydoc

    if options.unittest:
            sys.argv = [sys.argv[0]]
            if options.verbose: sys.argv.append('-v')
            unittest.main()

    outfile = sys.stdout
    if None!=options.outputFileName:
            outfile = file(options.outputFileName,'w')

''')



    ##############################
    # encode
    ##############################
    # FIX: make everything tabs.  Grr.
    o.write('\n    if options.doEncode:\n')
    o.write('        # Make sure all non required options are specified.\n')
    for field in msgET.xpath('field'):
        name      = field.attrib['name']
        fieldType = field.attrib['type']
        varName = prefix + name + 'Field'
        if not hasSubTag(field,'required'):
            o.write('        if None==options.' + varName +
                    ': parser.error("missing value for ' + varName + '")\n')

    # Build dict
    o.write('    msgDict = {\n')
    for field in msgET.xpath('field'):
        name    = field.attrib['name']
        varName = prefix+name+'Field'
        if hasSubTag(field,'required'):
            o.write('        \''+name+'\': \''+field.xpath('required')[0].text+'\',\n')
        else:
            o.write('        \''+name+'\': options.'+varName+',\n')
    o.write('    }\n')

    encodeFunction = 'encode'
    if prefixName:
        encodeFunction = msgName + 'Encode'
    o.write('''
    bits = '''+encodeFunction+'''(msgDict)
    if 'binary' == options.ioType:
        print str(bits)
    elif 'nmeapayload'==options.ioType:
        # FIX: figure out if this might be necessary at compile time
        bitLen=len(bits)
        if bitLen % 6 != 0:
            bits = bits + BitVector(size=(6 - (bitLen%6)))  # Pad out to multiple of 6
        print binary.bitvectoais6(bits)[0]

    # FIX: Do not emit this option for the binary message payloads.  Does not make sense.
    elif 'nmea' == options.ioType:
        nmea = uscg.create_nmea(bits)
        print nmea
    else:
        sys.exit('ERROR: unknown ioType.  Help!')
''')


    ##############################
    # decode all
    ##############################
    decodeFunction = 'decode'
    printFields='printFields'
    if prefixName:
        decodeFunction = msgName+'Decode'
        printFields    = msgName+'PrintFields'
    o.write('''

        if options.sqlCreate:
                sqlCreateStr(outfile,options.fieldList,dbType=options.dbType)

        if options.latexDefinitionTable:
                latexDefinitionTable(outfile)

        # For conversion to word tables
        if options.textDefinitionTable:
                textDefinitionTable(outfile,options.delimTextDefinitionTable)

        if options.printCsvfieldList:
                # Make a csv separated list of fields that will be displayed for csv
                if None == options.fieldList: options.fieldList = fieldList
                import StringIO
                buf = StringIO.StringIO()
                for field in options.fieldList:
                        buf.write(field+',')
                result = buf.getvalue()
                if result[-1] == ',': print result[:-1]
                else: print result

        if options.doDecode:
                if len(args)==0: args = sys.stdin
                for msg in args:
                        bv = None

                        if msg[0] in ('$','!') and msg[3:6] in ('VDM','VDO'):
                                # Found nmea
                                # FIX: do checksum
                                bv = binary.ais6tobitvec(msg.split(',')[5])
                        else: # either binary or nmeapayload... expect mostly nmeapayloads
                                # assumes that an all 0 and 1 string can not be a nmeapayload
                                binaryMsg=True
                                for c in msg:
                                        if c not in ('0','1'):
                                                binaryMsg=False
                                                break
                                if binaryMsg:
                                        bv = BitVector(bitstring=msg)
                                else: # nmeapayload
                                        bv = binary.ais6tobitvec(msg)

                        '''+printFields+'''('''+decodeFunction+'''(bv)
                                    ,out=outfile
                                    ,format=options.outputType
                                    ,fieldList=options.fieldList
                                    ,dbType=options.dbType
                                    )

############################################################
if __name__=='__main__':
    main()
''')


######################################################################
if __name__=='__main__':
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options]")

    parser.add_option('-o','--output',dest='outputFileName',default=None,
                        help='Name of the python file to write')

    parser.add_option('-i','-x','--xml-definition',dest='xmlFileName',default=None,
                        help='XML definition file for the msg to use')

    parser.add_option('--doc-test',dest='doctest',default=False,action='store_true',
                        help='run the documentation tests')

    parser.add_option('-p','--prefix',dest='prefix',default=False,action='store_true',
                        help='put the field name in front of all function names.'
                        +'  Allows multiple messages in one file')

    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',
                        help='run the tests run in verbose mode')

    (options,args) = parser.parse_args()

    success=True

    if options.doctest:
        import os; print os.path.basename(sys.argv[0]), 'doctests ...',
        argvOrig = sys.argv
        sys.argv= [sys.argv[0]]
        if options.verbose: sys.argv.append('-v')
        import doctest
        numfail,numtests=doctest.testmod()
        if numfail==0: print 'ok'
        else:
            print 'FAILED'
            success=False
        sys.argv = argvOrig # Restore the original args
        del argvOrig # hide from epydoc
        sys.exit() # FIX: Will this exit success?

    #infile=sys.stdin
    #if options.xmlFileName: infile = file(options.xmlFileName,'r')

    #outfile=sys.stdout
    #if options.outputFile: outfile = file(options.xmlDefinition,'w')

    # FIX: down the road, it would be good to allow either filenames of std{in/out}

    if None==options.xmlFileName:
        sys.exit('ERROR: must specify an xml definition file.')
    if None==options.outputFileName:
        sys.exit('ERROR: must specify an python file to write to.')
    generatePython(options.xmlFileName,options.outputFileName,prefixName=options.prefix, verbose=options.verbose)

    print '\nrecommend running pychecker like this:'
    print '  pychecker -q',options.outputFileName
#!/usr/bin/env python Tools to generate python code to serialize/deserialize messages between python and ais binary.  Trying to be as inline as possible, so no XML on the fly like in ais-py.  serialize: python to ais binary deserialize: ais binary to python  The generated code uses translators.py, binary.py, and aisstring.py which should be packaged with the resulting files.  @license: Apache 2.0   TODO(schwehr):add a link to generated doc string to bring up the html for the pretty version  TODO(schwehr):write a separate validation script that distinguishes standard messages and bin messages  TODO(schwehr):make sure binary is only used in AIS ITU messages and not within the binary messages!  TODO(schwehr):arrays  TODO(schwehr):handle entry ranges in lookup tables  TODO(schwehr):arraylength of -1 for messages 12 and 14  TODO(schwehr):in the table html/kml form output, if no LUT and value is the unknown case, say so in the converted column!  TODO(schwehr):stop using sqlhelp.  Can have trouble with corrupted data. @bug: NOT complete @note: This compiler assumes that all locations are in WGS 84 (EPSG 4326)  TODO(schwehr):add type checking of command line parameters for crafting AIS messages Try to suggest a type name if one did not work.      @param printout: if true, write a suggestion to stdout.      >>> suggestType('myFieldName', 'unsigned int')     Recommend switching "unsigned int" to "uint" for field "myFieldName"     'uint'      >>> suggestType('JohnWarfon', 'yoyodyne')     Sorry!  No recommendation available for bad type "yoyodyne" for field "JohnWarfon" @return: true if the tag a sub tag with name subtag Write the doc string header for the message file.      @param o: Open output file to write code to.     Must be pre-expanded with the expandais.py command. # Need to pass in info about the source file, etc. # Minor trickery to get svn to ignore the keywords in the next few lines #!/usr/bin/env python  \"\"\"Autogenerated python functions to serialize/deserialize binary messages.  Generated by: Need to then wrap these functions with the outer AIS packet and then convert the whole binary blob to a NMEA string.  Those functions are not currently provided in this file.  serialize: python to ais binary deserialize: ais binary to python  The generated code uses translators.py, binary.py, and aisstring.py which should be packaged with the resulting files. TODO(schwehr):FIX: put in a description of the message here with fields and types. \"\"\" import doctest import sys from decimal import Decimal import unittest  from aisutils.BitVector import BitVector  from aisutils import aisstring from aisutils import binary from aisutils import sqlhelp from aisutils import uscg  # FIX: check to see if these will be needed TrueBV  = BitVector(bitstring="1") "Why always rebuild the True bit?  This should speed things up a bunch" FalseBV = BitVector(bitstring="0") "Why always rebuild the False bit?  This should speed things up a bunch" @param infile: xml ais binary message definition file     @param outfile: where to dump the python code # functions that only decode one field ######################################################################\n') #####################################################################\n') ###################################################################### # Build Helpers ###################################################################### emit the fieldList and other things???      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node       TODO(schwehr):for lookuptable/entry values, make it also print the decoded value.      TODO(schwehr):use a different name for message and field      TODO(schwehr):put in comments with the python and sql data type for each field # FIX: add some documentation to the fieldList # FIX: like that it is used as the default csv list # postgis names # already done, so skip # PostGIS data type\n') \"\"\" Go to the Postgis field names from the straight field name \"\"\" # postgis names # already done, so skip \"\"\" Go from the Postgis field names to the straight field name \"\"\" # postgis names # already done, so skip \"\"\" Lookup table for each postgis field name to get its type. \"\"\" ###################################################################### # SIMPLE PRINT ###################################################################### Get the maximum string length of any field name Pad a string out to the length requested with spaces out to the right Make sure this message has both long/x and lat/y fields.     @rtype: bool #if getLongitudeFieldName(msgET) and getLatitudeFieldName(msgET): return True Dig up the first field name that include longitude and return it      TODO(schwehr):might want to allow a search for a special tag to mark this Dig up the first field name that include longitude and return it      TODO(schwehr):might want to allow a search for a special tag to mark this Write a simple in order print for the resulting dictionary.      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node       TODO(schwehr):for lookuptable/entry values, make it also print the decoded value.      TODO(schwehr):use a different name for message and field #msgName = msgname = name  # FIX: make these all msgName # FIX: verbose? # +1 for the ':' ############################## # Html builder ############################## #################### ####### HTML format #################### #o.write('    elif \'html\'==format:\n') #o.write('        out.write("<tr bgcolor=\\"#9acd32\\">\\n")\n') # Just duplicate the value through ############################## ####### KML GoogleEarth ############################## # FIX: use the first postgis print tagged lat and lon field #m_ylw-pushpin_copy0</styleUrl>\\n")\n') ############################## ##### Main print dispatch ############################## ######################################## # doc string Print a "+msgName+" message to stdout.\n\n")     o.write('    Fields in params:\n')     for field in msgET.xpath('field'):         desc = field[0].text.replace('\n',' ') # get ride of new lines         o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description         if len(field.xpath("required")) == 1:             o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')          o.write('\n')     o.write('    @param params: Dictionary of field names/values.\n')     o.write('    @param out: File like object to write to.\n')      o.write("    @rtype: stdout\n")     o.write("    @return: text to out\n")     o.write(" ######################################## # Actually build the code # FIX: elif aisstr6 strip @@@ and then print #################### ####### Comma separated values (csv) #################### elif 'csv'==format:                 if None == options.fieldList:                         options.fieldList = fieldList                 needComma = False;                 for field in fieldList:                         if needComma: out.write(',')                         needComma = True                         if field in params:                                 out.write(str(params[field]))                         # else: leave it empty                 out.write("\\n") #################### ####### Other dispatchers #################### elif 'sql'==format:                 sqlInsertStr(params,out,dbType=dbType) # Has a lon/lat pair somewhere in there. #        o.write('        out.write("    <name>Position</name>\\n")\n') #################### ####### Make safety check #################### # Nothing to return\n\n') ###################################################################### # Build Lookup tables for enumerated int or uint fields ###################################################################### ###################################################################### # DELIMITED TEXT FILE ###################################################################### Write functions for text definition output      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node     @param verbose: talk lots in the process     @param prefixName: set to a string to have the commands prefixed by that character.       TODO(schwehr):should this be a style sheet thing instead? \n###################################################################### # Text Definition ###################################################################### # Should this really default to true for the uscg fields? (outfile=sys.stdout ,delim='    '):     \"\"\"Return the text definition table for this message type      @param outfile: file like object to print to.     @type outfile: file obj     @return: text table string via the outfile     @rtype: str      \"\"\"     o = outfile # NOTE: using / to mean one  in the output compiled code # FIX: maybe cut down to the first sentence? # 168 in the first slot and following slots get to use the whole 256?? # FIX: is this right? \"\"\") # FIX: should it return a copy of outfile? ###################################################################### Write functions for LaTeX output      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node     @param verbose: talk lots in the process     @param prefixName: set to a string to have the commands prefixed by that character.       TODO(schwehr):should this be a style sheet thing instead? \n###################################################################### # LATEX SUPPORT ###################################################################### # Should this really default to true for the uscg fields? (outfile=sys.stdout                 ):         \"\"\"         Return the LaTeX definition table for this message type         @param outfile: file like object to print to.         @type outfile: file obj         @return: LaTeX table string via the outfile         @rtype: str          \"\"\"         o = outfile # NOTE: using / to mean one  in the output compiled code o.write(\"\"\" /begin{table}%[htb] /centering /begin{tabular}{|l|c|l|} /hline Parameter & Number of bits & Description //  /hline/hline # _ means subscript to latex # FIX: maybe cut down to the first sentence? # 168 in the first slot and following slots get to use the whole 256??  FIX: is this right? #    if verbose: #        sys.stderr.write (' msgnum:'+msgET.attrib['aismsgnum'] +' - ') #        sys.stderr.write (' desc: '+ str(msgET.xpath('description')[0])+' - ') #        sys.stderr.write (' name: '+ str(msgET.attrib['name'])+'\n') #sys.stderr.write (' :'+ str()+'\n') // /hline /end{tabular} /caption{AIS message number } /label{tab: } /end{table} \"\"\") # FIX: should it return a copy of outfile? ###################################################################### Write SQL code      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node     @param verbose: talk lots in the process     @param prefixName: set to a string to have the commands prefixed by that character. ###################################################################### # SQL SUPPORT ###################################################################### # Should this really default to true for the uscg fields? Str(outfile=sys.stdout, fields=None, extraFields=None                 ,addCoastGuardFields=True                 ,dbType='postgres'                 ):         \"\"\"         Return the SQL CREATE command for this message type         @param outfile: file like object to print to.         @param fields: which fields to put in the create.  Defaults to all.         @param extraFields: A sequence of tuples containing (name,sql type) for additional fields         @param addCoastGuardFields: Add the extra fields that come after the NMEA check some from the USCG N-AIS format         @param dbType: Which flavor of database we are using so that the create is tailored ('sqlite' or 'postgres')         @type addCoastGuardFields: bool         @return: sql create string         @rtype: str          @see: sqlCreate         \"\"\"         # FIX: should this sqlCreate be the same as in LaTeX (createFuncName) rather than hard coded?         outfile.write(str(sqlCreate(fields,extraFields,addCoastGuardFields,dbType=dbType))) (fields=None, extraFields=None, addCoastGuardFields=True, dbType='postgres'):     \"\"\"Return the sqlhelp object to create the table.      @param fields: which fields to put in the create.  Defaults to all.     @param extraFields: A sequence of tuples containing (name,sql type) for additional fields     @param addCoastGuardFields: Add the extra fields that come after the NMEA check some from the USCG N-AIS format     @type addCoastGuardFields: bool     @param dbType: Which flavor of database we are using so that the create is tailored ('sqlite' or 'postgres')     @return: An object that can be used to generate a return     @rtype: sqlhelp.create     \"\"\"     if fields is None:         fields = fieldList # FIX: add the ADD command here. # number of decimal places #precision += 1 # FIX: do I really need this safety factor # FIX: what is a good maximum for AIS? if addCoastGuardFields:         # c.addInt('cg_s_rssi')  # Relative signal strength indicator         # c.addInt('cg_d_strength')  # dBm receive strength         # c.addVarChar('cg_x',10)  # Idonno         c.addInt('cg_t_arrival')  # Receive timestamp from the AIS equipment 'T'         c.addInt('cg_s_slotnum')  # Slot received in         c.addVarChar('cg_r',15)  # Receiver station ID  -  should usually be an MMSI, but sometimes is a string         c.addInt('cg_sec')  # UTC seconds since the epoch          c.addTimestamp('cg_timestamp') # UTC decoded cg_sec - not actually in the data stream # # Postgres with PostGIS added # # Should be a set? #print '  ',field.attrib['name'],field.attrib['postgisName'] #print '   doing',pgName,pgType #print '    all components' #for c in components: #    print '     ',c.attrib['postgisName'] #--- EPSG 4326 : WGS 84\n') #INSERT INTO "spatial_ref_sys" ("srid","auth_name","auth_srid","srtext","proj4text") VALUES (4326,'EPSG',4326,'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]','+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs ');\n # FIX: function name def Str(params, outfile=sys.stdout, extraParams=None, dbType='postgres'):         \"\"\"         Return the SQL INSERT command for this message type         @param params: dictionary of values keyed by field name         @param outfile: file like object to print to.         @param extraParams: A sequence of tuples containing (name,sql type) for additional fields         @return: sql create string         @rtype: str          @see: sqlCreate         \"\"\"         outfile.write(str(sqlInsert(params,extraParams,dbType=dbType))) # FIX: function name (params,extraParams=None,dbType='postgres'):         \"\"\"         Give the SQL INSERT statement         @param params: dict keyed by field name of values         @param extraParams: any extra fields that you have created beyond the normal ais message fields         @rtype: sqlhelp.insert         @return: insert class instance          TODO(schwehr):allow optional type checking of params?         @warning: this will take invalid keys happily and do what???         \"\"\"          i = sqlhelp.insert(' ',dbType=dbType)          if dbType=='postgres':                 finished = []                 for key in params:                         if key in finished:                                 continue                          if key not in toPgFields and key not in fromPgFields:                                 if type(params[key])==Decimal: i.add(key,float(params[key]))                                 else: i.add(key,params[key])                         else:                                 if key in fromPgFields:                                         val = params[key]                                         # Had better be a WKT type like POINT(-88.1 30.321)                                         i.addPostGIS(key,val)                                         finished.append(key)                                 else:                                         # Need to construct the type.                                         pgName = toPgFields[key]                                         #valStr='GeomFromText(\\''+pgTypes[pgName]+'('                                         valStr=pgTypes[pgName]+'('                                         vals = []                                         for nonPgKey in fromPgFields[pgName]:                                                 vals.append(str(params[nonPgKey]))                                                 finished.append(nonPgKey)                                         valStr+=' '.join(vals)+')'                                         i.addPostGIS(pgName,valStr)         else:                 for key in params:                         if type(params[key])==Decimal: i.add(key,float(params[key]))                         else: i.add(key,params[key])          if None != extraParams:                 for key in extraParams:                         i.add(key,extraParams[key])          return i ###################################################################### # Build Lookup tables for enumerated int or uint fields ###################################################################### Write lookup tables for enumerated types (uint or int, maybe bool too).       TODO(schwehr):FIX: what to do about multiple entries with the same text?  Need to ban that kind of thing      TODO(schwehr):Make doc strings for each LUT.      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node # FIX: verbose? #'+lutName+'EncodeLut\n') # FIX: make doc string for LUT here # '+lutName+'EncodeLut\n') # FIX: make doc string for LUT here ###################################################################### # ENCODERS ###################################################################### Build the encoder for boolean variables     @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: bool, etc.     @type numbits: int = 1     @param numbits: How many bits per unit datum (must be 1 for bools)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many bools will there be?  FIX: handle variable     @type unavailable: bool or None     @param unavailable: the default value to use if none given (if not None)     @return: None # FIX... handle arrays ### FIELD: '+name+' (type=bool)\n') # Have a default value that can be filled in Build the encoder for unsigned integer variables.      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: uint, bool, etc.     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..32)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many unsigned ints will there be?  FIX: handle variable     @type unavailable: bool or None     @param unavailable: the default value to use if none given (if not None)     @return: None # FIX... handle arrays ### FIELD: '+name+' (type='+type+')\n') # Have a default value that can be filled in # Make sure unavailable is a number object Build the encoder for IEEE float variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type fieldType: str     @param fieldType: uint, bool, etc.     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..32)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many unsigned ints will there be?  FIX: handle variable     @type unavailable: bool or None     @param unavailable: the default value to use if none given (if not None)     @return: None # Force by the IEEE spec # FIX... handle arrays ### FIELD: '+name+' (type='+fieldType+')\n') # Have a default value that can be filled in # Make sure unavailable is a number object Build the encoder for aisstr6 variables.  Generally are arrays.     @bug: do we need to optionally check for a valid string?      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type fieldType: str     @param fieldType: uint, bool, etc.     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..32)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many unsigned ints will there be?  FIX: handle variable     @type unavailable: bool or None     @param unavailable: the default value to use if none given (if not None)     @return: None # Each character must be 6 bits ### FIELD: '+name+' (type='+fieldType+')\n') # Have a default value that can be filled in Build the encoder for signed integer variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: uint, bool, etc.     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..32)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many signed ints will there be?  FIX: handle variable     @type unavailable: number or None     @param unavailable: the default value to use if none given (if not None)     @return: None # FIX... handle arrays ### FIELD: '+name+' (type='+type+')\n') # Have a default value that can be filled in #assert type(unavailable)== # Make sure unavailable is a number object # FIX: Ummm... why am I passing the type?  I guess it makes the one print statement easier Build the encoder for signed decimal variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: decimal     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..32)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many decimals will there be?  FIX: handle variable     @type unavailable: Decimal or None     @param unavailable: the default value to use if none given (if not None)     @return: None # FIX... handle arrays ### FIELD: '+name+' (type='+type+')\n') # FIX: optimize to not emit the scaling when it is not needed... or tell the user to switch to an int! # FIX: can I get rid of the Decimal around params? # Have a default value that can be filled in Build the encoder for signed decimal variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: decimal     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..32)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many decimals will there be?  FIX: handle variable     @type unavailable: Decimal or None     @param unavailable: the default value to use if none given (if not None)     @return: None # FIX... handle arrays ### FIELD: '+name+' (type='+type+')\n') # FIX: optimize to not emit the scaling when it is not needed... or tell the user to switch to an int! # FIX: can I get rid of the Decimal around params? # Have a default value that can be filled in Build the encoder for binary variables.  This is just a pass through.     This is used for the ais binary message wrapper (e.g. msg 8).  Do     not use it within binary messages.      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: binary     @type numbits: int >= 1     @param numbits: How many bits per unit datum (must be 1..1024 or so)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many decimals will there be?  FIX: handle variable     @type unavailable: Decimal or None     @param unavailable: the default value to use if none given (if not None)     @return: None # don't allow this # don't allow this # Do not handle arrays.  Arrays of bits is just not necessary. ### FIELD: '+name+' (type='+type+')\n') # FIX: can I get rid of the Decimal around params? # Just pass it through ###################################################################### # DECODERS ###################################################################### Build the decoder for boolean variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: uint, bool, etc.     @type startindex: int     @param startindex: bit that begins the bool(s)     @type numbits: int = 1     @param numbits: How many bits per unit datum (must be 1 for bools)     @type required: bool or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many bools will there be?  FIX: handle variable     @type unavailable: bool or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read #int(startindex); int(numbits)  # Make sure it is a number # FIX... handle arrays Build the decoder for unsigned integer variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: uint, etc.     @type startindex: int     @param startindex: bit that begins the uint(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum     @type required: int or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: int or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX... handle arrays # Make sure required is a number Build the decoder for unsigned integer variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: int     @type startindex: int     @param startindex: bit that begins the int(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum     @type required: int or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: int or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX: handle arrays # Make sure required is a number Build the decoder for IEEE float variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: int     @type startindex: int     @param startindex: bit that begins the int(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum     @type required: float or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: float or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX... handle arrays # Make sure required is a number Build the decoder for aisstr6 variables.  Generally arrays.     @bug: FIX: validate strings??     @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: 'aisstr6'     @type startindex: int     @param startindex: bit that begins the int(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum     @type required: restricted str or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: restricted str or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX... handle arrays # Make sure required is a number Build the decoder for signed decimal variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: 'decimal'     @type startindex: int     @param startindex: bit that begins the int(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum     @type required: Decimal or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: Decimal or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX... handle arrays # Warning about this was in the encode section # Make sure required is a number Build the decoder for unsigned decimal variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: 'udecimal'     @type startindex: int     @param startindex: bit that begins the int(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum     @type required: Decimal or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: Decimal or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX... handle arrays # Warning about this was in the encode section # Make sure required is a number and not negative Build the decoder for unsigned decimal variables      @type o: file like obj     @param o: where write the code     @type name: str     @param name: field name     @type type: str     @param type: 'udecimal'     @type startindex: int     @param startindex: bit that begins the int(s)     @type numbits: int >= 1     @param numbits: How many bits per unit datum.  If -1, then read to the end of the message     @type required: Decimal or None     @param required: If not None, then the value must be set to this.     @type arraylen: int >= 1     @param arraylen: many ints will there be?  FIX: handle variable     @type unavailable: Decimal or None     @param unavailable: the default value to use if none given (if not None)     @type bv: str     @param bv: BitVector containing the incoming data     @type dataDict: str     @param dataDict: dictionary in which to place the results     @type decodeOnly: bool     @param decodeOnly: Set to true to only get the code for decoding     @rtype: int     @return: index one past the end of where this read # FIX... handle arrays # What is good max? # FIX: assert not required and not an array an not unavailable # -1 means go to the end of the message ###################################################################### # THE REST ###################################################################### Scrape the testvalues to make a basic param      @bug: FIX: make this create a dictionary that sits in the overall namespace and spit out deep copies? Return a params file base on the testvalue tags.\n    @rtype: dict\n    @return: params based on testvalue tags\n Write the unittests for a message      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node Use testvalue tag text from each type to build test case the "+name+" message # Check that each parameter came through ok.\n') # float, decimal, udecimal # FIX: look up the decimal places if decimal Write the encoder/decoder for a message      http://jaynes.colorado.edu/PythonIdioms.html      @param o: open file where resulting code will be written     @param msgET: Element Tree starting at a message node      TODO(schwehr):handle ais message 20 optional.  very troublesome # fix rename this variable to avoid hiding later on by field name # FIX: verbose? ######################################## # doc string Create a "+name+" binary message payload to pack into an AIS Msg "+name+".\n\n")     o.write('    Fields in params:\n')     for field in msgET.xpath('field'):         if verbose:             print field.tag,field.attrib['name']             print field.attrib         desc = field[0].text.replace('\n',' ') # get ride of new lines         o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description         if len(field.xpath("required")) == 1:             o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')          o.write('\n')     o.write('    @param params: Dictionary of field names/values.  Throws a ValueError exception if required is missing\n')     o.write('    @param validate: Set to true to cause checking to occur.  Runs slower.  FIX: not implemented.\n')      o.write("    @rtype: BitVector\n")     o.write("    @return: encoded binary message (for binary messages, this needs to be wrapped in a msg 8\n")     o.write("    @note: The returned bits may not be 6 bit aligned.  It is up to you to pad out the bits.\n")     o.write(" ######################################## # Actually build the code # Set to true when positioning must be calculated #    o.write('\n    bv=binary.joinBV(bvList)\n') #    o.write('\n    bvLen=len(bv)\n') #    o.write('\n    if bvLen%6!=0:\n') #    o.write('\n        bv = bv + BitVector(size=bvLen%6)\n') #    o.write('\n    return bv\n\n') ###################################################################### # DECODER ONE PART AT A TIME - if only using a small part, save some cycles! #, msgDict=None): Write the decoder for a message      @param o: open file where resulting code will be written     @type msgET: elementtree     @param prefixName: if True, put the name of the message on the functions.     @param msgET: Element Tree starting at a message node     @return: None       TODO(schwehr):FIX: doc strings for each decode!      TODO(schwehr):FIX: check for a dac,fid, or efid.  If exists, then this is an AIS Msg 8 payload      TODO(schwehr):May want to take a dictionary of already decoded fields to speed things that need prior info     for things like variable length arrays # Where we are in the bitvector... FIX: what about variable length jobs? # Follow the same convention of decoding into a dict so that code is the same #o.write('    r={};') #o.write('    return(r[\''+name+'\'])\n\n') ###################################################################### # DECODER RING Write the decoder for a message      @param o: open file where resulting code will be written     @type msgET: elementtree     @param msgET: Element Tree starting at a message node     @return: None       TODO(schwehr):FIX: check for a dac,fid, or efid.  If exists, then this is an AIS Msg 8 payload ######################################## # doc string Unpack a "+name+" message.\n\n")     o.write('    Fields in params:\n')     for field in msgET.xpath('field'):         desc = field[0].text.replace('\n',' ') # get ride of new lines         o.write('      - '+field.attrib['name']+'('+field.attrib['type']+'): '+desc) # give the description         if len(field.xpath("required")) == 1:             o.write(' (field automatically set to "'+field.xpath("required")[0].text+'")')          o.write('\n')     o.write('    @type bv: BitVector\n')     o.write('    @param bv: Bits defining a message\n')     o.write('    @param validate: Set to true to cause checking to occur.  Runs slower.  FIX: not implemented.\n')      o.write("    @rtype: dict\n")     o.write("    @return: params\n")     o.write(" #Would be nice to check the bit count here..\n') #if validate:\n') #    assert (len(bv)==FIX: SOME NUMBER)\n') ######################################## # Actually build the code # Set to true when positioning must be calculated # Where we are in the bitvector... FIX: what about variable length jobs? # NoReturn means that this field is not a point that needs a check to see if we are done # FIX: need to do something more efficient # Found a NoReturn # Message 20 is a pain! # All fields below are optional\n') #def getPythonType(aType): #    ''' #    Translate a #    @rtype: str #    @return: name of the python type #    ''' # FIX: make these appear as a flag variable # FIX: Is this a good choice? Create a function that adds the options to a parse object # FIX: write out a doc string parser.add_option('-d','--decode',dest='doDecode',default=False,action='store_true',                 help='decode a " " AIS message')     parser.add_option('-e','--encode',dest='doEncode',default=False,action='store_true',                 help='encode a " " AIS message') #print 'here...', msgName, prefixName #print 'there',name,fieldType def main():     from optparse import OptionParser     parser = OptionParser(usage="%prog [options]")      parser.add_option('--doc-test',dest='doctest',default=False,action='store_true',         help='run the documentation tests')     parser.add_option('--unit-test',dest='unittest',default=False,action='store_true',         help='run the unit tests')     parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',         help='Make the test output verbose')      # FIX: remove nmea from binary messages.  No way to build the whole packet?     # FIX: or build the surrounding msg 8 for a broadcast?     typeChoices = ('binary','nmeapayload','nmea') # FIX: what about a USCG type message?     parser.add_option('-t', '--type', choices=typeChoices, type='choice',         dest='ioType', default='nmeapayload',         help='What kind of string to write for encoding ('+', '.join(typeChoices)+') [default: %default]')       outputChoices = ('std','html','csv','sql' #    'xml' -  FIX: need to add xml output )     parser.add_option('-T', '--output-type', choices=outputChoices,         type='choice', dest='outputType', default='std',         help='What kind of string to output ('+', '.join(outputChoices)+') '         '[default: %default]')      parser.add_option('-o','--output',dest='outputFileName',default=None,         help='Name of the python file to write [default: stdout]')      parser.add_option('-f', '--fields', dest='fieldList', default=None,         action='append', choices=fieldList,         help='Which fields to include in the output.  Currently only for csv '         'output [default: all]')      parser.add_option('-p', '--print-csv-field-list', dest='printCsvfieldList',         default=False,action='store_true',         help='Print the field name for csv')      parser.add_option('-c', '--sql-create', dest='sqlCreate', default=False,         action='store_true',         help='Print out an sql create command for the table.')      parser.add_option('--latex-table', dest='latexDefinitionTable',         default=False,action='store_true',         help='Print a LaTeX table of the type')      parser.add_option('--text-table', dest='textDefinitionTable', default=False,         action='store_true',         help='Print delimited table of the type (for Word table importing)')      parser.add_option('--delimt-text-table', dest='delimTextDefinitionTable',         default='    ',         help='Delimiter for text table [default: \\\'%default\\\'] '         '(for Word table importing)')      dbChoices = ('sqlite','postgres')     parser.add_option('-D', '--db-type', dest='dbType', default='postgres',         choices=dbChoices,type='choice',         help='What kind of database ('+', '.join(dbChoices)+') '         '[default: %default]') addMsgOptions(parser)      (options,args) = parser.parse_args()     success = True      if options.doctest:             import os; print os.path.basename(sys.argv[0]), 'doctests ...',             sys.argv = [sys.argv[0]]             if options.verbose:               sys.argv.append('-v')              numfail, numtests = doctest.testmod()             if not numfail:                 print 'ok'             else:                 print 'FAILED'                 success = False      if not success: sys.exit('Something Failed')     del success # Hide success from epydoc      if options.unittest:             sys.argv = [sys.argv[0]]             if options.verbose: sys.argv.append('-v')             unittest.main()      outfile = sys.stdout     if None!=options.outputFileName:             outfile = file(options.outputFileName,'w') ############################## # encode ############################## # FIX: make everything tabs.  Grr. # Make sure all non required options are specified.\n') # Build dict bits = (msgDict)     if 'binary' == options.ioType:         print str(bits)     elif 'nmeapayload'==options.ioType:         # FIX: figure out if this might be necessary at compile time         bitLen=len(bits)         if bitLen % 6 != 0:             bits = bits + BitVector(size=(6 - (bitLen%6)))  # Pad out to multiple of 6         print binary.bitvectoais6(bits)[0]      # FIX: Do not emit this option for the binary message payloads.  Does not make sense.     elif 'nmea' == options.ioType:         nmea = uscg.create_nmea(bits)         print nmea     else:         sys.exit('ERROR: unknown ioType.  Help!') ############################## # decode all ############################## if options.sqlCreate:                 sqlCreateStr(outfile,options.fieldList,dbType=options.dbType)          if options.latexDefinitionTable:                 latexDefinitionTable(outfile)          # For conversion to word tables         if options.textDefinitionTable:                 textDefinitionTable(outfile,options.delimTextDefinitionTable)          if options.printCsvfieldList:                 # Make a csv separated list of fields that will be displayed for csv                 if None == options.fieldList: options.fieldList = fieldList                 import StringIO                 buf = StringIO.StringIO()                 for field in options.fieldList:                         buf.write(field+',')                 result = buf.getvalue()                 if result[-1] == ',': print result[:-1]                 else: print result          if options.doDecode:                 if len(args)==0: args = sys.stdin                 for msg in args:                         bv = None                          if msg[0] in ('$','!') and msg[3:6] in ('VDM','VDO'):                                 # Found nmea                                 # FIX: do checksum                                 bv = binary.ais6tobitvec(msg.split(',')[5])                         else: # either binary or nmeapayload... expect mostly nmeapayloads                                 # assumes that an all 0 and 1 string can not be a nmeapayload                                 binaryMsg=True                                 for c in msg:                                         if c not in ('0','1'):                                                 binaryMsg=False                                                 break                                 if binaryMsg:                                         bv = BitVector(bitstring=msg)                                 else: # nmeapayload                                         bv = binary.ais6tobitvec(msg) ( (bv)                                     ,out=outfile                                     ,format=options.outputType                                     ,fieldList=options.fieldList                                     ,dbType=options.dbType                                     )  ############################################################ if __name__=='__main__':     main() ###################################################################### # Restore the original args # hide from epydoc # FIX: Will this exit success? #infile=sys.stdin #if options.xmlFileName: infile = file(options.xmlFileName,'r') #outfile=sys.stdout #if options.outputFile: outfile = file(options.xmlDefinition,'w') # FIX: down the road, it would be good to allow either filenames of std{in/out}