<gh_stars>1-10
#!/usr/bin/env python

import sys,os,glob,copy
import optparse
import logging,traceback
import gzip,zipfile,tempfile
import output,util
import dshell
try: import pcap
except:
    pcap=None
    print 'pcap not available: decoders requiring pcap are not usable'

def import_module(name=None,silent=False,search={}):
    try:
        #we will first check search[name] for the module
        #else split foo.bar:baz to get from foo.bar import baz
        #else split dotted path to perform a 'from foo import bar' operation
        try: module=search[name] #a -> from search[a] import a
        except:
            module,name=name.split('.')[:-1],name.split('.')[-1] #a.b.c from a.b import c
            if module: module='.'.join(module)
            else: module=name
        path=None
        if os.path.sep in module: #was a full path to a decoder given?
            path,module=os.path.dirname(module),os.path.basename(module)
        #print module,name
        if path: sys.path.append(path)
        obj=__import__(module,fromlist=[name])
        if path: sys.path.remove(path)
        if 'dObj' in dir(obj) or 'obj' in dir(obj): return obj
        elif name in dir(obj):
            obj = getattr(obj, name)
            if 'dObj' in dir(obj) or 'obj' in dir(obj): return obj
    except Exception, err:
        if not silent: sys.stderr.write( "Error '%s' loading module %s\n" % (str(err),module))
    return False

def setDecoderPath(decoder_path):
    '''set the base decoder path,
        add it to sys.path for importing,
        and walk it to return all subpaths'''
    paths=[]
    paths.append(decoder_path) #append base path first
    # walk decoder directories an add to sys.path
    for root, dirs, files in os.walk(decoder_path):
        [dirs.remove(d) for d in dirs if d.startswith('.')] #skip hidden dirs like .svn
        for d in sorted(dirs): paths.append(os.path.join(root,d))
    return paths #return the paths we found

def getDecoders(decoder_paths):
    ''' find all decoders and map decoder to import.path.decoder
    expect common prefix to start with basepath'''
    import_base=os.path.commonprefix(decoder_paths).split(os.path.sep)[:-1]  #keep last part as base
    decoders={}
    for path in decoder_paths:
        # split path and trim off part before base
        import_path=path.split(os.path.sep)[len(import_base):]
        for f in glob.iglob("%s/*.py" % path):
            name=os.path.splitext(os.path.basename(f))[0]
            if name != '__init__': #skip package stubs
        #build topdir.path...module name from topdir/dir.../file
                decoders[name]='.'.join(import_path+[name])
    return decoders

def printDecoders(decoder_map,silent=True):
    '''Print list of decoders with additional info'''
    dList = []
    FS='  %-40s %-30s %-10s %s %1s  %s'
    for name,module in sorted(decoder_map.iteritems()):
        try:
            try: decoder=import_module(module,silent).dObj
            except Exception, e:
                print "Exception loading module '%s': %s" % (module, e)
                continue
            # get the type of decoder it is
            dtype = 'RAW'
            if 'IP' in dir(decoder): dtype = 'IP '
            if 'UDP' in dir(decoder): dtype = 'UDP'
            if 'TCP' in dir(decoder): dtype = 'TCP'
            dList.append( FS % (module,decoder.name,
                decoder.author,
                dtype,'+' if decoder.chainable else '',
                decoder.description))
        except: pass

    print FS %('module','name','author','   ',' ','desc')
    print FS %('-'*40,'-'*30,'-'*10,'---','-','-'*50)
    for d in sorted(dList): print d

def readInFilter(fname):
    '''Read in a BPF filter provided by a command line argument'''
    filter = ''
    tmpfd = open(fname,'r')
    for line in tmpfd:
        if '#' in line:
            line = line.split('#')[0]+'\n'    # keep \n for visual output sanity
        filter += line
    tmpfd.close()

    return filter

def decode_live(out,options,decoder,decoder_args,decoder_options):

        #set decoder options
    initDecoderOptions(decoder,out,options,decoder_args,decoder_options)

    if 'preModule' in dir(decoder):
        decoder.preModule()

    decoder.input_file = options.interface      # give the interface name to the decoder
    stats=None
    if options.verbose: log('Attempting to listen on %s' % options.interface)
    try:

        if not pcap: raise NotImplementedError("raw capture support not implemented")
        decoder.capture=pcap.pcap(options.interface,65535,True)
        if decoder.filter: decoder.capture.setfilter(decoder.filter)
        while not options.count or decoder.count<options.count:
            decoder.capture.dispatch(1,decoder.decode) #use dispatch so we can handle signals
    except KeyboardInterrupt:
        pass
    except Exception,e:
        log(str(e),level=logging.ERROR)

    if 'cleanConnectionStore' in dir(decoder):
        decoder.cleanConnectionStore()

    if 'postModule' in dir(decoder):
        decoder.postModule()

def expandCompressedFile(fname,verbose,tmpdir):
    ''' Expand a file compressed with gzip, bzip2, or zip.
    Only handles zip files with 1 file. Need to add handling
    for zip files containing multiple pcap files.'''
    try:
    #print fname
        ext = os.path.splitext(fname)[1]
        if verbose:
            log('+Attempting to process %s compressed file' % (ext))
        if ext == '.gz':
            f = gzip.open(fname,'rb')
        elif ext == '.bz2':
            f = bz2.BZ2File(fname,'r')
        elif ext == '.zip':
            print "Enter password for .zip file [default:none]:",
            pswd=raw_input()
            z = zipfile.ZipFile(fname)
            f = z.open(z.namelist()[0],'r',pswd)
        else:
            log('+Error decompressing %s' % (fname),level=logging.ERROR)
            return

        h = tempfile.NamedTemporaryFile(dir=tmpdir,delete=False)
        if verbose:
            log('+Temp directory: %s' % (tempfile.gettempdir()))
            log('+Expanding to tempfile %s' % (h.name))

        for line in f.readlines():
            h.write(line)
        h.close()
        f.close()
        return h.name
    except:
        return None

# This is a support function for the code in main() that supports
# recursive directory crawling when looking for pcap files to parse
# This function recurses through a directory structure, applying
# the wildcard (if any) from the command line.  If no wildcard
# is specified, then all files are included.
def addFilesFromDirectory(inputs, curDir, wildcard='*'):
    # STEP 1: Add files matching wildcard from current directory...

    # concatenate into path
    full_path = os.path.join(curDir, wildcard)
    inputs.extend(glob.glob(full_path))

    #STEP 2: Recurse into child directories
    for path in os.listdir(curDir):
        fullDir = os.path.join(curDir, path)

        if os.path.isdir(fullDir):
            addFilesFromDirectory(inputs, fullDir, wildcard)

# The default OptionParser will raise an error when it encounters an option that
# it doesn't understand.  By creating a new option parser, dshellOptionParser,
# we can ignore the unknown options (read: Module specific options)
class dshellOptionParser(optparse.OptionParser):

    def error(self,msg): pass

    #create options for all loaded decoders
    def add_decoder_options(self,d):
        if d.subDecoder: self.add_decoder_options(d.subDecoder) #if we have a subdecoder, recurse down until we don't
        try:
            if d.optiondict:
                group = optparse.OptionGroup(self, "%s decoder options" % d.name)
                for argname, optargs in d.optiondict.iteritems():
                    optname="%s_%s" % (d.name, argname)
                    group.add_option("--"+optname, dest=optname, **optargs)
                self.add_option_group(group)
        except: raise

    #pass thru to parse_args, but add in kwargs
    def parse_args(self,args,**kwargs):
        try:
            options,args=optparse.OptionParser.parse_args(self,args)
            options.__dict__.update(kwargs)
        except UnboundLocalError: # probably missing a value for an argument, e.g. 'decode -d' without a decoder
            self.print_help()
            return None, None
        return options,args

        # Fix for handling unknown options (e.g. decoder-specific options)
        # reference: http://stackoverflow.com/questions/1885161/how-can-i-get-optparses-optionparser-to-ignore-invalid-arguments
    def _process_args(self, largs, rargs, values):
        while rargs:
            try:
                optparse.OptionParser._process_args(self,largs,rargs,values)
            except (optparse.BadOptionError,optparse.AmbiguousOptionError), e:
                largs.append(e.opt_str)

def printDecoderBriefs(decoders):
    """Prints a brief overview of a decoder when using --help with a decoder"""
    print
    for d in decoders.values():
        print 'Module name:',d.name
        print '='*20
        if d.longdescription: print d.longdescription
        else: print d.description
        print 'Default filter: %s' % (d.filter)
    return


def initDecoderOptions(decoder, out, options, decoder_args,decoder_options):
    """
    pass global config to decoder
    """

    #recurse from the bottom of the chain to the top
    if decoder.subDecoder:
        initDecoderOptions(decoder.subDecoder, out, options, decoder_args, decoder_options)

    # give the decoder the output object if the decoder doesn't pick one
    # or if an output object is specified via command line options
    if not decoder.out or options.output != 'output':
        decoder.out=out
    else: #initialize the decoder's custom output using the channels from the global
        decoder.globalout=out #provide global output module under alternate name
#        decoder.out.__init__(fh=out.fh) #re-init the decoder
        try:
        # If the decoder's default output doesn't have a filehandle set, use the user provided one
            if decoder.out.fh == sys.stdout: decoder.out.fh = out.fh
        except AttributeError:
            # A filehandle doesn't always exist, such as with QueueOutput
            pass
        if not decoder.out.sessionwriter: decoder.out.sessionwriter = out.sessionwriter
        if not decoder.out.pcapwriter: decoder.out.pcapwriter = out.pcapwriter
    #set the logger
    decoder.out.logger=logging.getLogger(decoder.name)

    #set output format string, or reset to default
    #do not override --oformat specified string
    if decoder.format and not options.oformat: decoder.out.setformat(decoder.format)

    #set verbosity
    decoder.verbose = options.verbose
    if options.debug: decoder._DEBUG = options.debug #debug() is already taken, and _DEBUG might already be set

    #override decoder BPF
    if options.bpf != None: decoder.filter = options.bpf

    #override decoder filterfn
    if options.nofilterfn: decoder.filterfn = lambda addr: True

    #read BPF from file
    if options.filefilter != None:
        try:
            tmpbpf = readInFilter(options.filefilter)
        except:
            log("Invalid tcpdump filter file: %s" % (options.filefilter),level=logging.ERROR)
            return

        decoder.filter = tmpbpf

    # extend bpf filter if necessary
    if options.ebpf != None:
        ebpf = options.ebpf
        if not decoder.filter:
            decoder.filter = ebpf
        elif ebpf.startswith('or '):
            decoder.filter = decoder.filter + ' ' + ebpf
        else:
            decoder.filter = decoder.filter + ' and ' + ebpf

    # do we change the layer-2 decoder for raw capture
    if options.layer2:
        import dpkt
        decoder.l2decoder=eval('dpkt.'+options.layer2)

    #strip extra layers?
    if options.striplayers: decoder.striplayers=int(options.striplayers)

    if not options.novlan and not(decoder.filter.startswith('vlan')):
        if decoder.filter:
            decoder.filter = '( '+decoder.filter+' ) or ( vlan and ( '+decoder.filter+' ) )'
        else: decoder.filter='' #fix for null filter case

    #pass args and config file to decoder
    decoder.parseArgs(decoder_args,decoder_options)

    log('Using module '+repr(decoder))


def main(*largs,**kwargs):
    global log
    bin_path = os.environ['BINPATH']
    sys.path.insert(0,bin_path)
    #get map of name to module import path
    decoder_map=getDecoders(setDecoderPath(os.environ['DECODERPATH']))

    # The main argument parser. It will have every command line option available and should be used when actually parsing
    parser = dshellOptionParser(usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]", version="%prog "+str(dshell.__version__), add_help_option=False)
    # A short argument parser, meant to only hold the shorter list of arguments for when a decoder is called without a pcap file. DO NOT USE for any serious argument parsing.
    parser_short = dshellOptionParser(usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]", version="%prog "+str(dshell.__version__), add_help_option=False)
    parser.add_option('-h', '-?', '--help', dest='help', help="Print common command-line flags and exit", action='store_true', default=False)
    parser_short.add_option('-h', '-?', '--help', dest='help', help="Print common command-line flags and exit", action='store_true', default=False)
    parser.add_option('-d','--decoder',dest="decoder",action='append',help="Use a specific decoder module")
    parser.add_option('-l','--ls','--list',action="store_true",help='List all available decoders',dest='list')
    parser.add_option('-C','--config',dest='config',help='specify config.ini file')
    parser.add_option('--tmpdir', dest='tmpdir', type='string', default=tempfile.gettempdir(), help='alternate temp directory (for use when processing compressed pcap files)')
    parser.add_option('-r','--recursive',dest='recursive', action='store_true',help='recursively process all PCAP files under input directory')

    group = optparse.OptionGroup(parser,"Multiprocessing options")
    group.add_option('-p','--parallel',dest='parallel',action='store_true',help='process multiple files in parallel')
    group.add_option('-t','--threaded',dest='threaded',action='store_true',help='run multiple decoders in parallel')
    group.add_option('-n','--nprocs',dest='numprocs',type='int',default=4,help='number of simultaneous processes')
    parser.add_option_group(group)

    # decode-pcap specific options
    group = optparse.OptionGroup(parser,"Input options")
    group.add_option('-i','--interface',dest='interface',default=None,help='listen live on INTERFACE')
    group.add_option('-c','--count',dest='count',type='int',help='number of packets to process',default=0)
    group.add_option('-f','--bpf',dest='bpf',help='replace default decoder filter (use carefully)')
    group.add_option('--nofilterfn',dest='nofilterfn',action="store_true",help='Set filterfn to pass-thru')
    group.add_option('-F',dest='filefilter',help='Use filefilter as input for the filter expression.  An additional expression given on the command line is ignored.')
    group.add_option('--ebpf',dest='ebpf',help='BPF filter to exclude traffic, extends other filters')
    group.add_option('--no-vlan',dest='novlan',action="store_true",help='do not examine traffic which has VLAN headers present')
    group.add_option('--layer2',dest='layer2',default='ethernet.Ethernet',help='select the layer-2 protocol module')
    group.add_option('--strip',dest='striplayers',default=0,help='extra data-link layers to strip')
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser_short,"Input options")
    group.add_option('-i','--interface',dest='interface',default=None,help='listen live on INTERFACE')
    group.add_option('-c','--count',dest='count',type='int',help='number of packets to process',default=0)
    group.add_option('-f','--bpf',dest='bpf',help='replace default decoder filter (use carefully)')
    group.add_option('--nofilterfn',dest='nofilterfn',action="store_true",help='Set filterfn to pass-thru')
    group.add_option('-F',dest='filefilter',help='Use filefilter as input for the filter expression.  An additional expression given on the command line is ignored.')
    group.add_option('--ebpf',dest='ebpf',help='BPF filter to exclude traffic, extends other filters')
    group.add_option('--no-vlan',dest='novlan',action="store_true",help='do not examine traffic which has VLAN headers present')
    group.add_option('--layer2',dest='layer2',default='ethernet.Ethernet',help='select the layer-2 protocol module')
    group.add_option('--strip',dest='striplayers',default=0,help='extra data-link layers to strip')
    parser_short.add_option_group(group)


    group = optparse.OptionGroup(parser,"Output options")
    group.add_option('-o','--outfile',dest='outfile',help='write output to the file OUTFILE. Additional output can be set with KEYWORD=VALUE,...\n'+\
        '\tmode=<w: write (default), a: append, noclobber: do not overwrite, use a  a OUTFILE.1 (.2,.3) file if file(s) exists\n'+\
        '\tpcap=PCAPFILE to write packets to a PCAP file\n'+\
        '\tsession=SESSION to write session text\n'+\
        '\tdirection=data direction to write (c,s,both,split)')
    group.add_option('-w','--session',dest='session',help='write session file, same as -o session=')
    group.add_option('-W','--pcap',dest='pcap',default=None,help='output decoded packets to PCAP (same as -o pcap=....)')
    group.add_option('--db',dest='db',default=None,help='output to db. Supply "config=file" or "param=...,param=..." ')
    group.add_option('--oformat',dest='oformat',help='define the output format')
    group.add_option('-x','--extra',dest='oextra',action='store_true',help='output a lot of extra information')
    group.add_option('-O','--output',dest='output',default=None,help='Use a custom output module. Supply "modulename,option=value,..."')
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser,"Logging options")
    group.add_option('-L','--logfile',dest="logfile",help="log to file")
    group.add_option('--debug',action="store_true", dest="debug",help="debug logging (debug may also affect decoding behavior)")
    group.add_option('-v','--verbose',action="store_true", dest="verbose",help="verbose logging")
    group.add_option('-q','--quiet',action="store_true", dest="quiet",help="practically zero logging")
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser_short,"Logging options")
    group.add_option('-L','--logfile',dest="logfile",help="log to file")
    group.add_option('--debug',action="store_true", dest="debug",help="debug logging (debug may also affect decoding behavior)")
    group.add_option('-v','--verbose',action="store_true", dest="verbose",help="verbose logging")
    group.add_option('-q','--quiet',action="store_true", dest="quiet",help="practically zero logging")
    parser_short.add_option_group(group)


    decoder_options={} #[decoder][option]=value dict of decoder options, set by config file
    decoder_args = []
    args = []
    extra_args = False
    for x in largs:
        if x == '--':
            extra_args = True
            continue
        if extra_args:
            decoder_args.append(x)
        else:
            args.append(x)

    #parse basic options and crdate the options object
    options = parser.parse_args(args,**kwargs)[0]

    if options == None:
        print "\nError processing provided arguments"
        return

    #dump list
    if options.list:
        printDecoders(decoder_map,not options.debug)
        return

    #parse config file, updating the options and decoder_options dicts
    if options.config:
        try:
            import ConfigParser
            config = ConfigParser.ConfigParser()
            config.read(options.config)
            for s in config.sections():
                if s.lower()=='dshell': #this is the main section, set the options
                    for k,v in config.items(s,raw=True):
                        if k in options.__dict__: options.__dict__[k]=v
        except: raise

    #are we a thread outputting to a queue?
    if 'queue' in options.__dict__: out=output.QueueOutput(options.queue)
    #if not, parse output args
    else:
        outfile=None
        outkw={}

        #set output file (and other args if -o filename,key=val...)
        if options.outfile: outfile,outkw=util.strtok(options.outfile)
        #output extra?
        if options.oextra: outkw.update(extra=True)
        #set session writer?
        if options.session: outkw.update(session=options.session)
        #add default pcap writer?
        if options.pcap: outkw.update(pcap=options.pcap)
        #use database?
        if options.db:
            a,kw=util.strtok(options.db,as_list=True)
            #add output options
            kw.update(outkw)
            out=output.DBOutput(*a,**kw)
        #if not db mode and no out module specd
        # use default output lib to get default module
        elif not options.output: options.output='output'
        #init output module
        if options.output:
            a,kw=util.strtok(options.output,as_list=True) #parse output arglist (-O module,args..,k=val...)
            kw.update(outkw) #set output options
            if outfile: kw.update(file=outfile) #set filename arg if -o given (can also be first arg in module arglist)
            outmod = import_module(name=os.path.basename(a[0])) #load module
            if outmod: out=outmod.obj(*a[1:],**kw) #pass remaining args and keyword args to init object

        #set the output format
        if options.oformat != None: out.setformat(options.oformat)

    #set global log functions
    out.logger=logging.getLogger('dshell')
    log=out.log

    #start up the logger
    if options.debug:       level=logging.DEBUG
    elif options.verbose:   level=logging.INFO
    elif options.quiet:     level=logging.FATAL
    else:                   level=logging.WARNING
    logging.basicConfig(filename=options.logfile,level=level)

    decoders = {}
    decoderNames = set()
    # check for a decoder
    if options.decoder != None:
        if type(options.decoder)==str: #single decoder or came from config file
            options.decoder=util.strtok(options.decoder,as_list=True)[0] #make it a list
        # we have a list of decoders
        for dnames in options.decoder:
            chain=dnames.split('+')
            #last module does not have a subdecoder
            module=chain.pop()
            try: module,n=module.split(':',1)
            except: n=None
            m=import_module(module,search=decoder_map)
            if m:
                dObj=copy.copy(m.dObj) #create copy in case we import multiple times under different names
                if n: dObj.name=n
            else: dObj=None
            try: decoderNames.add(dObj.name)
            except AttributeError: decoderNames.add(module)
            #walk up the chain, setting sub-decoders
            while chain:
                subObj=dObj
                module=chain.pop()
                try: module,n=module.split(':',1)
                except: n=None
                m=import_module(module,search=decoder_map)
                if m:
                    dObj=copy.copy(m.dObj)
                    if n: dObj.name=n
                else: dObj=None
                try: decoderNames.add(dObj.name)
                except AttributeError: decoderNames.add(module)
                if dObj and dObj.chainable: dObj.subDecoder=subObj
                elif dObj: sys.stderr.write( "Error %s is not chainable\n"%module)
            #insert the top decoder in the dict
            if dObj: decoders[dObj.name]=dObj

    #save option dict
    options_dict=options.__dict__.copy()

    #add in options for loaded decoders and subdecoders
    for d in decoders.itervalues(): parser.add_decoder_options(d)
    for d in decoders.itervalues(): parser_short.add_decoder_options(d)

    #reparse args to handle decoder options
    optionerror=False
    try: options,args=parser.parse_args(args,**kwargs)
    except: optionerror=True

    #replace base options
    options.__dict__.update(options_dict)

    #look for name_option keys and put them in decoder_options[name][option]
    for k,v in options.__dict__.iteritems():
        for decName in decoderNames:
            try:
                n=k.split(decName+'_', 1)[1]
                decoder_options.setdefault(decName,{}).setdefault(n,v)
            except IndexError: continue

    #reparse config file to handle decoder options
    if options.config:
        for s in config.sections():
            if s.lower() in decoder_options: #set the options for loaded decoders if they are present in the config file
                for k,v in config.items(s,raw=True):
                    if k in decoder_options[s]: #if this is a valid option
                        if v.isdigit(): v=int(v) #try conversion to int/float
                        elif '.' in v:
                            try: v=float(v)
                            except: pass
                        decoder_options[s][k]=v

    if any(x in ('-h', '-?', '--help') for x in sys.argv[1:]):
            # Print the verbose help message
        parser.print_help()
        printDecoderBriefs(decoders)
        return

    if optionerror or (not args and not options.interface):
        # Print the short help message
        parser_short.print_help()
        printDecoderBriefs(decoders)
        return


        #####################################################################################
        # listen live on the interface
        # this will not process any files
    if options.interface != None:
        if len(decoders)!=1:
            print 'Can only run one module live on an interface'
            return

        # handles options and arguments for dumping live on an interface
        decode_live(out,options,dObj, decoder_args,decoder_options)

        #close output
        out.close()

        return
        #####################################################################################

    # take all other command line arguments as files to process


    ####################################################
    # Works if directory (e.g. ~/data/) or * (e.g. ~/data/*
    # used on command line.  Does not work for partial
    # wildcards (e.g. ~/data/*.dat) because bash
    # expands wildcards before passing arguments into
    # decode-pcap.py.  Will work if no matches in root of
    # path specified.
    ####################################################
    inputs = []
    for file_path in args:
        # If this argument is a directory and RECURSIVE specified, then add
        # entire directory tree to the list of input files
        if os.path.isdir(file_path) and options.recursive:
            addFilesFromDirectory(inputs,file_path)

        # If a wildcard is specified, then handle accordingly
        elif file_path.find('*') > -1:
            (path,wildcard) = os.path.split(file_path)

            # If just file is specified (no path)
            if len(path) == 0:
                inputs.extend(glob.glob(wildcard))

            # If there is a path, but recursion not specified,
            # then just add matching files from specified dir
            elif not len(path) == 0 and not options.recursive:
                inputs.extend(glob.glob(file_path))

            # Otherwise, recursion specified and there is a directory.
            # Recurse directory and add files
            else:
                addFilesFromDirectory(inputs,path,wildcard)

        # Just a normal file, append to list of inputs
        else:
            inputs.append(file_path)

    if options.parallel or options.threaded:
        import multiprocessing
        procs=[]
        q=multiprocessing.Queue()
        kwargs=options.__dict__.copy() #put parsed base options in kwargs
        kwargs.update(config=None,outfile=None,queue=q) #pass the q,
        #do not pass the config file or outfile because we handled that here
        for d in decoder_options: #put pre-parsed decoder options in kwargs
            for k,v in decoder_options[d].items(): kwargs[d+'_'+k]=v

    #check here to see if we are running in parallel-file mode
    if options.parallel and len(inputs)>1:
        for f in inputs:
            #create a child process for each input file
            procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=[f]))
        runChildProcs(procs,q,out,numprocs=options.numprocs)

    #check here to see if we are running decoders multithreaded
    elif options.threaded and len(options.decoder)>1:
        for d in options.decoder:
            #create a child for each decoder
            kwargs.update(decoder=d)
            procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=inputs))
        runChildProcs(procs,q,out,numprocs=options.numprocs)

    #fall through to here (single threaded or child process)
    else:
        #
        # Here is where we use the decoder(s) to process the pcap
        #

        temporaryFiles = []    # used when uncompressing files

        for module in decoders.keys():
            decoder = decoders[module]
            initDecoderOptions(decoder,out,options,decoder_args,decoder_options)

            # If the decoder has a preModule function, will execute it now
            decoder.preModule()


            for input_file in inputs:
            # Decoder-specific options may be seen as input files
            # Skip anything starts with "--"
                if input_file[:2] == '--':
                    continue

                # Recursive directory processing is handled elsewhere,
                # so we should only be dealing with files at this point.
                if os.path.isdir(input_file): continue

                log('+Processing file %s' % input_file)

                # assume the input_file is not compressed
                # Allows the processing of .pcap files that are compressed with
                # gzip, bzip2, or zip. Writes uncompressed file to a
                # NamedTemporaryFile and unlinks the file once it is no longer
                # needed. Might consider using mkstemp() since this implementation
                # requires Python >= 2.6.
                try:
                    exts = ['.gz','.bz2','.zip']
                    if os.path.splitext(input_file)[1] not in exts:
                        pcapfile=input_file

                    else:
                        # we have a compressed file
                        tmpfile = expandCompressedFile(input_file,options.verbose,options.tmpdir)
                        temporaryFiles.append(tmpfile)
                        pcapfile=tmpfile
                except:
                    if options.verbose: sys.stderr.write('+Error processing file %s' % (input_file))
                    continue

                # give the decoder access to the input filename
                # motivation: run a decoder against a large number of pcap
                #             files and have the decoder print the filename
                #             so you can go straight to the pcap file for
                #             further analysis
                decoder.input_file = input_file

                # Check to see if the decoder has a preFile function
                # This will be called before the decoder processes each
                # input file
                decoder.preFile()

                try:
                    if not pcap: raise NotImplementedError("pcap support not implemented")
                    decoder.capture=pcap.pcap(pcapfile)
                    if decoder.filter: decoder.capture.setfilter(decoder.filter)
                    while not options.count or decoder.count<options.count:
                        try: ts,pkt=decoder.capture.next() #read next packet and break on EOF
                        except: break #no data
                        decoder.decode(ts,pkt)
                except KeyboardInterrupt: raise
                except: traceback.print_exc()

                if options.verbose:
                    log('+Done processing %s' % (input_file))

                # call that decoder's processFile()
                decoder.postFile()

            # check to see if the decoder is using the Messages class
            # if so, we need to clean up the connection store to
            # purge any unfinished connections
            if 'cleanConnectionStore' in dir(decoder): decoder.cleanConnectionStore()

            # Check to see if the decoder has a postModule function
            # A postModule function will be called when the module
            # has finished running against all of the input files
            if 'postModule' in dir(decoder): decoder.postModule()

        # remove any temporary files that were created during execution
        for tmpfile in temporaryFiles:
            if options.verbose: log('+Unlinking %s'%(tmpfile))
            os.unlink(tmpfile)

    #close output
    out.close()
    return

def runChildProcs(procs,q,out,numprocs=4):
    import Queue
    running=[]
    while procs or running: #while we still have processes to spawn or running
        if procs and len(running)<numprocs:
            proc=procs.pop(0)
            proc.start()
            out.log('started %d'%proc.pid,level=logging.INFO)
            running.append(proc)
        for proc in running:
            if not proc.is_alive(): #see if it finished
                out.log('%d exited (%d)'%(proc.pid,proc.exitcode),level=logging.INFO)
                running.remove(proc)
        try: #get from the output queue until empty
            while True:
                m,args,kw=q.get(True,1) #method, listargs, kwargs
                out.dispatch(m,*args,**kw) #dispatch to method
        except Queue.Empty: pass #q empty


if __name__ == '__main__':
    try: main(*sys.argv[1:])
    except KeyboardInterrupt: sys.exit(0)

#!/usr/bin/env python

import sys,os,glob,copy
import optparse
import logging,traceback
import gzip,zipfile,tempfile
import output,util
import dshell
try: import pcap
except:
    pcap=None
    print 'pcap not available: decoders requiring pcap are not usable'

def import_module(name=None,silent=False,search={}):
    try:
        #we will first check search[name] for the module
        #else split foo.bar:baz to get from foo.bar import baz
        #else split dotted path to perform a 'from foo import bar' operation
        try: module=search[name] #a -> from search[a] import a
        except:
            module,name=name.split('.')[:-1],name.split('.')[-1] #a.b.c from a.b import c
            if module: module='.'.join(module)
            else: module=name
        path=None
        if os.path.sep in module: #was a full path to a decoder given?
            path,module=os.path.dirname(module),os.path.basename(module)
        #print module,name
        if path: sys.path.append(path)
        obj=__import__(module,fromlist=[name])
        if path: sys.path.remove(path)
        if 'dObj' in dir(obj) or 'obj' in dir(obj): return obj
        elif name in dir(obj):
            obj = getattr(obj, name)
            if 'dObj' in dir(obj) or 'obj' in dir(obj): return obj
    except Exception, err:
        if not silent: sys.stderr.write( "Error '%s' loading module %s\n" % (str(err),module))
    return False

def setDecoderPath(decoder_path):
    '''set the base decoder path,
        add it to sys.path for importing,
        and walk it to return all subpaths'''
    paths=[]
    paths.append(decoder_path) #append base path first
    # walk decoder directories an add to sys.path
    for root, dirs, files in os.walk(decoder_path):
        [dirs.remove(d) for d in dirs if d.startswith('.')] #skip hidden dirs like .svn
        for d in sorted(dirs): paths.append(os.path.join(root,d))
    return paths #return the paths we found

def getDecoders(decoder_paths):
    ''' find all decoders and map decoder to import.path.decoder
    expect common prefix to start with basepath'''
    import_base=os.path.commonprefix(decoder_paths).split(os.path.sep)[:-1]  #keep last part as base
    decoders={}
    for path in decoder_paths:
        # split path and trim off part before base
        import_path=path.split(os.path.sep)[len(import_base):]
        for f in glob.iglob("%s/*.py" % path):
            name=os.path.splitext(os.path.basename(f))[0]
            if name != '__init__': #skip package stubs
        #build topdir.path...module name from topdir/dir.../file
                decoders[name]='.'.join(import_path+[name])
    return decoders

def printDecoders(decoder_map,silent=True):
    '''Print list of decoders with additional info'''
    dList = []
    FS='  %-40s %-30s %-10s %s %1s  %s'
    for name,module in sorted(decoder_map.iteritems()):
        try:
            try: decoder=import_module(module,silent).dObj
            except Exception, e:
                print "Exception loading module '%s': %s" % (module, e)
                continue
            # get the type of decoder it is
            dtype = 'RAW'
            if 'IP' in dir(decoder): dtype = 'IP '
            if 'UDP' in dir(decoder): dtype = 'UDP'
            if 'TCP' in dir(decoder): dtype = 'TCP'
            dList.append( FS % (module,decoder.name,
                decoder.author,
                dtype,'+' if decoder.chainable else '',
                decoder.description))
        except: pass

    print FS %('module','name','author','   ',' ','desc')
    print FS %('-'*40,'-'*30,'-'*10,'---','-','-'*50)
    for d in sorted(dList): print d

def readInFilter(fname):
    '''Read in a BPF filter provided by a command line argument'''
    filter = ''
    tmpfd = open(fname,'r')
    for line in tmpfd:
        if '#' in line:
            line = line.split('#')[0]+'\n'    # keep \n for visual output sanity
        filter += line
    tmpfd.close()

    return filter

def decode_live(out,options,decoder,decoder_args,decoder_options):

        #set decoder options
    initDecoderOptions(decoder,out,options,decoder_args,decoder_options)

    if 'preModule' in dir(decoder):
        decoder.preModule()

    decoder.input_file = options.interface      # give the interface name to the decoder
    stats=None
    if options.verbose: log('Attempting to listen on %s' % options.interface)
    try:

        if not pcap: raise NotImplementedError("raw capture support not implemented")
        decoder.capture=pcap.pcap(options.interface,65535,True)
        if decoder.filter: decoder.capture.setfilter(decoder.filter)
        while not options.count or decoder.count<options.count:
            decoder.capture.dispatch(1,decoder.decode) #use dispatch so we can handle signals
    except KeyboardInterrupt:
        pass
    except Exception,e:
        log(str(e),level=logging.ERROR)

    if 'cleanConnectionStore' in dir(decoder):
        decoder.cleanConnectionStore()

    if 'postModule' in dir(decoder):
        decoder.postModule()

def expandCompressedFile(fname,verbose,tmpdir):
    ''' Expand a file compressed with gzip, bzip2, or zip.
    Only handles zip files with 1 file. Need to add handling
    for zip files containing multiple pcap files.'''
    try:
    #print fname
        ext = os.path.splitext(fname)[1]
        if verbose:
            log('+Attempting to process %s compressed file' % (ext))
        if ext == '.gz':
            f = gzip.open(fname,'rb')
        elif ext == '.bz2':
            f = bz2.BZ2File(fname,'r')
        elif ext == '.zip':
            print "Enter password for .zip file [default:none]:",
            pswd=raw_input()
            z = zipfile.ZipFile(fname)
            f = z.open(z.namelist()[0],'r',pswd)
        else:
            log('+Error decompressing %s' % (fname),level=logging.ERROR)
            return

        h = tempfile.NamedTemporaryFile(dir=tmpdir,delete=False)
        if verbose:
            log('+Temp directory: %s' % (tempfile.gettempdir()))
            log('+Expanding to tempfile %s' % (h.name))

        for line in f.readlines():
            h.write(line)
        h.close()
        f.close()
        return h.name
    except:
        return None

# This is a support function for the code in main() that supports
# recursive directory crawling when looking for pcap files to parse
# This function recurses through a directory structure, applying
# the wildcard (if any) from the command line.  If no wildcard
# is specified, then all files are included.
def addFilesFromDirectory(inputs, curDir, wildcard='*'):
    # STEP 1: Add files matching wildcard from current directory...

    # concatenate into path
    full_path = os.path.join(curDir, wildcard)
    inputs.extend(glob.glob(full_path))

    #STEP 2: Recurse into child directories
    for path in os.listdir(curDir):
        fullDir = os.path.join(curDir, path)

        if os.path.isdir(fullDir):
            addFilesFromDirectory(inputs, fullDir, wildcard)

# The default OptionParser will raise an error when it encounters an option that
# it doesn't understand.  By creating a new option parser, dshellOptionParser,
# we can ignore the unknown options (read: Module specific options)
class dshellOptionParser(optparse.OptionParser):

    def error(self,msg): pass

    #create options for all loaded decoders
    def add_decoder_options(self,d):
        if d.subDecoder: self.add_decoder_options(d.subDecoder) #if we have a subdecoder, recurse down until we don't
        try:
            if d.optiondict:
                group = optparse.OptionGroup(self, "%s decoder options" % d.name)
                for argname, optargs in d.optiondict.iteritems():
                    optname="%s_%s" % (d.name, argname)
                    group.add_option("--"+optname, dest=optname, **optargs)
                self.add_option_group(group)
        except: raise

    #pass thru to parse_args, but add in kwargs
    def parse_args(self,args,**kwargs):
        try:
            options,args=optparse.OptionParser.parse_args(self,args)
            options.__dict__.update(kwargs)
        except UnboundLocalError: # probably missing a value for an argument, e.g. 'decode -d' without a decoder
            self.print_help()
            return None, None
        return options,args

        # Fix for handling unknown options (e.g. decoder-specific options)
        # reference: http://stackoverflow.com/questions/1885161/how-can-i-get-optparses-optionparser-to-ignore-invalid-arguments
    def _process_args(self, largs, rargs, values):
        while rargs:
            try:
                optparse.OptionParser._process_args(self,largs,rargs,values)
            except (optparse.BadOptionError,optparse.AmbiguousOptionError), e:
                largs.append(e.opt_str)

def printDecoderBriefs(decoders):
    """Prints a brief overview of a decoder when using --help with a decoder"""
    print
    for d in decoders.values():
        print 'Module name:',d.name
        print '='*20
        if d.longdescription: print d.longdescription
        else: print d.description
        print 'Default filter: %s' % (d.filter)
    return


def initDecoderOptions(decoder, out, options, decoder_args,decoder_options):
    """
    pass global config to decoder
    """

    #recurse from the bottom of the chain to the top
    if decoder.subDecoder:
        initDecoderOptions(decoder.subDecoder, out, options, decoder_args, decoder_options)

    # give the decoder the output object if the decoder doesn't pick one
    # or if an output object is specified via command line options
    if not decoder.out or options.output != 'output':
        decoder.out=out
    else: #initialize the decoder's custom output using the channels from the global
        decoder.globalout=out #provide global output module under alternate name
#        decoder.out.__init__(fh=out.fh) #re-init the decoder
        try:
        # If the decoder's default output doesn't have a filehandle set, use the user provided one
            if decoder.out.fh == sys.stdout: decoder.out.fh = out.fh
        except AttributeError:
            # A filehandle doesn't always exist, such as with QueueOutput
            pass
        if not decoder.out.sessionwriter: decoder.out.sessionwriter = out.sessionwriter
        if not decoder.out.pcapwriter: decoder.out.pcapwriter = out.pcapwriter
    #set the logger
    decoder.out.logger=logging.getLogger(decoder.name)

    #set output format string, or reset to default
    #do not override --oformat specified string
    if decoder.format and not options.oformat: decoder.out.setformat(decoder.format)

    #set verbosity
    decoder.verbose = options.verbose
    if options.debug: decoder._DEBUG = options.debug #debug() is already taken, and _DEBUG might already be set

    #override decoder BPF
    if options.bpf != None: decoder.filter = options.bpf

    #override decoder filterfn
    if options.nofilterfn: decoder.filterfn = lambda addr: True

    #read BPF from file
    if options.filefilter != None:
        try:
            tmpbpf = readInFilter(options.filefilter)
        except:
            log("Invalid tcpdump filter file: %s" % (options.filefilter),level=logging.ERROR)
            return

        decoder.filter = tmpbpf

    # extend bpf filter if necessary
    if options.ebpf != None:
        ebpf = options.ebpf
        if not decoder.filter:
            decoder.filter = ebpf
        elif ebpf.startswith('or '):
            decoder.filter = decoder.filter + ' ' + ebpf
        else:
            decoder.filter = decoder.filter + ' and ' + ebpf

    # do we change the layer-2 decoder for raw capture
    if options.layer2:
        import dpkt
        decoder.l2decoder=eval('dpkt.'+options.layer2)

    #strip extra layers?
    if options.striplayers: decoder.striplayers=int(options.striplayers)

    if not options.novlan and not(decoder.filter.startswith('vlan')):
        if decoder.filter:
            decoder.filter = '( '+decoder.filter+' ) or ( vlan and ( '+decoder.filter+' ) )'
        else: decoder.filter='' #fix for null filter case

    #pass args and config file to decoder
    decoder.parseArgs(decoder_args,decoder_options)

    log('Using module '+repr(decoder))


def main(*largs,**kwargs):
    global log
    bin_path = os.environ['BINPATH']
    sys.path.insert(0,bin_path)
    #get map of name to module import path
    decoder_map=getDecoders(setDecoderPath(os.environ['DECODERPATH']))

    # The main argument parser. It will have every command line option available and should be used when actually parsing
    parser = dshellOptionParser(usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]", version="%prog "+str(dshell.__version__), add_help_option=False)
    # A short argument parser, meant to only hold the shorter list of arguments for when a decoder is called without a pcap file. DO NOT USE for any serious argument parsing.
    parser_short = dshellOptionParser(usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]", version="%prog "+str(dshell.__version__), add_help_option=False)
    parser.add_option('-h', '-?', '--help', dest='help', help="Print common command-line flags and exit", action='store_true', default=False)
    parser_short.add_option('-h', '-?', '--help', dest='help', help="Print common command-line flags and exit", action='store_true', default=False)
    parser.add_option('-d','--decoder',dest="decoder",action='append',help="Use a specific decoder module")
    parser.add_option('-l','--ls','--list',action="store_true",help='List all available decoders',dest='list')
    parser.add_option('-C','--config',dest='config',help='specify config.ini file')
    parser.add_option('--tmpdir', dest='tmpdir', type='string', default=tempfile.gettempdir(), help='alternate temp directory (for use when processing compressed pcap files)')
    parser.add_option('-r','--recursive',dest='recursive', action='store_true',help='recursively process all PCAP files under input directory')

    group = optparse.OptionGroup(parser,"Multiprocessing options")
    group.add_option('-p','--parallel',dest='parallel',action='store_true',help='process multiple files in parallel')
    group.add_option('-t','--threaded',dest='threaded',action='store_true',help='run multiple decoders in parallel')
    group.add_option('-n','--nprocs',dest='numprocs',type='int',default=4,help='number of simultaneous processes')
    parser.add_option_group(group)

    # decode-pcap specific options
    group = optparse.OptionGroup(parser,"Input options")
    group.add_option('-i','--interface',dest='interface',default=None,help='listen live on INTERFACE')
    group.add_option('-c','--count',dest='count',type='int',help='number of packets to process',default=0)
    group.add_option('-f','--bpf',dest='bpf',help='replace default decoder filter (use carefully)')
    group.add_option('--nofilterfn',dest='nofilterfn',action="store_true",help='Set filterfn to pass-thru')
    group.add_option('-F',dest='filefilter',help='Use filefilter as input for the filter expression.  An additional expression given on the command line is ignored.')
    group.add_option('--ebpf',dest='ebpf',help='BPF filter to exclude traffic, extends other filters')
    group.add_option('--no-vlan',dest='novlan',action="store_true",help='do not examine traffic which has VLAN headers present')
    group.add_option('--layer2',dest='layer2',default='ethernet.Ethernet',help='select the layer-2 protocol module')
    group.add_option('--strip',dest='striplayers',default=0,help='extra data-link layers to strip')
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser_short,"Input options")
    group.add_option('-i','--interface',dest='interface',default=None,help='listen live on INTERFACE')
    group.add_option('-c','--count',dest='count',type='int',help='number of packets to process',default=0)
    group.add_option('-f','--bpf',dest='bpf',help='replace default decoder filter (use carefully)')
    group.add_option('--nofilterfn',dest='nofilterfn',action="store_true",help='Set filterfn to pass-thru')
    group.add_option('-F',dest='filefilter',help='Use filefilter as input for the filter expression.  An additional expression given on the command line is ignored.')
    group.add_option('--ebpf',dest='ebpf',help='BPF filter to exclude traffic, extends other filters')
    group.add_option('--no-vlan',dest='novlan',action="store_true",help='do not examine traffic which has VLAN headers present')
    group.add_option('--layer2',dest='layer2',default='ethernet.Ethernet',help='select the layer-2 protocol module')
    group.add_option('--strip',dest='striplayers',default=0,help='extra data-link layers to strip')
    parser_short.add_option_group(group)


    group = optparse.OptionGroup(parser,"Output options")
    group.add_option('-o','--outfile',dest='outfile',help='write output to the file OUTFILE. Additional output can be set with KEYWORD=VALUE,...\n'+\
        '\tmode=<w: write (default), a: append, noclobber: do not overwrite, use a  a OUTFILE.1 (.2,.3) file if file(s) exists\n'+\
        '\tpcap=PCAPFILE to write packets to a PCAP file\n'+\
        '\tsession=SESSION to write session text\n'+\
        '\tdirection=data direction to write (c,s,both,split)')
    group.add_option('-w','--session',dest='session',help='write session file, same as -o session=')
    group.add_option('-W','--pcap',dest='pcap',default=None,help='output decoded packets to PCAP (same as -o pcap=....)')
    group.add_option('--db',dest='db',default=None,help='output to db. Supply "config=file" or "param=...,param=..." ')
    group.add_option('--oformat',dest='oformat',help='define the output format')
    group.add_option('-x','--extra',dest='oextra',action='store_true',help='output a lot of extra information')
    group.add_option('-O','--output',dest='output',default=None,help='Use a custom output module. Supply "modulename,option=value,..."')
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser,"Logging options")
    group.add_option('-L','--logfile',dest="logfile",help="log to file")
    group.add_option('--debug',action="store_true", dest="debug",help="debug logging (debug may also affect decoding behavior)")
    group.add_option('-v','--verbose',action="store_true", dest="verbose",help="verbose logging")
    group.add_option('-q','--quiet',action="store_true", dest="quiet",help="practically zero logging")
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser_short,"Logging options")
    group.add_option('-L','--logfile',dest="logfile",help="log to file")
    group.add_option('--debug',action="store_true", dest="debug",help="debug logging (debug may also affect decoding behavior)")
    group.add_option('-v','--verbose',action="store_true", dest="verbose",help="verbose logging")
    group.add_option('-q','--quiet',action="store_true", dest="quiet",help="practically zero logging")
    parser_short.add_option_group(group)


    decoder_options={} #[decoder][option]=value dict of decoder options, set by config file
    decoder_args = []
    args = []
    extra_args = False
    for x in largs:
        if x == '--':
            extra_args = True
            continue
        if extra_args:
            decoder_args.append(x)
        else:
            args.append(x)

    #parse basic options and crdate the options object
    options = parser.parse_args(args,**kwargs)[0]

    if options == None:
        print "\nError processing provided arguments"
        return

    #dump list
    if options.list:
        printDecoders(decoder_map,not options.debug)
        return

    #parse config file, updating the options and decoder_options dicts
    if options.config:
        try:
            import ConfigParser
            config = ConfigParser.ConfigParser()
            config.read(options.config)
            for s in config.sections():
                if s.lower()=='dshell': #this is the main section, set the options
                    for k,v in config.items(s,raw=True):
                        if k in options.__dict__: options.__dict__[k]=v
        except: raise

    #are we a thread outputting to a queue?
    if 'queue' in options.__dict__: out=output.QueueOutput(options.queue)
    #if not, parse output args
    else:
        outfile=None
        outkw={}

        #set output file (and other args if -o filename,key=val...)
        if options.outfile: outfile,outkw=util.strtok(options.outfile)
        #output extra?
        if options.oextra: outkw.update(extra=True)
        #set session writer?
        if options.session: outkw.update(session=options.session)
        #add default pcap writer?
        if options.pcap: outkw.update(pcap=options.pcap)
        #use database?
        if options.db:
            a,kw=util.strtok(options.db,as_list=True)
            #add output options
            kw.update(outkw)
            out=output.DBOutput(*a,**kw)
        #if not db mode and no out module specd
        # use default output lib to get default module
        elif not options.output: options.output='output'
        #init output module
        if options.output:
            a,kw=util.strtok(options.output,as_list=True) #parse output arglist (-O module,args..,k=val...)
            kw.update(outkw) #set output options
            if outfile: kw.update(file=outfile) #set filename arg if -o given (can also be first arg in module arglist)
            outmod = import_module(name=os.path.basename(a[0])) #load module
            if outmod: out=outmod.obj(*a[1:],**kw) #pass remaining args and keyword args to init object

        #set the output format
        if options.oformat != None: out.setformat(options.oformat)

    #set global log functions
    out.logger=logging.getLogger('dshell')
    log=out.log

    #start up the logger
    if options.debug:       level=logging.DEBUG
    elif options.verbose:   level=logging.INFO
    elif options.quiet:     level=logging.FATAL
    else:                   level=logging.WARNING
    logging.basicConfig(filename=options.logfile,level=level)

    decoders = {}
    decoderNames = set()
    # check for a decoder
    if options.decoder != None:
        if type(options.decoder)==str: #single decoder or came from config file
            options.decoder=util.strtok(options.decoder,as_list=True)[0] #make it a list
        # we have a list of decoders
        for dnames in options.decoder:
            chain=dnames.split('+')
            #last module does not have a subdecoder
            module=chain.pop()
            try: module,n=module.split(':',1)
            except: n=None
            m=import_module(module,search=decoder_map)
            if m:
                dObj=copy.copy(m.dObj) #create copy in case we import multiple times under different names
                if n: dObj.name=n
            else: dObj=None
            try: decoderNames.add(dObj.name)
            except AttributeError: decoderNames.add(module)
            #walk up the chain, setting sub-decoders
            while chain:
                subObj=dObj
                module=chain.pop()
                try: module,n=module.split(':',1)
                except: n=None
                m=import_module(module,search=decoder_map)
                if m:
                    dObj=copy.copy(m.dObj)
                    if n: dObj.name=n
                else: dObj=None
                try: decoderNames.add(dObj.name)
                except AttributeError: decoderNames.add(module)
                if dObj and dObj.chainable: dObj.subDecoder=subObj
                elif dObj: sys.stderr.write( "Error %s is not chainable\n"%module)
            #insert the top decoder in the dict
            if dObj: decoders[dObj.name]=dObj

    #save option dict
    options_dict=options.__dict__.copy()

    #add in options for loaded decoders and subdecoders
    for d in decoders.itervalues(): parser.add_decoder_options(d)
    for d in decoders.itervalues(): parser_short.add_decoder_options(d)

    #reparse args to handle decoder options
    optionerror=False
    try: options,args=parser.parse_args(args,**kwargs)
    except: optionerror=True

    #replace base options
    options.__dict__.update(options_dict)

    #look for name_option keys and put them in decoder_options[name][option]
    for k,v in options.__dict__.iteritems():
        for decName in decoderNames:
            try:
                n=k.split(decName+'_', 1)[1]
                decoder_options.setdefault(decName,{}).setdefault(n,v)
            except IndexError: continue

    #reparse config file to handle decoder options
    if options.config:
        for s in config.sections():
            if s.lower() in decoder_options: #set the options for loaded decoders if they are present in the config file
                for k,v in config.items(s,raw=True):
                    if k in decoder_options[s]: #if this is a valid option
                        if v.isdigit(): v=int(v) #try conversion to int/float
                        elif '.' in v:
                            try: v=float(v)
                            except: pass
                        decoder_options[s][k]=v

    if any(x in ('-h', '-?', '--help') for x in sys.argv[1:]):
            # Print the verbose help message
        parser.print_help()
        printDecoderBriefs(decoders)
        return

    if optionerror or (not args and not options.interface):
        # Print the short help message
        parser_short.print_help()
        printDecoderBriefs(decoders)
        return


        #####################################################################################
        # listen live on the interface
        # this will not process any files
    if options.interface != None:
        if len(decoders)!=1:
            print 'Can only run one module live on an interface'
            return

        # handles options and arguments for dumping live on an interface
        decode_live(out,options,dObj, decoder_args,decoder_options)

        #close output
        out.close()

        return
        #####################################################################################

    # take all other command line arguments as files to process


    ####################################################
    # Works if directory (e.g. ~/data/) or * (e.g. ~/data/*
    # used on command line.  Does not work for partial
    # wildcards (e.g. ~/data/*.dat) because bash
    # expands wildcards before passing arguments into
    # decode-pcap.py.  Will work if no matches in root of
    # path specified.
    ####################################################
    inputs = []
    for file_path in args:
        # If this argument is a directory and RECURSIVE specified, then add
        # entire directory tree to the list of input files
        if os.path.isdir(file_path) and options.recursive:
            addFilesFromDirectory(inputs,file_path)

        # If a wildcard is specified, then handle accordingly
        elif file_path.find('*') > -1:
            (path,wildcard) = os.path.split(file_path)

            # If just file is specified (no path)
            if len(path) == 0:
                inputs.extend(glob.glob(wildcard))

            # If there is a path, but recursion not specified,
            # then just add matching files from specified dir
            elif not len(path) == 0 and not options.recursive:
                inputs.extend(glob.glob(file_path))

            # Otherwise, recursion specified and there is a directory.
            # Recurse directory and add files
            else:
                addFilesFromDirectory(inputs,path,wildcard)

        # Just a normal file, append to list of inputs
        else:
            inputs.append(file_path)

    if options.parallel or options.threaded:
        import multiprocessing
        procs=[]
        q=multiprocessing.Queue()
        kwargs=options.__dict__.copy() #put parsed base options in kwargs
        kwargs.update(config=None,outfile=None,queue=q) #pass the q,
        #do not pass the config file or outfile because we handled that here
        for d in decoder_options: #put pre-parsed decoder options in kwargs
            for k,v in decoder_options[d].items(): kwargs[d+'_'+k]=v

    #check here to see if we are running in parallel-file mode
    if options.parallel and len(inputs)>1:
        for f in inputs:
            #create a child process for each input file
            procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=[f]))
        runChildProcs(procs,q,out,numprocs=options.numprocs)

    #check here to see if we are running decoders multithreaded
    elif options.threaded and len(options.decoder)>1:
        for d in options.decoder:
            #create a child for each decoder
            kwargs.update(decoder=d)
            procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=inputs))
        runChildProcs(procs,q,out,numprocs=options.numprocs)

    #fall through to here (single threaded or child process)
    else:
        #
        # Here is where we use the decoder(s) to process the pcap
        #

        temporaryFiles = []    # used when uncompressing files

        for module in decoders.keys():
            decoder = decoders[module]
            initDecoderOptions(decoder,out,options,decoder_args,decoder_options)

            # If the decoder has a preModule function, will execute it now
            decoder.preModule()


            for input_file in inputs:
            # Decoder-specific options may be seen as input files
            # Skip anything starts with "--"
                if input_file[:2] == '--':
                    continue

                # Recursive directory processing is handled elsewhere,
                # so we should only be dealing with files at this point.
                if os.path.isdir(input_file): continue

                log('+Processing file %s' % input_file)

                # assume the input_file is not compressed
                # Allows the processing of .pcap files that are compressed with
                # gzip, bzip2, or zip. Writes uncompressed file to a
                # NamedTemporaryFile and unlinks the file once it is no longer
                # needed. Might consider using mkstemp() since this implementation
                # requires Python >= 2.6.
                try:
                    exts = ['.gz','.bz2','.zip']
                    if os.path.splitext(input_file)[1] not in exts:
                        pcapfile=input_file

                    else:
                        # we have a compressed file
                        tmpfile = expandCompressedFile(input_file,options.verbose,options.tmpdir)
                        temporaryFiles.append(tmpfile)
                        pcapfile=tmpfile
                except:
                    if options.verbose: sys.stderr.write('+Error processing file %s' % (input_file))
                    continue

                # give the decoder access to the input filename
                # motivation: run a decoder against a large number of pcap
                #             files and have the decoder print the filename
                #             so you can go straight to the pcap file for
                #             further analysis
                decoder.input_file = input_file

                # Check to see if the decoder has a preFile function
                # This will be called before the decoder processes each
                # input file
                decoder.preFile()

                try:
                    if not pcap: raise NotImplementedError("pcap support not implemented")
                    decoder.capture=pcap.pcap(pcapfile)
                    if decoder.filter: decoder.capture.setfilter(decoder.filter)
                    while not options.count or decoder.count<options.count:
                        try: ts,pkt=decoder.capture.next() #read next packet and break on EOF
                        except: break #no data
                        decoder.decode(ts,pkt)
                except KeyboardInterrupt: raise
                except: traceback.print_exc()

                if options.verbose:
                    log('+Done processing %s' % (input_file))

                # call that decoder's processFile()
                decoder.postFile()

            # check to see if the decoder is using the Messages class
            # if so, we need to clean up the connection store to
            # purge any unfinished connections
            if 'cleanConnectionStore' in dir(decoder): decoder.cleanConnectionStore()

            # Check to see if the decoder has a postModule function
            # A postModule function will be called when the module
            # has finished running against all of the input files
            if 'postModule' in dir(decoder): decoder.postModule()

        # remove any temporary files that were created during execution
        for tmpfile in temporaryFiles:
            if options.verbose: log('+Unlinking %s'%(tmpfile))
            os.unlink(tmpfile)

    #close output
    out.close()
    return

def runChildProcs(procs,q,out,numprocs=4):
    import Queue
    running=[]
    while procs or running: #while we still have processes to spawn or running
        if procs and len(running)<numprocs:
            proc=procs.pop(0)
            proc.start()
            out.log('started %d'%proc.pid,level=logging.INFO)
            running.append(proc)
        for proc in running:
            if not proc.is_alive(): #see if it finished
                out.log('%d exited (%d)'%(proc.pid,proc.exitcode),level=logging.INFO)
                running.remove(proc)
        try: #get from the output queue until empty
            while True:
                m,args,kw=q.get(True,1) #method, listargs, kwargs
                out.dispatch(m,*args,**kw) #dispatch to method
        except Queue.Empty: pass #q empty


if __name__ == '__main__':
    try: main(*sys.argv[1:])
    except KeyboardInterrupt: sys.exit(0)
#!/usr/bin/env python #we will first check search[name] for the module #else split foo.bar:baz to get from foo.bar import baz #else split dotted path to perform a 'from foo import bar' operation #a -> from search[a] import a #a.b.c from a.b import c #was a full path to a decoder given? #print module,name set the base decoder path,         add it to sys.path for importing,         and walk it to return all subpaths #append base path first # walk decoder directories an add to sys.path #skip hidden dirs like .svn #return the paths we found find all decoders and map decoder to import.path.decoder     expect common prefix to start with basepath #keep last part as base # split path and trim off part before base #skip package stubs #build topdir.path...module name from topdir/dir.../file Print list of decoders with additional info # get the type of decoder it is Read in a BPF filter provided by a command line argument # keep \n for visual output sanity #set decoder options # give the interface name to the decoder #use dispatch so we can handle signals Expand a file compressed with gzip, bzip2, or zip.     Only handles zip files with 1 file. Need to add handling     for zip files containing multiple pcap files. #print fname # This is a support function for the code in main() that supports # recursive directory crawling when looking for pcap files to parse # This function recurses through a directory structure, applying # the wildcard (if any) from the command line.  If no wildcard # is specified, then all files are included. # STEP 1: Add files matching wildcard from current directory... # concatenate into path #STEP 2: Recurse into child directories # The default OptionParser will raise an error when it encounters an option that # it doesn't understand.  By creating a new option parser, dshellOptionParser, # we can ignore the unknown options (read: Module specific options) #create options for all loaded decoders #if we have a subdecoder, recurse down until we don't #pass thru to parse_args, but add in kwargs # probably missing a value for an argument, e.g. 'decode -d' without a decoder # Fix for handling unknown options (e.g. decoder-specific options) # reference: http://stackoverflow.com/questions/1885161/how-can-i-get-optparses-optionparser-to-ignore-invalid-arguments Prints a brief overview of a decoder when using --help with a decoder pass global config to decoder #recurse from the bottom of the chain to the top # give the decoder the output object if the decoder doesn't pick one # or if an output object is specified via command line options #initialize the decoder's custom output using the channels from the global #provide global output module under alternate name #        decoder.out.__init__(fh=out.fh) #re-init the decoder # If the decoder's default output doesn't have a filehandle set, use the user provided one # A filehandle doesn't always exist, such as with QueueOutput #set the logger #set output format string, or reset to default #do not override --oformat specified string #set verbosity #debug() is already taken, and _DEBUG might already be set #override decoder BPF #override decoder filterfn #read BPF from file # extend bpf filter if necessary # do we change the layer-2 decoder for raw capture #strip extra layers? #fix for null filter case #pass args and config file to decoder #get map of name to module import path # The main argument parser. It will have every command line option available and should be used when actually parsing # A short argument parser, meant to only hold the shorter list of arguments for when a decoder is called without a pcap file. DO NOT USE for any serious argument parsing. # decode-pcap specific options #[decoder][option]=value dict of decoder options, set by config file #parse basic options and crdate the options object #dump list #parse config file, updating the options and decoder_options dicts #this is the main section, set the options #are we a thread outputting to a queue? #if not, parse output args #set output file (and other args if -o filename,key=val...) #output extra? #set session writer? #add default pcap writer? #use database? #add output options #if not db mode and no out module specd # use default output lib to get default module #init output module #parse output arglist (-O module,args..,k=val...) #set output options #set filename arg if -o given (can also be first arg in module arglist) #load module #pass remaining args and keyword args to init object #set the output format #set global log functions #start up the logger # check for a decoder #single decoder or came from config file #make it a list # we have a list of decoders #last module does not have a subdecoder #create copy in case we import multiple times under different names #walk up the chain, setting sub-decoders #insert the top decoder in the dict #save option dict #add in options for loaded decoders and subdecoders #reparse args to handle decoder options #replace base options #look for name_option keys and put them in decoder_options[name][option] #reparse config file to handle decoder options #set the options for loaded decoders if they are present in the config file #if this is a valid option #try conversion to int/float # Print the verbose help message # Print the short help message ##################################################################################### # listen live on the interface # this will not process any files # handles options and arguments for dumping live on an interface #close output ##################################################################################### # take all other command line arguments as files to process #################################################### # Works if directory (e.g. ~/data/) or * (e.g. ~/data/* # used on command line.  Does not work for partial # wildcards (e.g. ~/data/*.dat) because bash # expands wildcards before passing arguments into # decode-pcap.py.  Will work if no matches in root of # path specified. #################################################### # If this argument is a directory and RECURSIVE specified, then add # entire directory tree to the list of input files # If a wildcard is specified, then handle accordingly # If just file is specified (no path) # If there is a path, but recursion not specified, # then just add matching files from specified dir # Otherwise, recursion specified and there is a directory. # Recurse directory and add files # Just a normal file, append to list of inputs #put parsed base options in kwargs #pass the q, #do not pass the config file or outfile because we handled that here #put pre-parsed decoder options in kwargs #check here to see if we are running in parallel-file mode #create a child process for each input file #check here to see if we are running decoders multithreaded #create a child for each decoder #fall through to here (single threaded or child process) # # Here is where we use the decoder(s) to process the pcap # # used when uncompressing files # If the decoder has a preModule function, will execute it now # Decoder-specific options may be seen as input files # Skip anything starts with "--" # Recursive directory processing is handled elsewhere, # so we should only be dealing with files at this point. # assume the input_file is not compressed # Allows the processing of .pcap files that are compressed with # gzip, bzip2, or zip. Writes uncompressed file to a # NamedTemporaryFile and unlinks the file once it is no longer # needed. Might consider using mkstemp() since this implementation # requires Python >= 2.6. # we have a compressed file # give the decoder access to the input filename # motivation: run a decoder against a large number of pcap #             files and have the decoder print the filename #             so you can go straight to the pcap file for #             further analysis # Check to see if the decoder has a preFile function # This will be called before the decoder processes each # input file #read next packet and break on EOF #no data # call that decoder's processFile() # check to see if the decoder is using the Messages class # if so, we need to clean up the connection store to # purge any unfinished connections # Check to see if the decoder has a postModule function # A postModule function will be called when the module # has finished running against all of the input files # remove any temporary files that were created during execution #close output #while we still have processes to spawn or running #see if it finished #get from the output queue until empty #method, listargs, kwargs #dispatch to method #q empty