<reponame>abelard2008/overlog
#!/usr/bin/env python2
# -*- Mode: python -*-
#
# DESCRIPTION: Setup and run n chord nodes.
#
#
import getopt
import os
import sys
import re

def print_usage():
    print
    print "Usage: process_output [-e -n <num_nodes>] [-w <bw_window>] [-l <simple_lookup_output>] output_dir"
    print

def parse_cmdline(argv): 
    global emulab
    emulab = False

    shortopts = "l:n:w:e"
    flags = {"num_nodes" : 0, "bw_window" : 20.0, "simple_lookup" : None}
    opts, args = getopt.getopt(argv[1:], shortopts)
    for o, v in opts:
        if   o == "-n": flags["num_nodes"]     = int(v)
        elif o == "-w": flags["bw_window"]     = int(v)
        elif o == "-l": flags["simple_lookup"] = v
        elif o == "-e": emulab = True
        else:
            print_usage()
            exit(3)
    return flags, args

def ts2sec(sec, ns):
    return (float(sec) + (float(ns) / 1000000000.0))

def process_node(out_f, lookups, results):
    matchlookup = re.compile(r"""^Print\[(.*)\:\[.*\],\s*
                                 ([0-9]+),\s*                                  # seconds
                                 ([0-9]+)\]\:\s*                               # nanoseconds
                                 \[\<lookup,\s*                                # token
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address
                                 ([a-f0-9]+),\s*                               # Lookup key 
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address
                                 ([a-zA-Z0-9\.\:]+)\s*                         # Event ID
                             \>\]$\n""", re.VERBOSE)

    matchresults = re.compile(r"""^Print\[.*\],\s*
                                 ([0-9]+),\s*                                  # seconds
                                 ([0-9]+)\]\:\s*                               # nanoseconds
                                 \[\<lookupResults,\s*                         # token
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address
                                 ([a-f0-9]+),\s*                               # key 
                                 ([a-f0-9]+),\s*                               # key 
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address
                                 ([a-zA-Z0-9\.\:]+)\s*                         # Event ID
                             \>\]$\n""", re.VERBOSE)

    matchbandwidth = re.compile(r"""^Print\[Out\:\[.*\],\s*
                                    ([0-9]+),\s*                               # seconds
                                    ([0-9]+)\]\:\s*                            # nanoseconds
                                    \[\<([a-zA-Z]+),\s*                        # token
                                    .*\>\]$\n""", re.VERBOSE)

    bw_measures = []
    start_t     = 0.0
    bw_window   = float(flags["bw_window"])
    window_bytes = 0.0
    min_bw = sys.maxint
    max_bw = 0.0
    cnt_bw = 0.0
    sum_bw = 0.0
    
    for line in out_f: 
        lookup  = []
        result  = []
        measure = []
        if matchlookup.match(line):
            lookup = [x for x in matchlookup.split(line) if x]
            ekey   = lookup[-1]
            lookup_list = [lookup]    
            if lookups.has_key(ekey): lookup_list += lookups[ekey]
            lookups[ekey] = lookup_list 
        if matchresults.match(line):
            result = [x for x in matchresults.split(line) if x]
            ekey   = result[-1]
            results[ekey] = result[:-1]                                # Hash key:=eventID
            if result[2] != result[5] and not resultsHash.has_key(result[5]):
                resultsHash[result[5]] = True
        if matchbandwidth.match(line):
            measure = [x for x in matchbandwidth.split(line) if x]
            if lookup and simple_lookup.has_key(lookup[-1]): continue
            if result and simple_lookup.has_key(result[-1]): continue

            ts = ts2sec(measure[0], measure[1])
            if not start_t: start_t = ts
            if not bytes.has_key(measure[2]):
                print "UNKNOWN MESSAGE: ", measure[2]
                sys.exit(3)
            if (ts - start_t) > bw_window:
                # FINALIZE WINDOW HERE
                # postpone this line to the next window, and finalize
                cnt_bw += 1
                bw = window_bytes/bw_window
                if max_bw < bw: max_bw = bw
                if min_bw > bw: min_bw = bw
                sum_bw += bw
                
                # set up next window
                mt_wins = -1 # don't treat the first tumble as an empty window!
                while ((ts - start_t) > bw_window):
                    start_t += bw_window # tumble the window
                    mt_wins += 1
                window_bytes = bytes[measure[2]]
                if (mt_wins > 0):
                    min_bw = 0
                    cnt_bw += mt_wins
            else: # MIDDLE OF WINDOW
                window_bytes += bytes[measure[2]]
    theNewAverage = 0
    if cnt_bw == 0: theNewAverage = 0
    else : theNewAverage = sum_bw/cnt_bw

    return min_bw, theNewAverage, max_bw

#               ts = ts2sec(measure[0], measure[1])
#             if not start_t: start_t = ts
#             if (ts - start_t) > bw_window:
#                 if not bytes.has_key(measure[2]):
#                     print "UNKNOWN MESSAGE: ", measure[2]
#                     sys.exit(3)
#                 bw_measures.append(bytes[measure[2]]/(ts - start_t))
#                 start_t = 0.0

#     if not bw_measures: return 0.0, 0.0, 0.0
#     norm   = 1.0 / float(len(bw_measures))
#     avg_bw = 0.0
#     max_bw = 0.0
#     min_bw = sys.maxint
#     for bw in bw_measures:
#         if max_bw < bw: max_bw = bw
#         if min_bw > bw: min_bw = bw
#         avg_bw += (bw * norm)
#    return min_bw, avg_bw, max_bw


def eval_lookups(lookups, results):
    unsuccessful = 0
    i_hop_counts = {}
    e_hop_counts = {}
    htimes   = open('./hop_time.dat', 'w')
    ilookups = open('./hopCount_internal.dat', 'w')
    elookups = open('./hopCount_external.dat', 'w')
    tlookups = open('./hopCount_total.dat', 'w')
    llatency = open('./latency.dat', 'w')
    latency     = []
    hop_time    = []

    num_lookups = 0.0
    avg_hc      = 0.0
    for event in lookups.keys(): 
       if not results.has_key(event): 
           unsuccessful += 1
           continue                                          # Only care about successful lookups
       lookup = lookups[event]
       lookup.sort()                                         # sort([type, sec, ns, ...])
       ins   = []
       outs  = []
       wraps = []
       start_sec = start_ns = -1.0 
       for t in lookup:                                      # For each lookup tuple t
           if t[0]   == "In":   ins   += [t] 
           elif t[0] == "Out":  outs  += [t]
           elif t[0] == "Wrap": wraps += [t]
           else: print "UNKNOWN TUPLE TYPE: ", t[0]
           sec = float(t[1])
           ns  = float(t[2])
           if start_sec < 0.0 or sec < start_sec or (sec == start_sec and ns < start_ns): 
               start_sec = sec 
               start_ns  = ns      

       hops = 0
       if outs: hops = len(outs) - 1    # THE hop count
       
       hop_time.append([start_sec, start_ns, hops])

       r_sec = results[event][0]
       r_ns  = results[event][1]
       if hops: latency.append([(ts2sec(r_sec, r_ns) - ts2sec(start_sec, start_ns)), hops])

       looks = 1.0                                                           # Number of dudes with above hop count.
       if wraps:
           if i_hop_counts.has_key(hops): looks += i_hop_counts[hops]
           i_hop_counts[hops] = looks
       else: 
           if e_hop_counts.has_key(hops): looks += e_hop_counts[hops]
           e_hop_counts[hops] = looks


    if latency:
        latency.sort()
        for lat in latency: print >> llatency, "%f %d" % (float(lat[0]), int(lat[1]))
    llatency.close()

    hop_time.sort()
    for x in hop_time: 
        print >> htimes, "%s %s %s" % (x[0], x[1], x[2]) 
    htimes.close()

    thops = []

    # INTERNAL LOOKUP COUNT
    itotal = 0
    for looks in i_hop_counts.values(): itotal += float(looks)        # Add up total internal lookups
    ihops = i_hop_counts.keys()
    ihops.sort()
    for h in ihops:        
        if not h in thops: thops += [h]                             # Save for later compute of total hops
        print >> ilookups, "%d %f %d" % (h, float(i_hop_counts[h]), itotal)
    ilookups.close()

    # EXTERNAL LOOKUP COUNT
    etotal = 0
    for looks in e_hop_counts.values(): etotal += float(looks)        # Add up total external lookups
    ehops = e_hop_counts.keys()
    ehops.sort()
    for h in ehops:        
        if not h in thops: thops += [h]                             # Save for later compute of total hops
        print >> elookups, "%d %f %d" % (h, float(e_hop_counts[h]), etotal)
    elookups.close()

    # TOTAL LOOKUP COUNT
    thops.sort()
    print "TOTAL HOP TYPES: ", thops
    for h in thops:        
        ih = 0
        eh = 0
        if i_hop_counts.has_key(h): ih += float(i_hop_counts[h])
        if e_hop_counts.has_key(h): eh += float(e_hop_counts[h])
        print >> tlookups, "%d %f %d" % (h, (ih + eh), (itotal + etotal))
    tlookups.close()

def get_simple_lookup():
    slookups = open(flags["simple_lookup"], 'r')
    matchlookup = re.compile(r"""^Print\[lookup]\:\s*
                                 \[\<lookup,\s*                                # token
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address
                                 ([a-f0-9]+),\s*                               # Lookup key 
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address
                                 ([a-zA-Z0-9\.\:]+)\s*                         # Event ID
                             \>\]$\n""", re.VERBOSE)

    simple_lookup = {}
    for line in slookups: 
        if matchlookup.match(line):
            lookup = [x for x in matchlookup.split(line) if x]
            key    = lookup[1]
            events = [lookup[-1]]                         # Only need the eventid
            if simple_lookup.has_key(key): events += simple_lookup[key]
            simple_lookup[key] = events 
    return simple_lookup

def eval_simple_lookup(lookups, results):
    out = open("./simple_lookup.dat", 'w')
    for key in simple_lookup.keys():
       responseIPs = []
       for event in simple_lookup[key]:
           if results.has_key(event):
               response = results[event]
               responseIPs += [response[-1]]
       responseIPs.sort()
       maxsofar = -1
       curip    = "" 
       cnt      = 0 
       for ip in responseIPs: 
           if ip == curip:
               cnt += 1
           else:
               if maxsofar < cnt: maxsofar = cnt
               cnt = 1
               curip = ip 
       if len(responseIPs) == 0:
           print >> out, "LOOKUP(%s): %f " % (key, 0), responseIPs
       else:
           print >> out, "LOOKUP(%s): %f " % (key, (float(maxsofar)/float(len(responseIPs)))), responseIPs

if __name__ == "__main__":
    global simple_lookup
    global ip_map
    global bytes
    global resultsHash

    resultsHash = {}
    bytes = {"startJoin" : 136, "lookup" : 132, "lookupResults" : 164, "stabilizeRequest" : 136,
             "notifyPredecessor" : 108, "sendPredecessor" : 140, "returnSuccessor" : 140,
             "sendSuccessors" : 140}

    try:
        flags, args = parse_cmdline(sys.argv)
    except:
        print "EXCEPTION"
        print_usage()
        sys.exit(3)
    if len(args) < 1:
        print_usage()        
        sys.exit(3)

    files = filter(lambda files: files[-4:] == '.log', os.listdir(args[0]) )

#    for files in os.walk(args[0]):
#        if emulab:
#	    for host in dirs:
#                for n in range(flags["num_nodes"]): 
#                    files += [os.path.join(root, host, "chord_node%d.out" % (n))]
#                    if int(host) == 0: break
#        else: 
#            for f in datfiles: 
#                files += [os.path.join(root, f)]

    simple_lookup = {}
    if flags["simple_lookup"]: simple_lookup = get_simple_lookup()

    node_bw = open('./node_bw.dat', 'w')
    nodeid  = 0
    lookups = {}
    results = {}

    avg_avg_bw = 0.0
    max_max_bw = 0.0
    min_min_bw = sys.maxint
    
    avg_counter = 0
    for file in files:
        nodeid = nodeid + 1
        if not os.path.exists(file):
            print "ERROR: file does not exist ", file
            exit(3)
        fh = open(file, "r")

        min_bw, avg_bw, max_bw = process_node(fh, lookups, results)
	if avg_bw != 0:
            print >> node_bw, "%d %f %f %f" % (nodeid, float(min_bw), float(avg_bw), float(max_bw))

            if max_max_bw < max_bw: max_max_bw = max_bw
            if min_min_bw > min_bw: min_min_bw = min_bw
            avg_avg_bw += avg_bw
            avg_counter += 1

        node_bw.flush()
        fh.close()
    if avg_counter != 0:
        avg_avg_bw /= avg_counter
    else: avg_avg_bw = -1
    print >> node_bw, "#%d %f %f %f" % (avg_counter, float(min_min_bw), float(avg_avg_bw), float(max_max_bw))
    node_bw.close()

    print "SANITY CHECK: counted %d distinct result tuples." % (len(resultsHash.keys()))

    if simple_lookup: eval_simple_lookup(lookups, results)
    eval_lookups(lookups, results)
    

#!/usr/bin/env python2
# -*- Mode: python -*-
#
# DESCRIPTION: Setup and run n chord nodes.
#
#
import getopt
import os
import sys
import re

def print_usage():
    print
    print "Usage: process_output [-e -n <num_nodes>] [-w <bw_window>] [-l <simple_lookup_output>] output_dir"
    print

def parse_cmdline(argv): 
    global emulab
    emulab = False

    shortopts = "l:n:w:e"
    flags = {"num_nodes" : 0, "bw_window" : 20.0, "simple_lookup" : None}
    opts, args = getopt.getopt(argv[1:], shortopts)
    for o, v in opts:
        if   o == "-n": flags["num_nodes"]     = int(v)
        elif o == "-w": flags["bw_window"]     = int(v)
        elif o == "-l": flags["simple_lookup"] = v
        elif o == "-e": emulab = True
        else:
            print_usage()
            exit(3)
    return flags, args

def ts2sec(sec, ns):
    return (float(sec) + (float(ns) / 1000000000.0))

def process_node(out_f, lookups, results):
    matchlookup = re.compile(r"""^Print\[(.*)\:\[.*\],\s*
                                 ([0-9]+),\s*                                  # seconds
                                 ([0-9]+)\]\:\s*                               # nanoseconds
                                 \[\<lookup,\s*                                # token
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address
                                 ([a-f0-9]+),\s*                               # Lookup key 
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address
                                 ([a-zA-Z0-9\.\:]+)\s*                         # Event ID
                             \>\]$\n""", re.VERBOSE)

    matchresults = re.compile(r"""^Print\[.*\],\s*
                                 ([0-9]+),\s*                                  # seconds
                                 ([0-9]+)\]\:\s*                               # nanoseconds
                                 \[\<lookupResults,\s*                         # token
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address
                                 ([a-f0-9]+),\s*                               # key 
                                 ([a-f0-9]+),\s*                               # key 
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address
                                 ([a-zA-Z0-9\.\:]+)\s*                         # Event ID
                             \>\]$\n""", re.VERBOSE)

    matchbandwidth = re.compile(r"""^Print\[Out\:\[.*\],\s*
                                    ([0-9]+),\s*                               # seconds
                                    ([0-9]+)\]\:\s*                            # nanoseconds
                                    \[\<([a-zA-Z]+),\s*                        # token
                                    .*\>\]$\n""", re.VERBOSE)

    bw_measures = []
    start_t     = 0.0
    bw_window   = float(flags["bw_window"])
    window_bytes = 0.0
    min_bw = sys.maxint
    max_bw = 0.0
    cnt_bw = 0.0
    sum_bw = 0.0
    
    for line in out_f: 
        lookup  = []
        result  = []
        measure = []
        if matchlookup.match(line):
            lookup = [x for x in matchlookup.split(line) if x]
            ekey   = lookup[-1]
            lookup_list = [lookup]    
            if lookups.has_key(ekey): lookup_list += lookups[ekey]
            lookups[ekey] = lookup_list 
        if matchresults.match(line):
            result = [x for x in matchresults.split(line) if x]
            ekey   = result[-1]
            results[ekey] = result[:-1]                                # Hash key:=eventID
            if result[2] != result[5] and not resultsHash.has_key(result[5]):
                resultsHash[result[5]] = True
        if matchbandwidth.match(line):
            measure = [x for x in matchbandwidth.split(line) if x]
            if lookup and simple_lookup.has_key(lookup[-1]): continue
            if result and simple_lookup.has_key(result[-1]): continue

            ts = ts2sec(measure[0], measure[1])
            if not start_t: start_t = ts
            if not bytes.has_key(measure[2]):
                print "UNKNOWN MESSAGE: ", measure[2]
                sys.exit(3)
            if (ts - start_t) > bw_window:
                # FINALIZE WINDOW HERE
                # postpone this line to the next window, and finalize
                cnt_bw += 1
                bw = window_bytes/bw_window
                if max_bw < bw: max_bw = bw
                if min_bw > bw: min_bw = bw
                sum_bw += bw
                
                # set up next window
                mt_wins = -1 # don't treat the first tumble as an empty window!
                while ((ts - start_t) > bw_window):
                    start_t += bw_window # tumble the window
                    mt_wins += 1
                window_bytes = bytes[measure[2]]
                if (mt_wins > 0):
                    min_bw = 0
                    cnt_bw += mt_wins
            else: # MIDDLE OF WINDOW
                window_bytes += bytes[measure[2]]
    theNewAverage = 0
    if cnt_bw == 0: theNewAverage = 0
    else : theNewAverage = sum_bw/cnt_bw

    return min_bw, theNewAverage, max_bw

#               ts = ts2sec(measure[0], measure[1])
#             if not start_t: start_t = ts
#             if (ts - start_t) > bw_window:
#                 if not bytes.has_key(measure[2]):
#                     print "UNKNOWN MESSAGE: ", measure[2]
#                     sys.exit(3)
#                 bw_measures.append(bytes[measure[2]]/(ts - start_t))
#                 start_t = 0.0

#     if not bw_measures: return 0.0, 0.0, 0.0
#     norm   = 1.0 / float(len(bw_measures))
#     avg_bw = 0.0
#     max_bw = 0.0
#     min_bw = sys.maxint
#     for bw in bw_measures:
#         if max_bw < bw: max_bw = bw
#         if min_bw > bw: min_bw = bw
#         avg_bw += (bw * norm)
#    return min_bw, avg_bw, max_bw


def eval_lookups(lookups, results):
    unsuccessful = 0
    i_hop_counts = {}
    e_hop_counts = {}
    htimes   = open('./hop_time.dat', 'w')
    ilookups = open('./hopCount_internal.dat', 'w')
    elookups = open('./hopCount_external.dat', 'w')
    tlookups = open('./hopCount_total.dat', 'w')
    llatency = open('./latency.dat', 'w')
    latency     = []
    hop_time    = []

    num_lookups = 0.0
    avg_hc      = 0.0
    for event in lookups.keys(): 
       if not results.has_key(event): 
           unsuccessful += 1
           continue                                          # Only care about successful lookups
       lookup = lookups[event]
       lookup.sort()                                         # sort([type, sec, ns, ...])
       ins   = []
       outs  = []
       wraps = []
       start_sec = start_ns = -1.0 
       for t in lookup:                                      # For each lookup tuple t
           if t[0]   == "In":   ins   += [t] 
           elif t[0] == "Out":  outs  += [t]
           elif t[0] == "Wrap": wraps += [t]
           else: print "UNKNOWN TUPLE TYPE: ", t[0]
           sec = float(t[1])
           ns  = float(t[2])
           if start_sec < 0.0 or sec < start_sec or (sec == start_sec and ns < start_ns): 
               start_sec = sec 
               start_ns  = ns      

       hops = 0
       if outs: hops = len(outs) - 1    # THE hop count
       
       hop_time.append([start_sec, start_ns, hops])

       r_sec = results[event][0]
       r_ns  = results[event][1]
       if hops: latency.append([(ts2sec(r_sec, r_ns) - ts2sec(start_sec, start_ns)), hops])

       looks = 1.0                                                           # Number of dudes with above hop count.
       if wraps:
           if i_hop_counts.has_key(hops): looks += i_hop_counts[hops]
           i_hop_counts[hops] = looks
       else: 
           if e_hop_counts.has_key(hops): looks += e_hop_counts[hops]
           e_hop_counts[hops] = looks


    if latency:
        latency.sort()
        for lat in latency: print >> llatency, "%f %d" % (float(lat[0]), int(lat[1]))
    llatency.close()

    hop_time.sort()
    for x in hop_time: 
        print >> htimes, "%s %s %s" % (x[0], x[1], x[2]) 
    htimes.close()

    thops = []

    # INTERNAL LOOKUP COUNT
    itotal = 0
    for looks in i_hop_counts.values(): itotal += float(looks)        # Add up total internal lookups
    ihops = i_hop_counts.keys()
    ihops.sort()
    for h in ihops:        
        if not h in thops: thops += [h]                             # Save for later compute of total hops
        print >> ilookups, "%d %f %d" % (h, float(i_hop_counts[h]), itotal)
    ilookups.close()

    # EXTERNAL LOOKUP COUNT
    etotal = 0
    for looks in e_hop_counts.values(): etotal += float(looks)        # Add up total external lookups
    ehops = e_hop_counts.keys()
    ehops.sort()
    for h in ehops:        
        if not h in thops: thops += [h]                             # Save for later compute of total hops
        print >> elookups, "%d %f %d" % (h, float(e_hop_counts[h]), etotal)
    elookups.close()

    # TOTAL LOOKUP COUNT
    thops.sort()
    print "TOTAL HOP TYPES: ", thops
    for h in thops:        
        ih = 0
        eh = 0
        if i_hop_counts.has_key(h): ih += float(i_hop_counts[h])
        if e_hop_counts.has_key(h): eh += float(e_hop_counts[h])
        print >> tlookups, "%d %f %d" % (h, (ih + eh), (itotal + etotal))
    tlookups.close()

def get_simple_lookup():
    slookups = open(flags["simple_lookup"], 'r')
    matchlookup = re.compile(r"""^Print\[lookup]\:\s*
                                 \[\<lookup,\s*                                # token
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address
                                 ([a-f0-9]+),\s*                               # Lookup key 
                                 ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address
                                 ([a-zA-Z0-9\.\:]+)\s*                         # Event ID
                             \>\]$\n""", re.VERBOSE)

    simple_lookup = {}
    for line in slookups: 
        if matchlookup.match(line):
            lookup = [x for x in matchlookup.split(line) if x]
            key    = lookup[1]
            events = [lookup[-1]]                         # Only need the eventid
            if simple_lookup.has_key(key): events += simple_lookup[key]
            simple_lookup[key] = events 
    return simple_lookup

def eval_simple_lookup(lookups, results):
    out = open("./simple_lookup.dat", 'w')
    for key in simple_lookup.keys():
       responseIPs = []
       for event in simple_lookup[key]:
           if results.has_key(event):
               response = results[event]
               responseIPs += [response[-1]]
       responseIPs.sort()
       maxsofar = -1
       curip    = "" 
       cnt      = 0 
       for ip in responseIPs: 
           if ip == curip:
               cnt += 1
           else:
               if maxsofar < cnt: maxsofar = cnt
               cnt = 1
               curip = ip 
       if len(responseIPs) == 0:
           print >> out, "LOOKUP(%s): %f " % (key, 0), responseIPs
       else:
           print >> out, "LOOKUP(%s): %f " % (key, (float(maxsofar)/float(len(responseIPs)))), responseIPs

if __name__ == "__main__":
    global simple_lookup
    global ip_map
    global bytes
    global resultsHash

    resultsHash = {}
    bytes = {"startJoin" : 136, "lookup" : 132, "lookupResults" : 164, "stabilizeRequest" : 136,
             "notifyPredecessor" : 108, "sendPredecessor" : 140, "returnSuccessor" : 140,
             "sendSuccessors" : 140}

    try:
        flags, args = parse_cmdline(sys.argv)
    except:
        print "EXCEPTION"
        print_usage()
        sys.exit(3)
    if len(args) < 1:
        print_usage()        
        sys.exit(3)

    files = filter(lambda files: files[-4:] == '.log', os.listdir(args[0]) )

#    for files in os.walk(args[0]):
#        if emulab:
#	    for host in dirs:
#                for n in range(flags["num_nodes"]): 
#                    files += [os.path.join(root, host, "chord_node%d.out" % (n))]
#                    if int(host) == 0: break
#        else: 
#            for f in datfiles: 
#                files += [os.path.join(root, f)]

    simple_lookup = {}
    if flags["simple_lookup"]: simple_lookup = get_simple_lookup()

    node_bw = open('./node_bw.dat', 'w')
    nodeid  = 0
    lookups = {}
    results = {}

    avg_avg_bw = 0.0
    max_max_bw = 0.0
    min_min_bw = sys.maxint
    
    avg_counter = 0
    for file in files:
        nodeid = nodeid + 1
        if not os.path.exists(file):
            print "ERROR: file does not exist ", file
            exit(3)
        fh = open(file, "r")

        min_bw, avg_bw, max_bw = process_node(fh, lookups, results)
	if avg_bw != 0:
            print >> node_bw, "%d %f %f %f" % (nodeid, float(min_bw), float(avg_bw), float(max_bw))

            if max_max_bw < max_bw: max_max_bw = max_bw
            if min_min_bw > min_bw: min_min_bw = min_bw
            avg_avg_bw += avg_bw
            avg_counter += 1

        node_bw.flush()
        fh.close()
    if avg_counter != 0:
        avg_avg_bw /= avg_counter
    else: avg_avg_bw = -1
    print >> node_bw, "#%d %f %f %f" % (avg_counter, float(min_min_bw), float(avg_avg_bw), float(max_max_bw))
    node_bw.close()

    print "SANITY CHECK: counted %d distinct result tuples." % (len(resultsHash.keys()))

    if simple_lookup: eval_simple_lookup(lookups, results)
    eval_lookups(lookups, results)
#!/usr/bin/env python2 # -*- Mode: python -*- # # DESCRIPTION: Setup and run n chord nodes. # # ^Print\[(.*)\:\[.*\],\s*                                  ([0-9]+),\s*                                  # seconds                                  ([0-9]+)\]\:\s*                               # nanoseconds                                  \[\<lookup,\s*                                # token                                  ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address                                  ([a-f0-9]+),\s*                               # Lookup key                                   ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address                                  ([a-zA-Z0-9\.\:]+)\s*                         # Event ID                              \>\]$\n ^Print\[.*\],\s*                                  ([0-9]+),\s*                                  # seconds                                  ([0-9]+)\]\:\s*                               # nanoseconds                                  \[\<lookupResults,\s*                         # token                                  ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address                                  ([a-f0-9]+),\s*                               # key                                   ([a-f0-9]+),\s*                               # key                                   ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address                                  ([a-zA-Z0-9\.\:]+)\s*                         # Event ID                              \>\]$\n ^Print\[Out\:\[.*\],\s*                                     ([0-9]+),\s*                               # seconds                                     ([0-9]+)\]\:\s*                            # nanoseconds                                     \[\<([a-zA-Z]+),\s*                        # token                                     .*\>\]$\n # Hash key:=eventID # FINALIZE WINDOW HERE # postpone this line to the next window, and finalize # set up next window # don't treat the first tumble as an empty window! # tumble the window # MIDDLE OF WINDOW #               ts = ts2sec(measure[0], measure[1]) #             if not start_t: start_t = ts #             if (ts - start_t) > bw_window: #                 if not bytes.has_key(measure[2]): #                     print "UNKNOWN MESSAGE: ", measure[2] #                     sys.exit(3) #                 bw_measures.append(bytes[measure[2]]/(ts - start_t)) #                 start_t = 0.0 #     if not bw_measures: return 0.0, 0.0, 0.0 #     norm   = 1.0 / float(len(bw_measures)) #     avg_bw = 0.0 #     max_bw = 0.0 #     min_bw = sys.maxint #     for bw in bw_measures: #         if max_bw < bw: max_bw = bw #         if min_bw > bw: min_bw = bw #         avg_bw += (bw * norm) #    return min_bw, avg_bw, max_bw # Only care about successful lookups # sort([type, sec, ns, ...]) # For each lookup tuple t # THE hop count # Number of dudes with above hop count. # INTERNAL LOOKUP COUNT # Add up total internal lookups # Save for later compute of total hops # EXTERNAL LOOKUP COUNT # Add up total external lookups # Save for later compute of total hops # TOTAL LOOKUP COUNT ^Print\[lookup]\:\s*                                  \[\<lookup,\s*                                # token                                  ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Source IP address                                  ([a-f0-9]+),\s*                               # Lookup key                                   ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+),\s*  # Destination IP address                                  ([a-zA-Z0-9\.\:]+)\s*                         # Event ID                              \>\]$\n # Only need the eventid #    for files in os.walk(args[0]): #        if emulab: #	    for host in dirs: #                for n in range(flags["num_nodes"]): #                    files += [os.path.join(root, host, "chord_node%d.out" % (n))] #                    if int(host) == 0: break #        else: #            for f in datfiles: #                files += [os.path.join(root, f)]