<gh_stars>10-100
# Copyright 2019-present <NAME>, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

###############################################
# Creates graphs out of csv files from python #
# receiver; can be used included in views.py  #
# OR                                          #
# standalone by passing the --id flag         #
# OR                                          #
# standalone using the config in readme.txt   #
###############################################
import argparse
import csv
import json
import multiprocessing
import numpy as np
import os
import sys
import threading
import time

import matplotlib
matplotlib.use("Agg")
try:
    import matplotlib.pyplot as plt
except Exception as e:
    # prevent PEP3 warning because Agg import must be before pyplot
    raise e


csv.field_size_limit(sys.maxsize)
dir_path = os.path.dirname(os.path.realpath(__file__))
project_path = dir_path[0:dir_path.find("/analytics")]
lock = threading.RLock()


# read the csv files and plots the graphs
def main(file_id, multicast, results_path):
    def thread_join(thrs):
        for thread in thrs:
            thread.start()
        for thread in thrs:
            thread.join()

    fpath = project_path + "/results/" + str(
        file_id) + "/generated/extHost_results.json"
    if os.path.isfile(fpath):
        with open(fpath, "r") as f:
            print("Using cached version of " + fpath)
            return json.load(f)

    start = time.time()
    raw_packet_counter = read_csv(
        results_path, "raw_packet_counter", file_id)[0]
    packet_sizes = read_csv(results_path, "packet_sizes", file_id)
    timestamp1_list = read_csv(results_path, "timestamp1_list", file_id)
    timestamp2_list = read_csv(results_path, "timestamp2_list", file_id)
    end = time.time()
    print("It took " + str(end-start) +
          "s to load csv files for P4STA analytics.")

    # get list of cumulative packet sizes
    throughput_at_time = []
    total_throughput = 0
    for i in range(len(packet_sizes)):
        total_throughput = total_throughput + packet_sizes[i]
        throughput_at_time.append(total_throughput)

    latency_list = []
    min_latency = max_latency = ave_latency = \
        latency_variance = latency_std_deviation = 0
    total_ipdv = abs_total_ipdv = total_pdv = total_latencies = 0
    min_ipdv = max_ipdv = ave_ipdv = ave_abs_ipdv = 0
    min_pdv = max_pdv = ave_pdv = 0
    min_packets = max_packets = ave_packet_sec = 0
    ipdv_list = []
    count_list = []
    count_list_sec = []
    pdv_list = []
    mbit_list = []
    packet_list = []
    time_throughput = []
    if len(timestamp1_list) > 0 and len(timestamp2_list) > 0:
        if timestamp1_list[0] > 0 \
                and len(timestamp1_list) == len(timestamp2_list):
            # creates list of all latencies
            for i in range(0, len(timestamp1_list)):
                delta = timestamp2_list[i] - timestamp1_list[i]
                latency_list.append(delta)
                total_latencies = total_latencies + delta
                # sets the start time to 0 ms
                time_throughput.append(int(round(
                    (timestamp2_list[i] - timestamp2_list[0]) / 1000000)))
                # - [0] to set first time to 0 sec
                count_list_sec.append(
                    (timestamp2_list[i] - timestamp2_list[0]) / 1000000000)
            # minimum and maximum latency
            min_latency = min(latency_list, default=0)
            max_latency = max(latency_list, default=0)
            # fills pdv and ipdv lists
            for z in range(0, len(latency_list)):
                count_list.append(z)
                pdv = latency_list[z] - min_latency
                pdv_list.append(pdv)
                total_pdv = total_pdv + pdv
                if 0 < z < len(latency_list):
                    ipdv = latency_list[z] - latency_list[z - 1]
                    ipdv_list.append(ipdv)
                    total_ipdv = total_ipdv + ipdv
                    abs_total_ipdv = abs_total_ipdv + abs(ipdv)
                else:
                    ipdv_list.append(0)
            # calculate average latency, ipdv and pdv
            if len(latency_list) > 0:
                ave_latency = round(total_latencies / len(latency_list), 2)
                ave_ipdv = round(total_ipdv / len(ipdv_list))
                ave_abs_ipdv = round(abs_total_ipdv / len(ipdv_list))
                ave_pdv = round(total_pdv / len(pdv_list))
                # calculate standard deviation of latency
                total_sqr_dev = 0
                for z in range(0, len(latency_list)):
                    total_sqr_dev += (latency_list[z] - ave_latency)**2
                latency_variance = total_sqr_dev / len(ipdv_list)
                latency_std_deviation = latency_variance ** 0.5
            # minimum and maximum ipdv
            min_ipdv = min(ipdv_list, default=0)
            max_ipdv = max(ipdv_list, default=0)
            # minimum and maximum pdv
            min_pdv = min(pdv_list, default=0)
            max_pdv = max(pdv_list, default=0)
            # fill speed and packet rate lists
            last_time_hit = 0
            last_throughput_hit = 0
            last_packet_hit = 0
            mbit_list.append(0)
            packet_list.append(0)
            # prepares lists for speed and packet rate graph
            for y in range(0, len(throughput_at_time)):
                # more than 99ms difference -> 0.1s intervals
                if (time_throughput[y] - last_time_hit) >= 100:
                    if (time_throughput[y] - last_time_hit) >= 100:
                        amount = (time_throughput[y] - last_time_hit) / 100
                        # more than 200ms difference between two hits -> pause
                        if amount >= 2:
                            for i in range(0, int(round(amount))):
                                mbit_list.append(0)
                                packet_list.append(0)
                    last_time_hit = time_throughput[y]
                    # byte->megabit/10 measure every 0.1s but unit is mbit/s
                    mbit_list.append((throughput_at_time[y] -
                                      last_throughput_hit) * 8 / 100000)
                    # *10 measure for every 0.1s but unit is packets/seconds
                    packet_list.append((y - last_packet_hit)*10)
                    last_packet_hit = y
                    last_throughput_hit = throughput_at_time[y]
            mbit_list.append(0)  # set next entry to 0
            packet_list.append(0)

            # round by 9 digits after , float could result in weird fractions
            upsc_mbit_list = [round(x*int(multicast), 9) for x in mbit_list]

            upsc_packet_list = [x*int(multicast) for x in packet_list]
            # ignores the first and last second to prevent min = 0
            min_packets = min(packet_list[1:-1], default=0)
            max_packets = max(packet_list, default=0)
            if len(packet_list) != 2:
                # -2 because added 0 at start and end. *10 because of 0.1s step
                ave_packet_sec = round(
                    (len(throughput_at_time)/(len(packet_list)-2)) * 10, 2)
            else:
                ave_packet_sec = 0

            start = time.time()
            processes = []
            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list,
                    "Latency of DUT for every " + multicast + ". packet",
                    "Packets", "Latency", "latency", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list_sec,
                    "Latency of DUT for every " + multicast + ". packet",
                    "t[s]", "Latency", "latency_sec", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list,
                    "Latency of DUT for every " + multicast + ". packet",
                    "Packets", "Latency", "latency_y0", True, True, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list_sec,
                    "Latency of DUT for every " + multicast + ". packet",
                    "t[s]", "Latency", "latency_sec_y0", True, True, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    ipdv_list, count_list,
                    "IPDVs of DUT for every " + multicast + ". packet",
                    "IPDV", "Packets", "ipdv", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    ipdv_list, count_list_sec,
                    "IPDVs of DUT for every " + multicast + ". packet",
                    "t[s]", "IPDV", "ipdv_sec", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    pdv_list, count_list,
                    "PDVs of DUT for every " + multicast + ". packet",
                    "Packets", "PDV", "pdv", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    pdv_list, count_list_sec,
                    "PDVs of DUT for every " + multicast + ". packet",
                    "t[s]", "PDV", "pdv_sec", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    mbit_list, np.arange(0, len(mbit_list) / 10, 0.1),
                    "Throughput of DUT for every " + multicast + ". packet",
                    "t[s]", "Megabit/s", "speed", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    upsc_mbit_list,
                    np.arange(0, len(upsc_mbit_list) / 10, 0.1),
                    "Upscaled throughput of DUT", "t[s]", "Megabit/s",
                    "speed_upscaled", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    upsc_packet_list,
                    np.arange(0, len(upsc_packet_list) / 10, 0.1),
                    "Upscaled rate jitter of DUT", "t[s]", "Packet/s",
                    "packet_rate_upscaled", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    packet_list, np.arange(0, len(packet_list) / 10, 0.1),
                    "Rate jitter of DUT for every " + multicast + ". packet",
                    "t[s]", "Packet/s", "packet_rate", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_bar, args=(
                    latency_list, min_latency, max_latency, "latency_bar",
                    "Latency", "Packets", 10, True, file_id))
            processes.append(x)

            thread_join(processes)
            end = time.time()
            print("MULTIP: It took " + str(end - start) +
                  "s to plot graphs in analytics.")

    results = {"num_raw_packets": raw_packet_counter,
               "num_processed_packets": len(latency_list),
               "total_throughput": round(total_throughput/1000000, 2),
               "min_latency": min_latency, "max_latency": max_latency,
               "avg_latency": ave_latency, "min_ipdv": min_ipdv,
               "max_ipdv": max_ipdv, "avg_ipdv": ave_ipdv,
               "avg_abs_ipdv": ave_abs_ipdv, "min_pdv": min_pdv,
               "max_pdv": max_pdv, "avg_pdv": ave_pdv,
               "min_packets_per_second": min_packets,
               "max_packets_per_second": max_packets,
               "avg_packets_per_second": ave_packet_sec,
               "latency_std_deviation": latency_std_deviation,
               "latency_variance": latency_variance,
               "latency_list": latency_list}
    if __name__ == "__main__":
        fpath = "extHost_results.json"
    with open(fpath, 'w', encoding='utf-8') as f:
        json.dump(results, f, indent=4)

    return results


# plots the line charts
def plot_graph(value_list_input, index_list, titel, x_label, y_label,
               filename, adjust_unit, adjust_y_ax, file_id):
    fpath = project_path + "/results/" + str(
                    file_id) + "/generated/" + filename + ".svg"
    if not os.path.isfile(fpath) or __name__ == "__main__":
        with lock:
            if adjust_unit:
                value_list, unit = find_unit(value_list_input)
            else:
                value_list = value_list_input
                unit = ""
            fig, ax = plt.subplots()
            ax.plot(index_list, value_list)
            plt.title(titel)
            if adjust_unit:
                y_label = y_label + " [" + unit + "]"
            # if adjust_y_ax is True sets y-axis to 0
            if adjust_y_ax:
                try:
                    temp = max(value_list_input)
                    if adjust_unit:
                        if unit == "microseconds":
                            temp = temp / 1000
                        elif unit == "milliseconds":
                            temp = temp / 1000000
                    # -0.075 sets 0 point 7.5% under 0
                    # for equal 0 at x and y axis
                    ax.set_ylim([-0.075 * temp, temp + 0.1 * temp])
                except Exception:
                    # no adjustment of y axis if error
                    # (e.g. empty iperf3 results)
                    pass
            plt.xlabel(x_label, fontsize=12)
            plt.ylabel(y_label, fontsize=12)
            plt.tight_layout()
            if __name__ == "__main__":
                fig.savefig(filename + ".svg", format="svg")
            else:
                try:
                    os.mkdir(project_path + "/results/" + str(
                        file_id) + "/generated")
                except OSError:
                    # case where directory already exists
                    pass
                fig.savefig(fpath, format="svg")
            plt.close('all')
    else:
        print("Using cached version of " + fpath)


# input: list
# returns list and string with unit
def find_unit(value_list_input):
    if type(value_list_input) != list:
        value_list_input = [value_list_input]
    try:
        microsec_counter = 0
        millisec_counter = 0
        value_list = []
        for i in value_list_input:
            if abs(i)/1000 >= 1:
                microsec_counter = microsec_counter + 1
            if abs(i)/1000000 >= 1:
                millisec_counter = millisec_counter + 1
        # if more than 95% of the values are bigger than 1 millisec = millisec
        if millisec_counter > (0.95 * len(value_list_input)):
            unit = "milliseconds"
            for i in value_list_input:
                value_list.append(round(i/1000000, 2))
            return value_list, unit
        # if more than 95% of the values are bigger than 1 microsec = microsec
        elif microsec_counter > (0.95 * len(value_list_input)):
            unit = "microseconds"
            for i in value_list_input:
                value_list.append(round(i/1000, 2))
            return value_list, unit
        else:
            unit = "nanoseconds"
            return value_list_input, unit
    except Exception:
        unit = "nanoseconds"
        return value_list_input, unit


# b_type = "bit" or "byte"
def find_unit_bit_byte(value, b_type):
    unit = b_type
    new_value = value
    if (value/1000) >= 1:
        unit = "kilo" + b_type
        new_value = value/1000
    if (value/1000000) >= 1:
        unit = "mega" + b_type
        new_value = value/1000000
    if (value/1000000000) >= 1:
        unit = "giga" + b_type
        new_value = value/1000000000

    return [new_value, unit]  # 1000bit => [1, "kilobit"]


# returns the given input value(ns^2), scaled to a matching unit
def find_unit_sqr(value_ns2):
    try:
        if abs(value_ns2) >= 1500000*1000000:
            unit = "ms²"
            value = round(value_ns2 / (1000000*1000000), 2)
            return value, unit
        if abs(value_ns2) >= 1500000:
            unit = "us²"
            value = round(value_ns2 / 1000000, 2)
            return value, unit
        unit = "ns²"
        return round(value_ns2, 2), unit
    except Exception:
        unit = "ns²"
        return value_ns2, unit


# plots bar chart
def plot_bar(value_list_input, min, max, filename,
             x, y, slices, adjust_unit, file_id):
    fpath = project_path + "/results/" + str(
        file_id) + "/generated/" + filename + ".svg"
    if not os.path.isfile(fpath) or __name__ == "__main__":
        with lock:
            if adjust_unit:
                value_list, unit = find_unit(value_list_input)
                if unit == "microseconds":
                    min = float(min) / 1000
                    max = float(max) / 1000
                elif unit == "milliseconds":
                    min = float(min) / 1000000
                    max = float(max) / 1000000
            else:
                value_list = value_list_input
                unit = ""

            min = min - 0.1
            max = max + 0.1
            total_range = max - min
            parts = []
            stepwidth = round((total_range / slices)+0.1, 1)
            print("PLOT BAR: stepwidth: "+str(stepwidth))
            base = round(min, 1)
            for i in range(0, slices+1):
                parts.append(round(base + (i*stepwidth), 1))
            result = []
            for i in range(0, slices):
                result.append(0)
            label = []
            for i in range(0, len(parts) - 1):
                label.append(str(
                    round(parts[i]+0.01, 2))+"-\n" + str(parts[i+1]))
            for z in value_list:
                for i in range(0, len(parts) - 1):
                    if parts[i] < z <= parts[i+1]:
                        result[i] = result[i] + 1
            fig2 = plt.figure()
            index = np.arange(len(label))
            plt.bar(index, result)
            plt.title("Distribution of " + x)
            if adjust_unit:
                x = x + " [" + unit + "]"
            if adjust_unit:
                plt.xlabel(x, fontsize=10)
            else:
                plt.xlabel(x, fontsize=10)
            plt.ylabel(y, fontsize=10)
            plt.xticks(index, label, fontsize=8, rotation=30)
            plt.tight_layout()
            for a, b in zip(index, result):
                plt.text(a, b, str(b), fontsize=8)
            if __name__ == "__main__":
                fig2.savefig(filename + ".svg", format="svg")
            else:
                try:
                    os.mkdir(project_path + "/results/" + str(
                        file_id) + "/generated")
                except OSError:
                    # case where directory already exists
                    pass
                fig2.savefig(project_path + "/results/" + str(
                    file_id) + "/generated/" + filename + ".svg", format="svg")
            plt.close('all')
    else:
        print("Using cached version of " + fpath)


# reads csv file and returns list with elements from csv file
def read_csv(results_path, file_name, file_id, thread_return=[], thr_id=-1):
    temp = []
    try:
        with open(os.path.join(results_path, file_name + "_" + str(file_id)
                               + ".csv"), "r") as csv_input:
            reader = csv.reader(csv_input, lineterminator="\n")
            for elem in reader:
                temp.append(int(elem[0]))
    except Exception as e:
        print("exception in csv reader:" + str(e))
        temp.append(-1)
    # if thread id is set use passed list to store results
    if thr_id > -1:
        thread_return[thr_id] = temp
    else:
        return temp


# entry point if analytics gets execute directly as a script
# and NOT as an included module
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='CSV reader for external host results.')
    parser.add_argument(
        '--id', help='ID of the csv files. Not set: use cfg file in /data',
        type=str, action="store", required=True)
    args = parser.parse_args()
    if args.id is None:
        print("No ID given")
    else:
        id = args.id
        multicast = "n/a"
        dir_path = os.path.dirname(os.path.realpath(__file__))
        try:
            with open(dir_path[0:dir_path.find("analytics")]+"/data/config_"
                      + id + ".json", "r") as cfg:
                config = json.load(cfg)
                multicast = config["multicast"]
        except Exception:
            multicast = ""
            print("config.json not found. path: " + dir_path[0:dir_path.find(
                "analytics")]+"/data/config_" + id + ".json")

        if len(id) > 0 and len(multicast) > 0:
            path = dir_path[0:dir_path.find("analytics")]+"results/"+str(id)
            results = main(id, multicast, path)
        else:
            print("Aborted execution.")

# Copyright 2019-present <NAME>, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

###############################################
# Creates graphs out of csv files from python #
# receiver; can be used included in views.py  #
# OR                                          #
# standalone by passing the --id flag         #
# OR                                          #
# standalone using the config in readme.txt   #
###############################################
import argparse
import csv
import json
import multiprocessing
import numpy as np
import os
import sys
import threading
import time

import matplotlib
matplotlib.use("Agg")
try:
    import matplotlib.pyplot as plt
except Exception as e:
    # prevent PEP3 warning because Agg import must be before pyplot
    raise e


csv.field_size_limit(sys.maxsize)
dir_path = os.path.dirname(os.path.realpath(__file__))
project_path = dir_path[0:dir_path.find("/analytics")]
lock = threading.RLock()


# read the csv files and plots the graphs
def main(file_id, multicast, results_path):
    def thread_join(thrs):
        for thread in thrs:
            thread.start()
        for thread in thrs:
            thread.join()

    fpath = project_path + "/results/" + str(
        file_id) + "/generated/extHost_results.json"
    if os.path.isfile(fpath):
        with open(fpath, "r") as f:
            print("Using cached version of " + fpath)
            return json.load(f)

    start = time.time()
    raw_packet_counter = read_csv(
        results_path, "raw_packet_counter", file_id)[0]
    packet_sizes = read_csv(results_path, "packet_sizes", file_id)
    timestamp1_list = read_csv(results_path, "timestamp1_list", file_id)
    timestamp2_list = read_csv(results_path, "timestamp2_list", file_id)
    end = time.time()
    print("It took " + str(end-start) +
          "s to load csv files for P4STA analytics.")

    # get list of cumulative packet sizes
    throughput_at_time = []
    total_throughput = 0
    for i in range(len(packet_sizes)):
        total_throughput = total_throughput + packet_sizes[i]
        throughput_at_time.append(total_throughput)

    latency_list = []
    min_latency = max_latency = ave_latency = \
        latency_variance = latency_std_deviation = 0
    total_ipdv = abs_total_ipdv = total_pdv = total_latencies = 0
    min_ipdv = max_ipdv = ave_ipdv = ave_abs_ipdv = 0
    min_pdv = max_pdv = ave_pdv = 0
    min_packets = max_packets = ave_packet_sec = 0
    ipdv_list = []
    count_list = []
    count_list_sec = []
    pdv_list = []
    mbit_list = []
    packet_list = []
    time_throughput = []
    if len(timestamp1_list) > 0 and len(timestamp2_list) > 0:
        if timestamp1_list[0] > 0 \
                and len(timestamp1_list) == len(timestamp2_list):
            # creates list of all latencies
            for i in range(0, len(timestamp1_list)):
                delta = timestamp2_list[i] - timestamp1_list[i]
                latency_list.append(delta)
                total_latencies = total_latencies + delta
                # sets the start time to 0 ms
                time_throughput.append(int(round(
                    (timestamp2_list[i] - timestamp2_list[0]) / 1000000)))
                # - [0] to set first time to 0 sec
                count_list_sec.append(
                    (timestamp2_list[i] - timestamp2_list[0]) / 1000000000)
            # minimum and maximum latency
            min_latency = min(latency_list, default=0)
            max_latency = max(latency_list, default=0)
            # fills pdv and ipdv lists
            for z in range(0, len(latency_list)):
                count_list.append(z)
                pdv = latency_list[z] - min_latency
                pdv_list.append(pdv)
                total_pdv = total_pdv + pdv
                if 0 < z < len(latency_list):
                    ipdv = latency_list[z] - latency_list[z - 1]
                    ipdv_list.append(ipdv)
                    total_ipdv = total_ipdv + ipdv
                    abs_total_ipdv = abs_total_ipdv + abs(ipdv)
                else:
                    ipdv_list.append(0)
            # calculate average latency, ipdv and pdv
            if len(latency_list) > 0:
                ave_latency = round(total_latencies / len(latency_list), 2)
                ave_ipdv = round(total_ipdv / len(ipdv_list))
                ave_abs_ipdv = round(abs_total_ipdv / len(ipdv_list))
                ave_pdv = round(total_pdv / len(pdv_list))
                # calculate standard deviation of latency
                total_sqr_dev = 0
                for z in range(0, len(latency_list)):
                    total_sqr_dev += (latency_list[z] - ave_latency)**2
                latency_variance = total_sqr_dev / len(ipdv_list)
                latency_std_deviation = latency_variance ** 0.5
            # minimum and maximum ipdv
            min_ipdv = min(ipdv_list, default=0)
            max_ipdv = max(ipdv_list, default=0)
            # minimum and maximum pdv
            min_pdv = min(pdv_list, default=0)
            max_pdv = max(pdv_list, default=0)
            # fill speed and packet rate lists
            last_time_hit = 0
            last_throughput_hit = 0
            last_packet_hit = 0
            mbit_list.append(0)
            packet_list.append(0)
            # prepares lists for speed and packet rate graph
            for y in range(0, len(throughput_at_time)):
                # more than 99ms difference -> 0.1s intervals
                if (time_throughput[y] - last_time_hit) >= 100:
                    if (time_throughput[y] - last_time_hit) >= 100:
                        amount = (time_throughput[y] - last_time_hit) / 100
                        # more than 200ms difference between two hits -> pause
                        if amount >= 2:
                            for i in range(0, int(round(amount))):
                                mbit_list.append(0)
                                packet_list.append(0)
                    last_time_hit = time_throughput[y]
                    # byte->megabit/10 measure every 0.1s but unit is mbit/s
                    mbit_list.append((throughput_at_time[y] -
                                      last_throughput_hit) * 8 / 100000)
                    # *10 measure for every 0.1s but unit is packets/seconds
                    packet_list.append((y - last_packet_hit)*10)
                    last_packet_hit = y
                    last_throughput_hit = throughput_at_time[y]
            mbit_list.append(0)  # set next entry to 0
            packet_list.append(0)

            # round by 9 digits after , float could result in weird fractions
            upsc_mbit_list = [round(x*int(multicast), 9) for x in mbit_list]

            upsc_packet_list = [x*int(multicast) for x in packet_list]
            # ignores the first and last second to prevent min = 0
            min_packets = min(packet_list[1:-1], default=0)
            max_packets = max(packet_list, default=0)
            if len(packet_list) != 2:
                # -2 because added 0 at start and end. *10 because of 0.1s step
                ave_packet_sec = round(
                    (len(throughput_at_time)/(len(packet_list)-2)) * 10, 2)
            else:
                ave_packet_sec = 0

            start = time.time()
            processes = []
            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list,
                    "Latency of DUT for every " + multicast + ". packet",
                    "Packets", "Latency", "latency", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list_sec,
                    "Latency of DUT for every " + multicast + ". packet",
                    "t[s]", "Latency", "latency_sec", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list,
                    "Latency of DUT for every " + multicast + ". packet",
                    "Packets", "Latency", "latency_y0", True, True, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    latency_list, count_list_sec,
                    "Latency of DUT for every " + multicast + ". packet",
                    "t[s]", "Latency", "latency_sec_y0", True, True, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    ipdv_list, count_list,
                    "IPDVs of DUT for every " + multicast + ". packet",
                    "IPDV", "Packets", "ipdv", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    ipdv_list, count_list_sec,
                    "IPDVs of DUT for every " + multicast + ". packet",
                    "t[s]", "IPDV", "ipdv_sec", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    pdv_list, count_list,
                    "PDVs of DUT for every " + multicast + ". packet",
                    "Packets", "PDV", "pdv", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    pdv_list, count_list_sec,
                    "PDVs of DUT for every " + multicast + ". packet",
                    "t[s]", "PDV", "pdv_sec", True, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    mbit_list, np.arange(0, len(mbit_list) / 10, 0.1),
                    "Throughput of DUT for every " + multicast + ". packet",
                    "t[s]", "Megabit/s", "speed", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    upsc_mbit_list,
                    np.arange(0, len(upsc_mbit_list) / 10, 0.1),
                    "Upscaled throughput of DUT", "t[s]", "Megabit/s",
                    "speed_upscaled", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    upsc_packet_list,
                    np.arange(0, len(upsc_packet_list) / 10, 0.1),
                    "Upscaled rate jitter of DUT", "t[s]", "Packet/s",
                    "packet_rate_upscaled", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_graph, args=(
                    packet_list, np.arange(0, len(packet_list) / 10, 0.1),
                    "Rate jitter of DUT for every " + multicast + ". packet",
                    "t[s]", "Packet/s", "packet_rate", False, False, file_id))
            processes.append(x)

            x = multiprocessing.Process(
                target=plot_bar, args=(
                    latency_list, min_latency, max_latency, "latency_bar",
                    "Latency", "Packets", 10, True, file_id))
            processes.append(x)

            thread_join(processes)
            end = time.time()
            print("MULTIP: It took " + str(end - start) +
                  "s to plot graphs in analytics.")

    results = {"num_raw_packets": raw_packet_counter,
               "num_processed_packets": len(latency_list),
               "total_throughput": round(total_throughput/1000000, 2),
               "min_latency": min_latency, "max_latency": max_latency,
               "avg_latency": ave_latency, "min_ipdv": min_ipdv,
               "max_ipdv": max_ipdv, "avg_ipdv": ave_ipdv,
               "avg_abs_ipdv": ave_abs_ipdv, "min_pdv": min_pdv,
               "max_pdv": max_pdv, "avg_pdv": ave_pdv,
               "min_packets_per_second": min_packets,
               "max_packets_per_second": max_packets,
               "avg_packets_per_second": ave_packet_sec,
               "latency_std_deviation": latency_std_deviation,
               "latency_variance": latency_variance,
               "latency_list": latency_list}
    if __name__ == "__main__":
        fpath = "extHost_results.json"
    with open(fpath, 'w', encoding='utf-8') as f:
        json.dump(results, f, indent=4)

    return results


# plots the line charts
def plot_graph(value_list_input, index_list, titel, x_label, y_label,
               filename, adjust_unit, adjust_y_ax, file_id):
    fpath = project_path + "/results/" + str(
                    file_id) + "/generated/" + filename + ".svg"
    if not os.path.isfile(fpath) or __name__ == "__main__":
        with lock:
            if adjust_unit:
                value_list, unit = find_unit(value_list_input)
            else:
                value_list = value_list_input
                unit = ""
            fig, ax = plt.subplots()
            ax.plot(index_list, value_list)
            plt.title(titel)
            if adjust_unit:
                y_label = y_label + " [" + unit + "]"
            # if adjust_y_ax is True sets y-axis to 0
            if adjust_y_ax:
                try:
                    temp = max(value_list_input)
                    if adjust_unit:
                        if unit == "microseconds":
                            temp = temp / 1000
                        elif unit == "milliseconds":
                            temp = temp / 1000000
                    # -0.075 sets 0 point 7.5% under 0
                    # for equal 0 at x and y axis
                    ax.set_ylim([-0.075 * temp, temp + 0.1 * temp])
                except Exception:
                    # no adjustment of y axis if error
                    # (e.g. empty iperf3 results)
                    pass
            plt.xlabel(x_label, fontsize=12)
            plt.ylabel(y_label, fontsize=12)
            plt.tight_layout()
            if __name__ == "__main__":
                fig.savefig(filename + ".svg", format="svg")
            else:
                try:
                    os.mkdir(project_path + "/results/" + str(
                        file_id) + "/generated")
                except OSError:
                    # case where directory already exists
                    pass
                fig.savefig(fpath, format="svg")
            plt.close('all')
    else:
        print("Using cached version of " + fpath)


# input: list
# returns list and string with unit
def find_unit(value_list_input):
    if type(value_list_input) != list:
        value_list_input = [value_list_input]
    try:
        microsec_counter = 0
        millisec_counter = 0
        value_list = []
        for i in value_list_input:
            if abs(i)/1000 >= 1:
                microsec_counter = microsec_counter + 1
            if abs(i)/1000000 >= 1:
                millisec_counter = millisec_counter + 1
        # if more than 95% of the values are bigger than 1 millisec = millisec
        if millisec_counter > (0.95 * len(value_list_input)):
            unit = "milliseconds"
            for i in value_list_input:
                value_list.append(round(i/1000000, 2))
            return value_list, unit
        # if more than 95% of the values are bigger than 1 microsec = microsec
        elif microsec_counter > (0.95 * len(value_list_input)):
            unit = "microseconds"
            for i in value_list_input:
                value_list.append(round(i/1000, 2))
            return value_list, unit
        else:
            unit = "nanoseconds"
            return value_list_input, unit
    except Exception:
        unit = "nanoseconds"
        return value_list_input, unit


# b_type = "bit" or "byte"
def find_unit_bit_byte(value, b_type):
    unit = b_type
    new_value = value
    if (value/1000) >= 1:
        unit = "kilo" + b_type
        new_value = value/1000
    if (value/1000000) >= 1:
        unit = "mega" + b_type
        new_value = value/1000000
    if (value/1000000000) >= 1:
        unit = "giga" + b_type
        new_value = value/1000000000

    return [new_value, unit]  # 1000bit => [1, "kilobit"]


# returns the given input value(ns^2), scaled to a matching unit
def find_unit_sqr(value_ns2):
    try:
        if abs(value_ns2) >= 1500000*1000000:
            unit = "ms²"
            value = round(value_ns2 / (1000000*1000000), 2)
            return value, unit
        if abs(value_ns2) >= 1500000:
            unit = "us²"
            value = round(value_ns2 / 1000000, 2)
            return value, unit
        unit = "ns²"
        return round(value_ns2, 2), unit
    except Exception:
        unit = "ns²"
        return value_ns2, unit


# plots bar chart
def plot_bar(value_list_input, min, max, filename,
             x, y, slices, adjust_unit, file_id):
    fpath = project_path + "/results/" + str(
        file_id) + "/generated/" + filename + ".svg"
    if not os.path.isfile(fpath) or __name__ == "__main__":
        with lock:
            if adjust_unit:
                value_list, unit = find_unit(value_list_input)
                if unit == "microseconds":
                    min = float(min) / 1000
                    max = float(max) / 1000
                elif unit == "milliseconds":
                    min = float(min) / 1000000
                    max = float(max) / 1000000
            else:
                value_list = value_list_input
                unit = ""

            min = min - 0.1
            max = max + 0.1
            total_range = max - min
            parts = []
            stepwidth = round((total_range / slices)+0.1, 1)
            print("PLOT BAR: stepwidth: "+str(stepwidth))
            base = round(min, 1)
            for i in range(0, slices+1):
                parts.append(round(base + (i*stepwidth), 1))
            result = []
            for i in range(0, slices):
                result.append(0)
            label = []
            for i in range(0, len(parts) - 1):
                label.append(str(
                    round(parts[i]+0.01, 2))+"-\n" + str(parts[i+1]))
            for z in value_list:
                for i in range(0, len(parts) - 1):
                    if parts[i] < z <= parts[i+1]:
                        result[i] = result[i] + 1
            fig2 = plt.figure()
            index = np.arange(len(label))
            plt.bar(index, result)
            plt.title("Distribution of " + x)
            if adjust_unit:
                x = x + " [" + unit + "]"
            if adjust_unit:
                plt.xlabel(x, fontsize=10)
            else:
                plt.xlabel(x, fontsize=10)
            plt.ylabel(y, fontsize=10)
            plt.xticks(index, label, fontsize=8, rotation=30)
            plt.tight_layout()
            for a, b in zip(index, result):
                plt.text(a, b, str(b), fontsize=8)
            if __name__ == "__main__":
                fig2.savefig(filename + ".svg", format="svg")
            else:
                try:
                    os.mkdir(project_path + "/results/" + str(
                        file_id) + "/generated")
                except OSError:
                    # case where directory already exists
                    pass
                fig2.savefig(project_path + "/results/" + str(
                    file_id) + "/generated/" + filename + ".svg", format="svg")
            plt.close('all')
    else:
        print("Using cached version of " + fpath)


# reads csv file and returns list with elements from csv file
def read_csv(results_path, file_name, file_id, thread_return=[], thr_id=-1):
    temp = []
    try:
        with open(os.path.join(results_path, file_name + "_" + str(file_id)
                               + ".csv"), "r") as csv_input:
            reader = csv.reader(csv_input, lineterminator="\n")
            for elem in reader:
                temp.append(int(elem[0]))
    except Exception as e:
        print("exception in csv reader:" + str(e))
        temp.append(-1)
    # if thread id is set use passed list to store results
    if thr_id > -1:
        thread_return[thr_id] = temp
    else:
        return temp


# entry point if analytics gets execute directly as a script
# and NOT as an included module
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='CSV reader for external host results.')
    parser.add_argument(
        '--id', help='ID of the csv files. Not set: use cfg file in /data',
        type=str, action="store", required=True)
    args = parser.parse_args()
    if args.id is None:
        print("No ID given")
    else:
        id = args.id
        multicast = "n/a"
        dir_path = os.path.dirname(os.path.realpath(__file__))
        try:
            with open(dir_path[0:dir_path.find("analytics")]+"/data/config_"
                      + id + ".json", "r") as cfg:
                config = json.load(cfg)
                multicast = config["multicast"]
        except Exception:
            multicast = ""
            print("config.json not found. path: " + dir_path[0:dir_path.find(
                "analytics")]+"/data/config_" + id + ".json")

        if len(id) > 0 and len(multicast) > 0:
            path = dir_path[0:dir_path.find("analytics")]+"results/"+str(id)
            results = main(id, multicast, path)
        else:
            print("Aborted execution.")
# Copyright 2019-present <NAME>, <NAME> # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #    http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################### # Creates graphs out of csv files from python # # receiver; can be used included in views.py  # # OR                                          # # standalone by passing the --id flag         # # OR                                          # # standalone using the config in readme.txt   # ############################################### # prevent PEP3 warning because Agg import must be before pyplot # read the csv files and plots the graphs # get list of cumulative packet sizes # creates list of all latencies # sets the start time to 0 ms # - [0] to set first time to 0 sec # minimum and maximum latency # fills pdv and ipdv lists # calculate average latency, ipdv and pdv # calculate standard deviation of latency # minimum and maximum ipdv # minimum and maximum pdv # fill speed and packet rate lists # prepares lists for speed and packet rate graph # more than 99ms difference -> 0.1s intervals # more than 200ms difference between two hits -> pause # byte->megabit/10 measure every 0.1s but unit is mbit/s # *10 measure for every 0.1s but unit is packets/seconds # set next entry to 0 # round by 9 digits after , float could result in weird fractions # ignores the first and last second to prevent min = 0 # -2 because added 0 at start and end. *10 because of 0.1s step # plots the line charts # if adjust_y_ax is True sets y-axis to 0 # -0.075 sets 0 point 7.5% under 0 # for equal 0 at x and y axis # no adjustment of y axis if error # (e.g. empty iperf3 results) # case where directory already exists # input: list # returns list and string with unit # if more than 95% of the values are bigger than 1 millisec = millisec # if more than 95% of the values are bigger than 1 microsec = microsec # b_type = "bit" or "byte" # 1000bit => [1, "kilobit"] # returns the given input value(ns^2), scaled to a matching unit # plots bar chart # case where directory already exists # reads csv file and returns list with elements from csv file # if thread id is set use passed list to store results # entry point if analytics gets execute directly as a script # and NOT as an included module