import os
import re
import sys
import argparse

'''
Proccess and generate appropriate data format 
for model training

state.log is used for keep track of the last states of
proccessed data so that it is able to automatically label
the new data

@<NAME>
 
'''
# TODO try catch block

def parse_commandline():
    parser = argparse.ArgumentParser(description='Driver for processing the graph data to model training')
    parser.add_argument('dir', help='path to the folder of the graph data')
    parser.add_argument('--id', nargs='?', default='loop_0')
    parser.add_argument('--order', nargs='?', default=0)

    args = parser.parse_args()
    
    return args


class DataGenerator():
    '''
    The data generator used for parsing the output from llvm pass (Orio postprocess)
    to generate the graph data for further analysis (e.g. embedding, classfication)
    '''
    def __init__(self):
        cmd_args = parse_commandline()

        # target directory
        self.dir_path = cmd_args.dir    
        # process mode
        self.loop_id = cmd_args.id
        self.loop_order = int(cmd_args.order)
        # node label
        self.nl = ['Flop', 'Mem', 'Ctrl', 'Int']
        

    def __read_log(self):
        '''
        Read log file to keep track of the parameters for the input of graph framework
        Parameters: number of node, number of graph, number of class
        '''
        try:
            with open('state.log', 'r+') as f:
                line = f.readline()
        except IOError:
            with open('state.log', 'w'): pass
            return 1, 1, 1  
    
        # parse the line 
        return map(int, re.split(r'\t+', line))

    def __write_log(self):
        '''
        Keep the state of three parameters in the log file
        '''
        with open('state.log', 'w') as f:
            f.write(str(self.node_num) + '\t' + str(self.graph_num) + '\t' + str(self.graph_class))

    def __check_file_name(self, file_name):
        '''
        Check whether the file is to process    
        '''
        if not file_name.endswith('.matrix'):
            return False

        # process file name
        # generated file naming follow:
        # original_file_name.loop_id@order.matrix
        fn_str = file_name.split('.')[-2]
        fn_id, fn_order = fn_str.split('@')
        if self.loop_id != 'loop_0':
            # process the file based on loop id
            if self.loop_id == fn_id:
                return True
        elif self.loop_order != 0:
            # process the file based on loop processing order
            if self.loop_order == int(fn_order):
                return True

        return False    

            
    def __iter_dir(self):
        # assume test case locates in a/b/c/src/
        # get src name
        src_name = os.path.basename(os.path.normpath(self.dir_path))
        process_flag = False

        for file_name in os.listdir(self.dir_path):
            if file_name.endswith('.temp'):
                # Temporary workaround for processing splitted loops data
                # TODO maybe we can have this as an argument
                skip_num = 0
                process_num = 20
                # temp directory
                sub_dir = self.dir_path + '/' + file_name 
                # get test version, hardcoded here
                # e.g __orio_perftest7.c.temp -> 7
                # will not work for non Orio generate data
                #ver = file_name[15:file_name.find('.')]
                
                # get test name: name + version
                # e.g jocobi20
                #test_name = src_name + ver
                # use to differentiate graph generate from same src
                # jacobi7_2 means second graph from jacobi7 test
                # graph_counter = 1
                for sub_name in os.listdir(sub_dir):
                    # skip validate.c.xx.matrix
                    if "validation." in sub_name:
                        continue
                   
                    if process_num == 0:
                        break
                    
                    if self.__check_file_name(sub_name):
                        # find matrix file
                        # each matrix is a graph
                        target_file = sub_dir+'/'+sub_name
                        if os.stat(target_file).st_size !=0:
                            # avoid empty file
                            print(sub_name+'\n')
                            test_name = src_name + '_' + sub_name[0:sub_name.rfind('.')]
                            #self.__process_matrix(target_file, test_name+'_'+str(graph_counter))
                            self.__process_matrix(target_file, test_name)
                            self.graph_num += 1
                            process_flag = True
                            #graph_counter += 1
        # iter over all graphs of the same class
        if process_flag:
            # at least processed one graph 
            self.graph_class += 1

    def __process_matrix(self, file_path, graph_id=''):
        # parse the .matrix file to get data

        base_node_idx = self.node_num
        base_graph_idx = self.graph_num
        with open(file_path, 'r') as f:
            # first line flag
            fl_flag = False
            # list of list for storing node attributes
            node_attr = []
            for line in f:
                if not fl_flag:
                    # first line
                    mat_size = int(line)
                    fl_flag = True
                    # list for storing inbound edge weight
                    edge_w = [0.0] * mat_size
                else:
                    line_value = re.split(r'\t+', line.rstrip())
                    edge_list = line_value[1:]  
                    # output edge
                    e_idx = self.node_num
                    for e in edge_list:
                        if float(e) != 0:
                            # into whole graph file 
                            self.graph_hdlr.write(str(base_node_idx) + ', ' + str(e_idx) + '\n')                    
                    
                            # into edge attribute file
                            self.edge_attr_hdlr.write(e + '\n')
                            # record inbound edge weight
                            edge_w[e_idx-self.node_num] += float(e)
                        #update idx
                        e_idx += 1

                    # ouput node
                    instr_cnt = re.split(r':+', line_value[0])
                    instr_list = map(float, instr_cnt)
                    # decide node label
                    node_label = instr_list.index(max(instr_list))  
                    self.node_label_hdlr.write(str(node_label) + '\n')
                    # since we combine the edge weight with node attribute
                    # not output here, instead save the attributes
                    #self.node_attr_hdlr.write(', '.join(instr_cnt) + '\n')
                    node_attr.append(instr_cnt)
                    # output graph
                    self.graph_indi_hdlr.write(str(base_graph_idx)+'\n')    

                    base_node_idx += 1  
            # end line loop
            self.node_num = base_node_idx
            self.graph_label_hdlr.write(str(self.graph_class) + '\n')   
            self.graph_id_hdlr.write(graph_id + '\n')   
            # write node attribute before moving to next matrix
            self.__edge_to_node(node_attr, edge_w)          


    def __edge_to_node(self, node_attr, edge_attr):
        # set the edge transition probability as a node attribute (last one) 
        for i in range(len(edge_attr)):
            self.node_attr_hdlr.write(', '.join(node_attr[i]) + ', ' + str(edge_attr[i]) + '\n')


    def __get_file_handler(self):
        # return file handlers to append
        self.graph_hdlr = open('CFG_A.txt', 'a')
        self.graph_indi_hdlr = open('CFG_graph_indicator.txt', 'a')
        self.graph_label_hdlr = open('CFG_graph_labels.txt', 'a')
        self.node_attr_hdlr = open('CFG_node_attributes.txt', 'a')
        self.node_label_hdlr = open('CFG_node_labels.txt', 'a')
        self.edge_attr_hdlr = open('CFG_edge_attributes.txt', 'a')
        self.graph_id_hdlr = open('CFG_graph_id.txt', 'a')

    def __close_file(self):
        # close opened file
        self.graph_hdlr.close()
        self.graph_indi_hdlr.close()
        self.graph_label_hdlr.close()
        self.node_attr_hdlr.close()
        self.node_label_hdlr.close()
        self.edge_attr_hdlr.close()
        self.graph_id_hdlr.close()

    def generate(self):
        if self.loop_id=='loop_0' and self.loop_order==0:
            print ('Missing argument, exit ...')
            sys.exit()
        # get the current index of node, graph and current graph class 
        self.node_num, self.graph_num, self.graph_class = self.__read_log()
        self.__get_file_handler()
        # start process
        self.__iter_dir()
        self.__close_file()
        self.__write_log()

if __name__ == "__main__":
    DataGenerator().generate()

import os
import re
import sys
import argparse

'''
Proccess and generate appropriate data format 
for model training

state.log is used for keep track of the last states of
proccessed data so that it is able to automatically label
the new data

@<NAME>
 
'''
# TODO try catch block

def parse_commandline():
    parser = argparse.ArgumentParser(description='Driver for processing the graph data to model training')
    parser.add_argument('dir', help='path to the folder of the graph data')
    parser.add_argument('--id', nargs='?', default='loop_0')
    parser.add_argument('--order', nargs='?', default=0)

    args = parser.parse_args()
    
    return args


class DataGenerator():
    '''
    The data generator used for parsing the output from llvm pass (Orio postprocess)
    to generate the graph data for further analysis (e.g. embedding, classfication)
    '''
    def __init__(self):
        cmd_args = parse_commandline()

        # target directory
        self.dir_path = cmd_args.dir    
        # process mode
        self.loop_id = cmd_args.id
        self.loop_order = int(cmd_args.order)
        # node label
        self.nl = ['Flop', 'Mem', 'Ctrl', 'Int']
        

    def __read_log(self):
        '''
        Read log file to keep track of the parameters for the input of graph framework
        Parameters: number of node, number of graph, number of class
        '''
        try:
            with open('state.log', 'r+') as f:
                line = f.readline()
        except IOError:
            with open('state.log', 'w'): pass
            return 1, 1, 1  
    
        # parse the line 
        return map(int, re.split(r'\t+', line))

    def __write_log(self):
        '''
        Keep the state of three parameters in the log file
        '''
        with open('state.log', 'w') as f:
            f.write(str(self.node_num) + '\t' + str(self.graph_num) + '\t' + str(self.graph_class))

    def __check_file_name(self, file_name):
        '''
        Check whether the file is to process    
        '''
        if not file_name.endswith('.matrix'):
            return False

        # process file name
        # generated file naming follow:
        # original_file_name.loop_id@order.matrix
        fn_str = file_name.split('.')[-2]
        fn_id, fn_order = fn_str.split('@')
        if self.loop_id != 'loop_0':
            # process the file based on loop id
            if self.loop_id == fn_id:
                return True
        elif self.loop_order != 0:
            # process the file based on loop processing order
            if self.loop_order == int(fn_order):
                return True

        return False    

            
    def __iter_dir(self):
        # assume test case locates in a/b/c/src/
        # get src name
        src_name = os.path.basename(os.path.normpath(self.dir_path))
        process_flag = False

        for file_name in os.listdir(self.dir_path):
            if file_name.endswith('.temp'):
                # Temporary workaround for processing splitted loops data
                # TODO maybe we can have this as an argument
                skip_num = 0
                process_num = 20
                # temp directory
                sub_dir = self.dir_path + '/' + file_name 
                # get test version, hardcoded here
                # e.g __orio_perftest7.c.temp -> 7
                # will not work for non Orio generate data
                #ver = file_name[15:file_name.find('.')]
                
                # get test name: name + version
                # e.g jocobi20
                #test_name = src_name + ver
                # use to differentiate graph generate from same src
                # jacobi7_2 means second graph from jacobi7 test
                # graph_counter = 1
                for sub_name in os.listdir(sub_dir):
                    # skip validate.c.xx.matrix
                    if "validation." in sub_name:
                        continue
                   
                    if process_num == 0:
                        break
                    
                    if self.__check_file_name(sub_name):
                        # find matrix file
                        # each matrix is a graph
                        target_file = sub_dir+'/'+sub_name
                        if os.stat(target_file).st_size !=0:
                            # avoid empty file
                            print(sub_name+'\n')
                            test_name = src_name + '_' + sub_name[0:sub_name.rfind('.')]
                            #self.__process_matrix(target_file, test_name+'_'+str(graph_counter))
                            self.__process_matrix(target_file, test_name)
                            self.graph_num += 1
                            process_flag = True
                            #graph_counter += 1
        # iter over all graphs of the same class
        if process_flag:
            # at least processed one graph 
            self.graph_class += 1

    def __process_matrix(self, file_path, graph_id=''):
        # parse the .matrix file to get data

        base_node_idx = self.node_num
        base_graph_idx = self.graph_num
        with open(file_path, 'r') as f:
            # first line flag
            fl_flag = False
            # list of list for storing node attributes
            node_attr = []
            for line in f:
                if not fl_flag:
                    # first line
                    mat_size = int(line)
                    fl_flag = True
                    # list for storing inbound edge weight
                    edge_w = [0.0] * mat_size
                else:
                    line_value = re.split(r'\t+', line.rstrip())
                    edge_list = line_value[1:]  
                    # output edge
                    e_idx = self.node_num
                    for e in edge_list:
                        if float(e) != 0:
                            # into whole graph file 
                            self.graph_hdlr.write(str(base_node_idx) + ', ' + str(e_idx) + '\n')                    
                    
                            # into edge attribute file
                            self.edge_attr_hdlr.write(e + '\n')
                            # record inbound edge weight
                            edge_w[e_idx-self.node_num] += float(e)
                        #update idx
                        e_idx += 1

                    # ouput node
                    instr_cnt = re.split(r':+', line_value[0])
                    instr_list = map(float, instr_cnt)
                    # decide node label
                    node_label = instr_list.index(max(instr_list))  
                    self.node_label_hdlr.write(str(node_label) + '\n')
                    # since we combine the edge weight with node attribute
                    # not output here, instead save the attributes
                    #self.node_attr_hdlr.write(', '.join(instr_cnt) + '\n')
                    node_attr.append(instr_cnt)
                    # output graph
                    self.graph_indi_hdlr.write(str(base_graph_idx)+'\n')    

                    base_node_idx += 1  
            # end line loop
            self.node_num = base_node_idx
            self.graph_label_hdlr.write(str(self.graph_class) + '\n')   
            self.graph_id_hdlr.write(graph_id + '\n')   
            # write node attribute before moving to next matrix
            self.__edge_to_node(node_attr, edge_w)          


    def __edge_to_node(self, node_attr, edge_attr):
        # set the edge transition probability as a node attribute (last one) 
        for i in range(len(edge_attr)):
            self.node_attr_hdlr.write(', '.join(node_attr[i]) + ', ' + str(edge_attr[i]) + '\n')


    def __get_file_handler(self):
        # return file handlers to append
        self.graph_hdlr = open('CFG_A.txt', 'a')
        self.graph_indi_hdlr = open('CFG_graph_indicator.txt', 'a')
        self.graph_label_hdlr = open('CFG_graph_labels.txt', 'a')
        self.node_attr_hdlr = open('CFG_node_attributes.txt', 'a')
        self.node_label_hdlr = open('CFG_node_labels.txt', 'a')
        self.edge_attr_hdlr = open('CFG_edge_attributes.txt', 'a')
        self.graph_id_hdlr = open('CFG_graph_id.txt', 'a')

    def __close_file(self):
        # close opened file
        self.graph_hdlr.close()
        self.graph_indi_hdlr.close()
        self.graph_label_hdlr.close()
        self.node_attr_hdlr.close()
        self.node_label_hdlr.close()
        self.edge_attr_hdlr.close()
        self.graph_id_hdlr.close()

    def generate(self):
        if self.loop_id=='loop_0' and self.loop_order==0:
            print ('Missing argument, exit ...')
            sys.exit()
        # get the current index of node, graph and current graph class 
        self.node_num, self.graph_num, self.graph_class = self.__read_log()
        self.__get_file_handler()
        # start process
        self.__iter_dir()
        self.__close_file()
        self.__write_log()

if __name__ == "__main__":
    DataGenerator().generate()

Proccess and generate appropriate data format  for model training  state.log is used for keep track of the last states of proccessed data so that it is able to automatically label the new data  @<NAME> # TODO try catch block The data generator used for parsing the output from llvm pass (Orio postprocess)     to generate the graph data for further analysis (e.g. embedding, classfication) # target directory # process mode # node label Read log file to keep track of the parameters for the input of graph framework         Parameters: number of node, number of graph, number of class # parse the line Keep the state of three parameters in the log file Check whether the file is to process # process file name # generated file naming follow: # original_file_name.loop_id@order.matrix # process the file based on loop id # process the file based on loop processing order # assume test case locates in a/b/c/src/ # get src name # Temporary workaround for processing splitted loops data # TODO maybe we can have this as an argument # temp directory # get test version, hardcoded here # e.g __orio_perftest7.c.temp -> 7 # will not work for non Orio generate data #ver = file_name[15:file_name.find('.')] # get test name: name + version # e.g jocobi20 #test_name = src_name + ver # use to differentiate graph generate from same src # jacobi7_2 means second graph from jacobi7 test # graph_counter = 1 # skip validate.c.xx.matrix # find matrix file # each matrix is a graph # avoid empty file #self.__process_matrix(target_file, test_name+'_'+str(graph_counter)) #graph_counter += 1 # iter over all graphs of the same class # at least processed one graph # parse the .matrix file to get data # first line flag # list of list for storing node attributes # first line # list for storing inbound edge weight # output edge # into whole graph file # into edge attribute file # record inbound edge weight #update idx # ouput node # decide node label # since we combine the edge weight with node attribute # not output here, instead save the attributes #self.node_attr_hdlr.write(', '.join(instr_cnt) + '\n') # output graph # end line loop # write node attribute before moving to next matrix # set the edge transition probability as a node attribute (last one) # return file handlers to append # close opened file # get the current index of node, graph and current graph class # start process