<reponame>ouyangzhiping/Info-extract<filename>prediction-experiments/python-nb/NodeSequenceRegression.py
#!/usr/bin/env python
# coding: utf-8

# ## Outcome value prediction from Behaviour Change Data
# 
# In this notebook, we set up a regression/classification pipeline to predict either the outcome value
# directly or an interval in which this value may fall into, the former being `regression` and the latter being `multi-class classification`.
# 
# Before getting started with this notebook, let's have a look at the format of the data files which this program expects. The data is specified as tsv (tab separated values) and is generated by running the following command from the HBCP project root directory.
# 
# ```
# mvn exec:java@svmreg -Dexec.args="true"
# ```
# The above command generates the train and the test files located at the directory `prediction/sentences/`.
# Each line of the data files (train and test) looks like the following:
# ```
# C:5579097:35.7 C:5594106:16.1 I:3674268:1 C:5579728:30.6 I:3674248:1 C:5579118:22 C:5579689:14.6 C:5579088:44.5 C:5579711:80.6 C:5580203:29.4 O:4087178:abstinence C:5594105:19.3 O:4087186:cotinine C:5580200:35.7 O:4087187:2 C:5579096:58.8 I:3675703:1 C:5580204:5.9 I:3675698:1 C:5579663:22 C:5579083:29.4 O:4087191:6 I:3673288:1 O:4087172:1 I:3674264:1 I:3675717:1 C:5580216:0 	2.8
# ```
# 
# Each token represents a `:`-separated `<attribute-type>:<attribute-id>:<value>` combination, where the attribute type is one of `{C, I, O}` (contextual, intervention or outcome qualifier) feature, an attribute-id is a unique integer and a value is the textual representation of an instance of this feature.   
# 
# 

# In[4]:


#All neccessary imports and global variables
import sys, getopt

from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import OneHotEncoder
import numpy as np

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.text import text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences
from keras import backend as K
from numpy.random import seed
#from tensorflow import set_random_seed

# import the necessary packages
from keras.models import Sequential
from keras import layers

seed(110781)

import tensorflow as tf
#tf.random.set_seed(110781)

import random
import re
import os

PATTERN = re.compile("(?<![0-9])-?[0-9]*\.?[0-9]+")


# In[ ]:


def rmse(y_true, y_pred):
        return K.sqrt(K.mean(K.square(y_pred - y_true))) 


# In[ ]:


class InputHelper(object):
    emb_dim = 0
    pre_emb = dict() # word--> vec
    vocab_size = 0
    tokenizer = None
    embedding_matrix = None
    
    def cleanText(self, s):
        s = re.sub(r"[^\x00-\x7F]+"," ", s)
        s = re.sub(r'[\~\!\`\^\*\{\}\[\]\#\<\>\?\+\=\-\_\(\)]+',"",s)
        s = re.sub(r'( [0-9,\.]+)',r"\1 ", s)
        s = re.sub(r'\$'," $ ", s)
        s = re.sub('[ ]+',' ', s)
        return s.lower()

    #the tokenizer needs to be trained on the pre-trained node vectors
    #join the names of the nodes in a string so that tokenizer could be fit on it
    def getAllNodes(self, emb_path):
        print("Collecting node names...")
        line_count = 0        
        node_names = []
        for line in open(emb_path):
            l = line.strip().split()
            if (line_count > 0):
                node_names.append(l[0])
            
            line_count = line_count + 1
        
        self.vocab_size = line_count # includes the +1
        print("Collected node names...")
        return node_names

    # call convertWordsToIds first followed by loadW2V
    def convertWordsToIds(self, emb_path):
        allNodeNames = self.getAllNodes(emb_path)
        
        print ("Converting words to ids...")
        # Map words to ids        
        self.tokenizer = Tokenizer(num_words=self.vocab_size, filters=[], lower=False, split=" ")
        self.tokenizer.fit_on_texts(allNodeNames)
        print ("Finished converting words to ids...")
    
    # Assumes that the tokenizer already has been fit on some text (for the time being the node vec names)
    def loadW2V(self, emb_path):
        print("Loading W2V data...")
        line_count = 0        
        
        for line in open(emb_path):
            l = line.strip().split()
            if (line_count == 0):
                self.emb_dim = int(l[1])
                self.embedding_matrix = np.zeros((self.vocab_size, self.emb_dim))
            else:
                st = l[0]
                self.pre_emb[st] = np.asarray(l[1:]) # rest goes as the vector components
                if st in self.tokenizer.word_index:
                    idx = self.tokenizer.word_index[st]
                    self.embedding_matrix[idx] = np.array(l[1:], dtype=np.float32)[:self.emb_dim]
                else:
                    print ("Word '{}' not found in vocabulary..".format(st))
                
            line_count = line_count + 1
            
        print("loaded word2vec for {} nodes".format(len(self.pre_emb)))
    
    # Load the data as two matrices - X and Y
    def getTsvData(self, filepath):
        print("Loading data from " + filepath)
        x = []
        y = []
        
        # positive samples from file
        for line in open(filepath):
            l = line.strip().split("\t")            
            y.append(l[1])
            words = l[0].split(" ")
            x.append(words)
            #for w in words:
            #    x.append(w)
            
        return np.asarray(x), np.asarray(y)
    
    # Build sequences from each data instance
    def getSequenceData(self, tsvDataPath, numClasses=0, seed=123456):
        x_text, y = self.getTsvData(tsvDataPath)
        
        # Convert each sentence (node name sequence) to a sequence of integer ids
        x = self.tokenizer.texts_to_sequences(x_text)
        #print (x)
        
        if (numClasses > 0):
            y = self.categorizeOutputs(y, numClasses)
        
        return x, y
    
    def categorizeOutputs(self, y, numClasses):
        # allowable integers: 0,1...nclasses-1 (nclasses values)
        scaler = MinMaxScaler(feature_range=(0, numClasses-1))
        y_scaled = scaler.fit_transform(y.reshape(-1, 1))
        y_scaled = np.trunc(y_scaled)
        
        return y_scaled


# In[ ]:


def add_value_feature(in_fn, out_fn):
    """Read (text) file of (dense) vectors and add a final value to the vector for the actual value of the node.
    This allows to represent the distance between nodes. Normalize this feature between -1 and 1.
    For numerical -1 is the minimum value in the range, 1 the max.
    For BCT, +1 is the presence, -1 is the absence
    For categorical we create ranges bet. -1 and 1.
    For other, pick some random number close to 0.
    """

    print ("Writing appended vec file at %s" %(out_fn))
    random.seed(123)
    att_values, type_att = collect_attribute_value_maps(in_fn)
    numeric_atts = infer_numerical_attributes(att_values, type_att)

    att_maxes, att_mins = get_att_max_min(att_values, numeric_atts)  # max/min used for normalization

    # debug -- print maxes and mins
    print("There are %d numeric attributes." % len(numeric_atts))
    for num_att_id in numeric_atts:
        print("Numeric att: %s -- Min: %f ; Max: %f" % (num_att_id, att_mins[num_att_id], att_maxes[num_att_id]))

    # go through the file again and add 'normalized' values
    with open(in_fn) as f:
        with open(out_fn, 'w') as f_out:
            for line in f:
                cols = line.split()
                if len(cols) == 2:  # first line
                    f_out.write(line)
                    continue
                prefix, att_id, val = cols[0].split(':', 2)
                # BCTs stay the same
                if prefix == 'I':
                    norm_val = val
                # numerical attributes get normalized
                elif att_id in numeric_atts:
                    match = PATTERN.search(val)
                    if match is not None:
                        num = float(match.group(0))
                        # max-min normalization
                        if att_maxes[att_id] == att_mins[att_id]:
                            norm_val = "1"
                        else:
                            norm_num = 2 * ((num - att_mins[att_id]) / (att_maxes[att_id] - att_mins[att_id])) - 1
                            norm_val = str(norm_num)
                    else:
                        norm_val = "%f" % random.gauss(0, 0.001)
                # TODO not keeping track of categorical attributes yet
                # remaining attributes will get a random value close to zero (not sure what else to do with them)
                else:
                    norm_val = "%f" % random.gauss(0, 0.001)
                # f_out.write(cols[0] + '\t' + norm_val + '\n')
                f_out.write('{0} {1}\n'.format(line.strip(), norm_val))


def get_att_max_min(att_values, numeric_atts):
    # normalize numeric attributes
    att_maxes = {}
    att_mins = {}
    for num_att_id in numeric_atts:
        # get max and min
        nums = []
        for val in att_values[num_att_id]:
            # does val have a number
            match = PATTERN.search(val)
            if match is not None:
                num = float(match.group(0))
                nums.append(num)
        att_mins[num_att_id] = min(nums)
        att_maxes[num_att_id] = max(nums)
    return att_maxes, att_mins


def infer_numerical_attributes(att_values, type_att):
    # check if attribute is numerical (use logic from Martin's Java code)
    numeric_atts = []
    for att_id, vals in att_values.items():
        if att_id in type_att['I']:
            continue  # interventions will all be '1'
        num_val = 0
        for val in vals:
            # does val have a number
            match = PATTERN.search(val)
            if match is not None:
                num_val += 1
        # if 80% or more have numbers, then consider numeric
        if num_val / len(vals) >= 0.8:
            numeric_atts.append(att_id)
    return numeric_atts


def collect_attribute_value_maps(fn):
    att_values = {}
    type_att = {'C': set(), 'I': set(), 'O': set(), 'V': set()}
    with open(fn) as f:
        for line in f:
            cols = line.split()
            if len(cols) == 2:  # first line, skip
                continue
            prefix, att_id, val = cols[0].split(':', 2)
            type_att[prefix].add(att_id)
            if att_id in att_values:
                att_values[att_id].append(val)
            else:
                att_values[att_id] = [val]
    print("There are %d attributes." % len(att_values.keys()))
    print("There are %d interventions." % len(type_att['I']))
    return att_values, type_att


# Set the global parameters. Set `NUM_CLASSES` to `0` if you want to run the regression flow, otherwise set this to the number of classes (outcome value ranges).
# 
# The function `convertWordsToIds` converts each word (e.g. `C:5579097:35.7`) into an id. 

# In[ ]:


#Load datasets and embedded vectors
def loadData(inpH, emb_file, train_file, test_file, numClasses=0):
    
    #Load the training and the test sets
    #Load the text as a sequence of inputs
    
    x_train, y_train = inpH.getSequenceData(train_file, numClasses)
    x_test, y_test = inpH.getSequenceData(test_file, numClasses)

    #First few sequences
    for i in range(4):
        print ("x[{}][:5]..., y[{}] = {}, {}".format(i, i, x_train[i][:5], y_train[i]))    
        
    if (numClasses > 0):    
        encoder = OneHotEncoder(sparse=False, categories='auto')
        y_all = np.vstack((y_train, y_test))

        encoder.fit(y_all)

        y_train = encoder.transform(y_train)
        y_test = encoder.transform(y_test)
        for i in range(2):
            print ("y_train[{}] = {}".format(i, y_train[i]))
            print ("y_test[{}] = {}".format(i, y_test[i]))
            
    #Load the word vectors
    inpH.loadW2V(emb_file)
    
    #Print the loaded words
    nwords=0
    for w in inpH.pre_emb:
        print ("Dimension of vectors: {}".format(inpH.pre_emb[w].shape))
        print ("{} {}".format(w, inpH.pre_emb[w][0:5]))
        nwords = nwords+1
        if (nwords >= 2): break

    print ("vocab size: {}".format(inpH.vocab_size))
    print ("emb-matrix: {}...".format(inpH.embedding_matrix[1][:5]))
    print (inpH.embedding_matrix.shape)
    
    return x_train, y_train, x_test, y_test


# In[ ]:


def trainRegression(inpH, x_train, y_train, x_test, y_test, batch_size, hidden_layer_dim=20, num_classes=0, epochs=30, maxlen=300):
    if (num_classes > 0):
        loss_fn = 'categorical_crossentropy'
        eval_metrics = ['accuracy']
        activation_fn = 'softmax'
        output_dim = NUM_CLASSES
    else:
        loss_fn = rmse
        eval_metrics = [rmse]
        activation_fn = 'linear'
        output_dim = 1
        
    model = Sequential()
    model.add(layers.Embedding(input_dim=inpH.vocab_size, 
                               output_dim=inpH.emb_dim, 
                               input_length=maxlen,
                               weights=[inpH.embedding_matrix],
                               trainable=False))
    model.add(layers.Flatten())
    model.add(layers.Dense(20, activation='relu'))
    model.add(layers.Dense(output_dim, activation=activation_fn, name='output_vals'))
    model.compile(optimizer='adam',
                  loss = loss_fn,
                  metrics=eval_metrics)
    model.summary()
    
    print ("Training model...")
    history = model.fit(x_train, y_train,
        epochs=epochs,
        verbose=True,
        #validation_data=(x_test, y_test),
        batch_size=batch_size)
    
    loss, accuracy = model.evaluate(x_test, y_test, verbose=True)
    if (num_classes > 0):
        print("Cross-entropy loss: {:.4f}, Accuracy: {:.4f}".format(loss, accuracy))
    else:
        print("Loss: {:.4f}, RMSE: {:.4f}".format(loss, accuracy))    
        
    return model


# In[ ]:


def testRegression(model, x_test, y_test, res_file, num_classes=0):
    #Print each prediction    
    y_preds = model.predict(x_test)
    
    f = open(res_file, "w")
    for i in range(y_preds.shape[0]):
        if (num_classes > 0):
            f.write ("{:d}\t{:d}\n".format(np.argmax(y_test[i]), np.argmax(y_preds[i])))
        else:
            f.write ("{}\t{:.4f}\n".format(y_test[i], y_preds[i][0]))

    f.close()    


# In[ ]:


def main(argv):
    NUM_CLASSES = 0 # set this to 0 for regression and a positive value for classification
    TRAIN_FILE = "../sentences/train.tsv"
    TEST_FILE = "../sentences/test.tsv"
    TO_ADD_VALUE = 0
    RES_FILE = "predictions.txt"
    EMB_FILE = "../graphs/nodevecs/refVecs.vec"
    MAXLEN = 300
    
    try:
        opts, args = getopt.getopt(argv,"ha:i:e:o:n:", ["appendnumeric=", "trainfile=", "testfile=", "resfile=", "nodevecs="])
    
        for opt, arg in opts:
            if opt == '-h':
                print ('NodeSequenceRegression.py -i <trainfile> -o <testfile> -a -o <resfile> -n <nodevecs>')
                sys.exit()
            elif opt in ("-i", "--trainfile"):
                TRAIN_FILE = arg
            elif opt in ("-e", "--testfile"):
                TEST_FILE = arg
            elif opt in ("-a", "--appendnumeric"):
                TO_ADD_VALUE = 1
            elif opt in ("-n", "--nodevecs"):
                EMB_FILE = arg
            elif opt in ("-o", "--resfile"):
                RES_FILE = arg
                
    except getopt.GetoptError:
        print ('usage: NodeSequenceRegression.py -i <trainfile> -o <testfile> -a -o <resfile> -n <nodevecs>')
            
    print ("Training file: %s" % (TRAIN_FILE))
    print ("Test file: %s" % (TEST_FILE))
    print ("Append numbers: %d" % (TO_ADD_VALUE))
    print ("Res file: %s" % (RES_FILE))
    print ("Emb file: %s" % (EMB_FILE))

    inpH = InputHelper()
    inpH.convertWordsToIds(EMB_FILE)
    
    VAL_ADDED_EMBFILE = EMB_FILE
    embfileDir = os.path.dirname(os.path.realpath(EMB_FILE))

    if (TO_ADD_VALUE == 1):
        VAL_ADDED_EMBFILE = embfileDir + '/ndvecswithvals.vec' 
        add_value_feature(EMB_FILE, VAL_ADDED_EMBFILE)
    
    x_train, y_train, x_test, y_test = loadData(inpH, VAL_ADDED_EMBFILE, TRAIN_FILE, TEST_FILE)
    
    BATCH_SIZE = int(len(x_train)/20) # 5% of the training set size
    x_train = pad_sequences(x_train, padding='post', maxlen=MAXLEN)
    x_test = pad_sequences(x_test, padding='post', maxlen=MAXLEN)
    
    model = trainRegression(inpH, x_train, y_train, x_test, y_test, BATCH_SIZE)
    testRegression(model, x_test, y_test, RES_FILE)


# In[ ]:


if __name__ == "__main__":
    main(sys.argv[1:])
    #main('-i ../sentences/train.tsv -e ../sentences/test.tsv -a -o predictions.txt -n ../graphs/nodevecs/refVecs.vec')


# In[ ]:





#!/usr/bin/env python
# coding: utf-8

# ## Outcome value prediction from Behaviour Change Data
# 
# In this notebook, we set up a regression/classification pipeline to predict either the outcome value
# directly or an interval in which this value may fall into, the former being `regression` and the latter being `multi-class classification`.
# 
# Before getting started with this notebook, let's have a look at the format of the data files which this program expects. The data is specified as tsv (tab separated values) and is generated by running the following command from the HBCP project root directory.
# 
# ```
# mvn exec:java@svmreg -Dexec.args="true"
# ```
# The above command generates the train and the test files located at the directory `prediction/sentences/`.
# Each line of the data files (train and test) looks like the following:
# ```
# C:5579097:35.7 C:5594106:16.1 I:3674268:1 C:5579728:30.6 I:3674248:1 C:5579118:22 C:5579689:14.6 C:5579088:44.5 C:5579711:80.6 C:5580203:29.4 O:4087178:abstinence C:5594105:19.3 O:4087186:cotinine C:5580200:35.7 O:4087187:2 C:5579096:58.8 I:3675703:1 C:5580204:5.9 I:3675698:1 C:5579663:22 C:5579083:29.4 O:4087191:6 I:3673288:1 O:4087172:1 I:3674264:1 I:3675717:1 C:5580216:0 	2.8
# ```
# 
# Each token represents a `:`-separated `<attribute-type>:<attribute-id>:<value>` combination, where the attribute type is one of `{C, I, O}` (contextual, intervention or outcome qualifier) feature, an attribute-id is a unique integer and a value is the textual representation of an instance of this feature.   
# 
# 

# In[4]:


#All neccessary imports and global variables
import sys, getopt

from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import OneHotEncoder
import numpy as np

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.text import text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences
from keras import backend as K
from numpy.random import seed
#from tensorflow import set_random_seed

# import the necessary packages
from keras.models import Sequential
from keras import layers

seed(110781)

import tensorflow as tf
#tf.random.set_seed(110781)

import random
import re
import os

PATTERN = re.compile("(?<![0-9])-?[0-9]*\.?[0-9]+")


# In[ ]:


def rmse(y_true, y_pred):
        return K.sqrt(K.mean(K.square(y_pred - y_true))) 


# In[ ]:


class InputHelper(object):
    emb_dim = 0
    pre_emb = dict() # word--> vec
    vocab_size = 0
    tokenizer = None
    embedding_matrix = None
    
    def cleanText(self, s):
        s = re.sub(r"[^\x00-\x7F]+"," ", s)
        s = re.sub(r'[\~\!\`\^\*\{\}\[\]\#\<\>\?\+\=\-\_\(\)]+',"",s)
        s = re.sub(r'( [0-9,\.]+)',r"\1 ", s)
        s = re.sub(r'\$'," $ ", s)
        s = re.sub('[ ]+',' ', s)
        return s.lower()

    #the tokenizer needs to be trained on the pre-trained node vectors
    #join the names of the nodes in a string so that tokenizer could be fit on it
    def getAllNodes(self, emb_path):
        print("Collecting node names...")
        line_count = 0        
        node_names = []
        for line in open(emb_path):
            l = line.strip().split()
            if (line_count > 0):
                node_names.append(l[0])
            
            line_count = line_count + 1
        
        self.vocab_size = line_count # includes the +1
        print("Collected node names...")
        return node_names

    # call convertWordsToIds first followed by loadW2V
    def convertWordsToIds(self, emb_path):
        allNodeNames = self.getAllNodes(emb_path)
        
        print ("Converting words to ids...")
        # Map words to ids        
        self.tokenizer = Tokenizer(num_words=self.vocab_size, filters=[], lower=False, split=" ")
        self.tokenizer.fit_on_texts(allNodeNames)
        print ("Finished converting words to ids...")
    
    # Assumes that the tokenizer already has been fit on some text (for the time being the node vec names)
    def loadW2V(self, emb_path):
        print("Loading W2V data...")
        line_count = 0        
        
        for line in open(emb_path):
            l = line.strip().split()
            if (line_count == 0):
                self.emb_dim = int(l[1])
                self.embedding_matrix = np.zeros((self.vocab_size, self.emb_dim))
            else:
                st = l[0]
                self.pre_emb[st] = np.asarray(l[1:]) # rest goes as the vector components
                if st in self.tokenizer.word_index:
                    idx = self.tokenizer.word_index[st]
                    self.embedding_matrix[idx] = np.array(l[1:], dtype=np.float32)[:self.emb_dim]
                else:
                    print ("Word '{}' not found in vocabulary..".format(st))
                
            line_count = line_count + 1
            
        print("loaded word2vec for {} nodes".format(len(self.pre_emb)))
    
    # Load the data as two matrices - X and Y
    def getTsvData(self, filepath):
        print("Loading data from " + filepath)
        x = []
        y = []
        
        # positive samples from file
        for line in open(filepath):
            l = line.strip().split("\t")            
            y.append(l[1])
            words = l[0].split(" ")
            x.append(words)
            #for w in words:
            #    x.append(w)
            
        return np.asarray(x), np.asarray(y)
    
    # Build sequences from each data instance
    def getSequenceData(self, tsvDataPath, numClasses=0, seed=123456):
        x_text, y = self.getTsvData(tsvDataPath)
        
        # Convert each sentence (node name sequence) to a sequence of integer ids
        x = self.tokenizer.texts_to_sequences(x_text)
        #print (x)
        
        if (numClasses > 0):
            y = self.categorizeOutputs(y, numClasses)
        
        return x, y
    
    def categorizeOutputs(self, y, numClasses):
        # allowable integers: 0,1...nclasses-1 (nclasses values)
        scaler = MinMaxScaler(feature_range=(0, numClasses-1))
        y_scaled = scaler.fit_transform(y.reshape(-1, 1))
        y_scaled = np.trunc(y_scaled)
        
        return y_scaled


# In[ ]:


def add_value_feature(in_fn, out_fn):
    """Read (text) file of (dense) vectors and add a final value to the vector for the actual value of the node.
    This allows to represent the distance between nodes. Normalize this feature between -1 and 1.
    For numerical -1 is the minimum value in the range, 1 the max.
    For BCT, +1 is the presence, -1 is the absence
    For categorical we create ranges bet. -1 and 1.
    For other, pick some random number close to 0.
    """

    print ("Writing appended vec file at %s" %(out_fn))
    random.seed(123)
    att_values, type_att = collect_attribute_value_maps(in_fn)
    numeric_atts = infer_numerical_attributes(att_values, type_att)

    att_maxes, att_mins = get_att_max_min(att_values, numeric_atts)  # max/min used for normalization

    # debug -- print maxes and mins
    print("There are %d numeric attributes." % len(numeric_atts))
    for num_att_id in numeric_atts:
        print("Numeric att: %s -- Min: %f ; Max: %f" % (num_att_id, att_mins[num_att_id], att_maxes[num_att_id]))

    # go through the file again and add 'normalized' values
    with open(in_fn) as f:
        with open(out_fn, 'w') as f_out:
            for line in f:
                cols = line.split()
                if len(cols) == 2:  # first line
                    f_out.write(line)
                    continue
                prefix, att_id, val = cols[0].split(':', 2)
                # BCTs stay the same
                if prefix == 'I':
                    norm_val = val
                # numerical attributes get normalized
                elif att_id in numeric_atts:
                    match = PATTERN.search(val)
                    if match is not None:
                        num = float(match.group(0))
                        # max-min normalization
                        if att_maxes[att_id] == att_mins[att_id]:
                            norm_val = "1"
                        else:
                            norm_num = 2 * ((num - att_mins[att_id]) / (att_maxes[att_id] - att_mins[att_id])) - 1
                            norm_val = str(norm_num)
                    else:
                        norm_val = "%f" % random.gauss(0, 0.001)
                # TODO not keeping track of categorical attributes yet
                # remaining attributes will get a random value close to zero (not sure what else to do with them)
                else:
                    norm_val = "%f" % random.gauss(0, 0.001)
                # f_out.write(cols[0] + '\t' + norm_val + '\n')
                f_out.write('{0} {1}\n'.format(line.strip(), norm_val))


def get_att_max_min(att_values, numeric_atts):
    # normalize numeric attributes
    att_maxes = {}
    att_mins = {}
    for num_att_id in numeric_atts:
        # get max and min
        nums = []
        for val in att_values[num_att_id]:
            # does val have a number
            match = PATTERN.search(val)
            if match is not None:
                num = float(match.group(0))
                nums.append(num)
        att_mins[num_att_id] = min(nums)
        att_maxes[num_att_id] = max(nums)
    return att_maxes, att_mins


def infer_numerical_attributes(att_values, type_att):
    # check if attribute is numerical (use logic from Martin's Java code)
    numeric_atts = []
    for att_id, vals in att_values.items():
        if att_id in type_att['I']:
            continue  # interventions will all be '1'
        num_val = 0
        for val in vals:
            # does val have a number
            match = PATTERN.search(val)
            if match is not None:
                num_val += 1
        # if 80% or more have numbers, then consider numeric
        if num_val / len(vals) >= 0.8:
            numeric_atts.append(att_id)
    return numeric_atts


def collect_attribute_value_maps(fn):
    att_values = {}
    type_att = {'C': set(), 'I': set(), 'O': set(), 'V': set()}
    with open(fn) as f:
        for line in f:
            cols = line.split()
            if len(cols) == 2:  # first line, skip
                continue
            prefix, att_id, val = cols[0].split(':', 2)
            type_att[prefix].add(att_id)
            if att_id in att_values:
                att_values[att_id].append(val)
            else:
                att_values[att_id] = [val]
    print("There are %d attributes." % len(att_values.keys()))
    print("There are %d interventions." % len(type_att['I']))
    return att_values, type_att


# Set the global parameters. Set `NUM_CLASSES` to `0` if you want to run the regression flow, otherwise set this to the number of classes (outcome value ranges).
# 
# The function `convertWordsToIds` converts each word (e.g. `C:5579097:35.7`) into an id. 

# In[ ]:


#Load datasets and embedded vectors
def loadData(inpH, emb_file, train_file, test_file, numClasses=0):
    
    #Load the training and the test sets
    #Load the text as a sequence of inputs
    
    x_train, y_train = inpH.getSequenceData(train_file, numClasses)
    x_test, y_test = inpH.getSequenceData(test_file, numClasses)

    #First few sequences
    for i in range(4):
        print ("x[{}][:5]..., y[{}] = {}, {}".format(i, i, x_train[i][:5], y_train[i]))    
        
    if (numClasses > 0):    
        encoder = OneHotEncoder(sparse=False, categories='auto')
        y_all = np.vstack((y_train, y_test))

        encoder.fit(y_all)

        y_train = encoder.transform(y_train)
        y_test = encoder.transform(y_test)
        for i in range(2):
            print ("y_train[{}] = {}".format(i, y_train[i]))
            print ("y_test[{}] = {}".format(i, y_test[i]))
            
    #Load the word vectors
    inpH.loadW2V(emb_file)
    
    #Print the loaded words
    nwords=0
    for w in inpH.pre_emb:
        print ("Dimension of vectors: {}".format(inpH.pre_emb[w].shape))
        print ("{} {}".format(w, inpH.pre_emb[w][0:5]))
        nwords = nwords+1
        if (nwords >= 2): break

    print ("vocab size: {}".format(inpH.vocab_size))
    print ("emb-matrix: {}...".format(inpH.embedding_matrix[1][:5]))
    print (inpH.embedding_matrix.shape)
    
    return x_train, y_train, x_test, y_test


# In[ ]:


def trainRegression(inpH, x_train, y_train, x_test, y_test, batch_size, hidden_layer_dim=20, num_classes=0, epochs=30, maxlen=300):
    if (num_classes > 0):
        loss_fn = 'categorical_crossentropy'
        eval_metrics = ['accuracy']
        activation_fn = 'softmax'
        output_dim = NUM_CLASSES
    else:
        loss_fn = rmse
        eval_metrics = [rmse]
        activation_fn = 'linear'
        output_dim = 1
        
    model = Sequential()
    model.add(layers.Embedding(input_dim=inpH.vocab_size, 
                               output_dim=inpH.emb_dim, 
                               input_length=maxlen,
                               weights=[inpH.embedding_matrix],
                               trainable=False))
    model.add(layers.Flatten())
    model.add(layers.Dense(20, activation='relu'))
    model.add(layers.Dense(output_dim, activation=activation_fn, name='output_vals'))
    model.compile(optimizer='adam',
                  loss = loss_fn,
                  metrics=eval_metrics)
    model.summary()
    
    print ("Training model...")
    history = model.fit(x_train, y_train,
        epochs=epochs,
        verbose=True,
        #validation_data=(x_test, y_test),
        batch_size=batch_size)
    
    loss, accuracy = model.evaluate(x_test, y_test, verbose=True)
    if (num_classes > 0):
        print("Cross-entropy loss: {:.4f}, Accuracy: {:.4f}".format(loss, accuracy))
    else:
        print("Loss: {:.4f}, RMSE: {:.4f}".format(loss, accuracy))    
        
    return model


# In[ ]:


def testRegression(model, x_test, y_test, res_file, num_classes=0):
    #Print each prediction    
    y_preds = model.predict(x_test)
    
    f = open(res_file, "w")
    for i in range(y_preds.shape[0]):
        if (num_classes > 0):
            f.write ("{:d}\t{:d}\n".format(np.argmax(y_test[i]), np.argmax(y_preds[i])))
        else:
            f.write ("{}\t{:.4f}\n".format(y_test[i], y_preds[i][0]))

    f.close()    


# In[ ]:


def main(argv):
    NUM_CLASSES = 0 # set this to 0 for regression and a positive value for classification
    TRAIN_FILE = "../sentences/train.tsv"
    TEST_FILE = "../sentences/test.tsv"
    TO_ADD_VALUE = 0
    RES_FILE = "predictions.txt"
    EMB_FILE = "../graphs/nodevecs/refVecs.vec"
    MAXLEN = 300
    
    try:
        opts, args = getopt.getopt(argv,"ha:i:e:o:n:", ["appendnumeric=", "trainfile=", "testfile=", "resfile=", "nodevecs="])
    
        for opt, arg in opts:
            if opt == '-h':
                print ('NodeSequenceRegression.py -i <trainfile> -o <testfile> -a -o <resfile> -n <nodevecs>')
                sys.exit()
            elif opt in ("-i", "--trainfile"):
                TRAIN_FILE = arg
            elif opt in ("-e", "--testfile"):
                TEST_FILE = arg
            elif opt in ("-a", "--appendnumeric"):
                TO_ADD_VALUE = 1
            elif opt in ("-n", "--nodevecs"):
                EMB_FILE = arg
            elif opt in ("-o", "--resfile"):
                RES_FILE = arg
                
    except getopt.GetoptError:
        print ('usage: NodeSequenceRegression.py -i <trainfile> -o <testfile> -a -o <resfile> -n <nodevecs>')
            
    print ("Training file: %s" % (TRAIN_FILE))
    print ("Test file: %s" % (TEST_FILE))
    print ("Append numbers: %d" % (TO_ADD_VALUE))
    print ("Res file: %s" % (RES_FILE))
    print ("Emb file: %s" % (EMB_FILE))

    inpH = InputHelper()
    inpH.convertWordsToIds(EMB_FILE)
    
    VAL_ADDED_EMBFILE = EMB_FILE
    embfileDir = os.path.dirname(os.path.realpath(EMB_FILE))

    if (TO_ADD_VALUE == 1):
        VAL_ADDED_EMBFILE = embfileDir + '/ndvecswithvals.vec' 
        add_value_feature(EMB_FILE, VAL_ADDED_EMBFILE)
    
    x_train, y_train, x_test, y_test = loadData(inpH, VAL_ADDED_EMBFILE, TRAIN_FILE, TEST_FILE)
    
    BATCH_SIZE = int(len(x_train)/20) # 5% of the training set size
    x_train = pad_sequences(x_train, padding='post', maxlen=MAXLEN)
    x_test = pad_sequences(x_test, padding='post', maxlen=MAXLEN)
    
    model = trainRegression(inpH, x_train, y_train, x_test, y_test, BATCH_SIZE)
    testRegression(model, x_test, y_test, RES_FILE)


# In[ ]:


if __name__ == "__main__":
    main(sys.argv[1:])
    #main('-i ../sentences/train.tsv -e ../sentences/test.tsv -a -o predictions.txt -n ../graphs/nodevecs/refVecs.vec')


# In[ ]:
#!/usr/bin/env python # coding: utf-8 # ## Outcome value prediction from Behaviour Change Data # # In this notebook, we set up a regression/classification pipeline to predict either the outcome value # directly or an interval in which this value may fall into, the former being `regression` and the latter being `multi-class classification`. # # Before getting started with this notebook, let's have a look at the format of the data files which this program expects. The data is specified as tsv (tab separated values) and is generated by running the following command from the HBCP project root directory. # # ``` # mvn exec:java@svmreg -Dexec.args="true" # ``` # The above command generates the train and the test files located at the directory `prediction/sentences/`. # Each line of the data files (train and test) looks like the following: # ``` # C:5579097:35.7 C:5594106:16.1 I:3674268:1 C:5579728:30.6 I:3674248:1 C:5579118:22 C:5579689:14.6 C:5579088:44.5 C:5579711:80.6 C:5580203:29.4 O:4087178:abstinence C:5594105:19.3 O:4087186:cotinine C:5580200:35.7 O:4087187:2 C:5579096:58.8 I:3675703:1 C:5580204:5.9 I:3675698:1 C:5579663:22 C:5579083:29.4 O:4087191:6 I:3673288:1 O:4087172:1 I:3674264:1 I:3675717:1 C:5580216:0 	2.8 # ``` # # Each token represents a `:`-separated `<attribute-type>:<attribute-id>:<value>` combination, where the attribute type is one of `{C, I, O}` (contextual, intervention or outcome qualifier) feature, an attribute-id is a unique integer and a value is the textual representation of an instance of this feature. # # # In[4]: #All neccessary imports and global variables #from tensorflow import set_random_seed # import the necessary packages #tf.random.set_seed(110781) # In[ ]: # In[ ]: # word--> vec #\<\>\?\+\=\-\_\(\)]+',"",s) #the tokenizer needs to be trained on the pre-trained node vectors #join the names of the nodes in a string so that tokenizer could be fit on it # includes the +1 # call convertWordsToIds first followed by loadW2V # Map words to ids # Assumes that the tokenizer already has been fit on some text (for the time being the node vec names) # rest goes as the vector components # Load the data as two matrices - X and Y # positive samples from file #for w in words: #    x.append(w) # Build sequences from each data instance # Convert each sentence (node name sequence) to a sequence of integer ids #print (x) # allowable integers: 0,1...nclasses-1 (nclasses values) # In[ ]: Read (text) file of (dense) vectors and add a final value to the vector for the actual value of the node.     This allows to represent the distance between nodes. Normalize this feature between -1 and 1.     For numerical -1 is the minimum value in the range, 1 the max.     For BCT, +1 is the presence, -1 is the absence     For categorical we create ranges bet. -1 and 1.     For other, pick some random number close to 0. # max/min used for normalization # debug -- print maxes and mins # go through the file again and add 'normalized' values # first line # BCTs stay the same # numerical attributes get normalized # max-min normalization # TODO not keeping track of categorical attributes yet # remaining attributes will get a random value close to zero (not sure what else to do with them) # f_out.write(cols[0] + '\t' + norm_val + '\n') # normalize numeric attributes # get max and min # does val have a number # check if attribute is numerical (use logic from Martin's Java code) # interventions will all be '1' # does val have a number # if 80% or more have numbers, then consider numeric # first line, skip # Set the global parameters. Set `NUM_CLASSES` to `0` if you want to run the regression flow, otherwise set this to the number of classes (outcome value ranges). # # The function `convertWordsToIds` converts each word (e.g. `C:5579097:35.7`) into an id. # In[ ]: #Load datasets and embedded vectors #Load the training and the test sets #Load the text as a sequence of inputs #First few sequences #Load the word vectors #Print the loaded words # In[ ]: #validation_data=(x_test, y_test), # In[ ]: #Print each prediction # In[ ]: # set this to 0 for regression and a positive value for classification # 5% of the training set size # In[ ]: #main('-i ../sentences/train.tsv -e ../sentences/test.tsv -a -o predictions.txt -n ../graphs/nodevecs/refVecs.vec') # In[ ]: