<jupyter_start><jupyter_text>Rhetorical relations classification used in tree building: ESIMPrepare data and model-related scripts.Evaluate models.Make and evaluate ansembles for ESIM and BiMPM model / ESIM and feature-based model.Output: - ``models/relation_predictor_esim/*``<jupyter_code>%load_ext autoreload
%autoreload 2
import os
import glob
import pandas as pd
import numpy as np
import pickle
from utils.file_reading import read_edus, read_gold, read_negative, read_annotation<jupyter_output><empty_output><jupyter_text>Make a directory<jupyter_code>MODEL_PATH = 'models/label_predictor_esim'
! mkdir $MODEL_PATH

TRAIN_FILE_PATH = os.path.join(MODEL_PATH, 'nlabel_cf_train.tsv')
DEV_FILE_PATH = os.path.join(MODEL_PATH, 'nlabel_cf_dev.tsv')
TEST_FILE_PATH = os.path.join(MODEL_PATH, 'nlabel_cf_test.tsv')<jupyter_output>mkdir: cannot create directory ‘models/label_predictor_esim’: File exists<jupyter_text>Prepare train/test sets<jupyter_code>IN_PATH = 'data_labeling'

train_samples = pd.read_pickle(os.path.join(IN_PATH, 'train_samples.pkl'))
dev_samples = pd.read_pickle(os.path.join(IN_PATH, 'dev_samples.pkl'))
test_samples = pd.read_pickle(os.path.join(IN_PATH, 'test_samples.pkl'))
counts = train_samples['relation'].value_counts(normalize=False).values
NUMBER_CLASSES = len(counts)
print("number of classes:", NUMBER_CLASSES)
print("class weights:")
np.round(counts.min() / counts, decimals=6)
counts = train_samples['relation'].value_counts()
counts
import razdel

def tokenize(text):
    result = ' '.join([tok.text for tok in razdel.tokenize(text)])
    return result
    
train_samples['snippet_x'] = train_samples.snippet_x.map(tokenize)
train_samples['snippet_y'] = train_samples.snippet_y.map(tokenize)

dev_samples['snippet_x'] = dev_samples.snippet_x.map(tokenize)
dev_samples['snippet_y'] = dev_samples.snippet_y.map(tokenize)

test_samples['snippet_x'] = test_samples.snippet_x.map(tokenize)
test_samples['snippet_y'] = test_samples.snippet_y.map(tokenize)
train_samples = train_samples.reset_index()
train_samples[['relation', 'snippet_x', 'snippet_y', 'index']].to_csv(TRAIN_FILE_PATH, sep='\t', header=False, index=False)

dev_samples = dev_samples.reset_index()
dev_samples[['relation', 'snippet_x', 'snippet_y', 'index']].to_csv(DEV_FILE_PATH, sep='\t', header=False, index=False)

test_samples = test_samples.reset_index()
test_samples[['relation', 'snippet_x', 'snippet_y', 'index']].to_csv(TEST_FILE_PATH, sep='\t', header=False, index=False)<jupyter_output><empty_output><jupyter_text>Modify model (Add F1, concatenated encoding)<jupyter_code>%%writefile models/bimpm_custom_package/model/esim.py

from typing import Dict, List, Any, Optional

import numpy
import torch

from allennlp.common.checks import check_dimensions_match
from allennlp.data import TextFieldTensors, Vocabulary
from allennlp.models.model import Model
from allennlp.modules import FeedForward, InputVariationalDropout
from allennlp.modules.matrix_attention.matrix_attention import MatrixAttention
from allennlp.modules import Seq2SeqEncoder, TextFieldEmbedder
from allennlp.nn import InitializerApplicator
from allennlp.nn.util import (
    get_text_field_mask,
    masked_softmax,
    weighted_sum,
    masked_max,
)
from allennlp.training.metrics import CategoricalAccuracy, F1Measure


@Model.register("custom_esim")
class CustomESIM(Model):
    """
    This `Model` implements the ESIM sequence model described in [Enhanced LSTM for Natural Language Inference]
    (https://api.semanticscholar.org/CorpusID:34032948) by Chen et al., 2017.
    Registered as a `Model` with name "esim".
    # Parameters
    vocab : `Vocabulary`
    text_field_embedder : `TextFieldEmbedder`
        Used to embed the `premise` and `hypothesis` `TextFields` we get as input to the
        model.
    encoder : `Seq2SeqEncoder`
        Used to encode the premise and hypothesis.
    matrix_attention : `MatrixAttention`
        This is the attention function used when computing the similarity matrix between encoded
        words in the premise and words in the hypothesis.
    projection_feedforward : `FeedForward`
        The feedforward network used to project down the encoded and enhanced premise and hypothesis.
    inference_encoder : `Seq2SeqEncoder`
        Used to encode the projected premise and hypothesis for prediction.
    output_feedforward : `FeedForward`
        Used to prepare the concatenated premise and hypothesis for prediction.
    output_logit : `FeedForward`
        This feedforward network computes the output logits.
    dropout : `float`, optional (default=`0.5`)
        Dropout percentage to use.
    initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
        Used to initialize the model parameters.
    """

    def __init__(
        self,
        vocab: Vocabulary,
        text_field_embedder: TextFieldEmbedder,
        encoder: Seq2SeqEncoder,
        matrix_attention: MatrixAttention,
        projection_feedforward: FeedForward,
        inference_encoder: Seq2SeqEncoder,
        output_feedforward: FeedForward,
        output_logit: FeedForward,
        encode_together: bool = False,
        dropout: float = 0.5,
        class_weights: list = [],
        initializer: InitializerApplicator = InitializerApplicator(),
        **kwargs,
    ) -> None:
        super().__init__(vocab, **kwargs)

        self._text_field_embedder = text_field_embedder
        self._encoder = encoder

        self._matrix_attention = matrix_attention
        self._projection_feedforward = projection_feedforward

        self._inference_encoder = inference_encoder

        if dropout:
            self.dropout = torch.nn.Dropout(dropout)
            self.rnn_input_dropout = InputVariationalDropout(dropout)
        else:
            self.dropout = None
            self.rnn_input_dropout = None

        self._output_feedforward = output_feedforward
        self._output_logit = output_logit
        self.encode_together = encode_together

        self._num_labels = vocab.get_vocab_size(namespace="labels")

        check_dimensions_match(
            text_field_embedder.get_output_dim(),
            encoder.get_input_dim(),
            "text field embedding dim",
            "encoder input dim",
        )
        check_dimensions_match(
            encoder.get_output_dim() * 4,
            projection_feedforward.get_input_dim(),
            "encoder output dim",
            "projection feedforward input",
        )
        check_dimensions_match(
            projection_feedforward.get_output_dim(),
            inference_encoder.get_input_dim(),
            "proj feedforward output dim",
            "inference lstm input dim",
        )

        self.metrics = {"accuracy": CategoricalAccuracy()}
        
        if class_weights:
            self.class_weights = class_weights
        else:
            self.class_weights = [1.] * self.classifier_feedforward.get_output_dim()
            
        for _class in range(len(self.class_weights)):
            self.metrics.update({
                f"f1_rel{_class}": F1Measure(_class),
            })
        
        self._loss = torch.nn.CrossEntropyLoss(weight=torch.FloatTensor(self.class_weights))

        initializer(self)

    def forward(  # type: ignore
        self,
        premise: TextFieldTensors,
        hypothesis: TextFieldTensors,
        label: torch.IntTensor = None,
        metadata: List[Dict[str, Any]] = None,
    ) -> Dict[str, torch.Tensor]:

        """
        # Parameters
        premise : `TextFieldTensors`
            From a `TextField`
        hypothesis : `TextFieldTensors`
            From a `TextField`
        label : `torch.IntTensor`, optional (default = `None`)
            From a `LabelField`
        metadata : `List[Dict[str, Any]]`, optional (default = `None`)
            Metadata containing the original tokenization of the premise and
            hypothesis with 'premise_tokens' and 'hypothesis_tokens' keys respectively.
        # Returns
        An output dictionary consisting of:
        label_logits : `torch.FloatTensor`
            A tensor of shape `(batch_size, num_labels)` representing unnormalised log
            probabilities of the entailment label.
        label_probs : `torch.FloatTensor`
            A tensor of shape `(batch_size, num_labels)` representing probabilities of the
            entailment label.
        loss : `torch.FloatTensor`, optional
            A scalar loss to be optimised.
        """
        embedded_premise = self._text_field_embedder(premise)
        embedded_hypothesis = self._text_field_embedder(hypothesis)
        premise_mask = get_text_field_mask(premise)
        hypothesis_mask = get_text_field_mask(hypothesis)

        # apply dropout for LSTM
        if self.rnn_input_dropout:
            embedded_premise = self.rnn_input_dropout(embedded_premise)
            embedded_hypothesis = self.rnn_input_dropout(embedded_hypothesis)

        # encode premise and hypothesis
        encoded_premise = self._encoder(embedded_premise, premise_mask)
        encoded_hypothesis = self._encoder(embedded_hypothesis, hypothesis_mask)

        # Shape: (batch_size, premise_length, hypothesis_length)
        similarity_matrix = self._matrix_attention(encoded_premise, encoded_hypothesis)

        # Shape: (batch_size, premise_length, hypothesis_length)
        p2h_attention = masked_softmax(similarity_matrix, hypothesis_mask)
        # Shape: (batch_size, premise_length, embedding_dim)
        attended_hypothesis = weighted_sum(encoded_hypothesis, p2h_attention)

        # Shape: (batch_size, hypothesis_length, premise_length)
        h2p_attention = masked_softmax(similarity_matrix.transpose(1, 2).contiguous(), premise_mask)
        # Shape: (batch_size, hypothesis_length, embedding_dim)
        attended_premise = weighted_sum(encoded_premise, h2p_attention)

        # the "enhancement" layer
        premise_enhanced = torch.cat(
            [
                encoded_premise,
                attended_hypothesis,
                encoded_premise - attended_hypothesis,
                encoded_premise * attended_hypothesis,
            ],
            dim=-1,
        )
        hypothesis_enhanced = torch.cat(
            [
                encoded_hypothesis,
                attended_premise,
                encoded_hypothesis - attended_premise,
                encoded_hypothesis * attended_premise,
            ],
            dim=-1,
        )

        # The projection layer down to the model dimension.  Dropout is not applied before
        # projection.
        projected_enhanced_premise = self._projection_feedforward(premise_enhanced)
        projected_enhanced_hypothesis = self._projection_feedforward(hypothesis_enhanced)

        # Run the inference layer
        if self.rnn_input_dropout:
            projected_enhanced_premise = self.rnn_input_dropout(projected_enhanced_premise)
            projected_enhanced_hypothesis = self.rnn_input_dropout(projected_enhanced_hypothesis)
        v_ai = self._inference_encoder(projected_enhanced_premise, premise_mask)
        v_bi = self._inference_encoder(projected_enhanced_hypothesis, hypothesis_mask)

        # The pooling layer -- max and avg pooling.
        # (batch_size, model_dim)
        v_a_max = masked_max(v_ai, premise_mask.unsqueeze(-1), dim=1)
        v_b_max = masked_max(v_bi, hypothesis_mask.unsqueeze(-1), dim=1)

        v_a_avg = torch.sum(v_ai * premise_mask.unsqueeze(-1), dim=1) / torch.sum(
            premise_mask, 1, keepdim=True
        )
        v_b_avg = torch.sum(v_bi * hypothesis_mask.unsqueeze(-1), dim=1) / torch.sum(
            hypothesis_mask, 1, keepdim=True
        )

        # Now concat
        # (batch_size, model_dim * 2 * 4)
        v_all = torch.cat([v_a_avg, v_a_max, v_b_avg, v_b_max], dim=1)

        # the final MLP -- apply dropout to input, and MLP applies to output & hidden
        if self.dropout:
            v_all = self.dropout(v_all)

        output_hidden = self._output_feedforward(v_all)
        label_logits = self._output_logit(output_hidden)
        label_probs = torch.nn.functional.softmax(label_logits, dim=-1)

        output_dict = {"label_logits": label_logits, "label_probs": label_probs}

        if label is not None:
            loss = self._loss(label_logits, label.long().view(-1))
            output_dict["loss"] = loss
            
            for metric in self.metrics.values():
                metric(label_logits, label.long().view(-1))

        return output_dict

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        metrics = {"accuracy": self.metrics["accuracy"].get_metric(reset=reset)}
        
        for _class in range(len(self.class_weights)):
            metrics.update({
                f"f1_rel{_class}": self.metrics[f"f1_rel{_class}"].get_metric(reset=reset)['f1'],
            })
        
        metrics["f1_macro"] = numpy.mean([metrics[f"f1_rel{_class}"] for _class in range(len(self.class_weights))])
        return metrics

    default_predictor = "textual_entailment"
! cp models/bimpm_custom_package/model/esim.py ../../../maintenance_rst/models/customization_package/model/esim.py<jupyter_output><empty_output><jupyter_text>2. Generate config files  ELMo<jupyter_code>%%writefile $MODEL_PATH/config_elmo.json

local NUM_EPOCHS = 200;
local LR = 1e-3;
local LSTM_ENCODER_HIDDEN = 25;

{
  "dataset_reader": {
    "type": "quora_paraphrase",
    "tokenizer": {
      "type": "just_spaces"
    },
    "token_indexers": {
      "token_characters": {
        "type": "characters",
        "min_padding_length": 30,
      },
      "elmo": {
        "type": "elmo_characters"
     }
    }
  },
  "train_data_path": "label_predictor_esim/nlabel_cf_train.tsv",
  "validation_data_path": "label_predictor_esim/nlabel_cf_dev.tsv",
  "test_data_path": "label_predictor_esim/nlabel_cf_test.tsv",
  "model": {
    "type": "custom_esim",
    "dropout": 0.5,
    "class_weights": [
        0.027483, 0.032003, 0.080478, 0.102642, 0.121394, 0.135027,
        0.136856, 0.170897, 0.172355, 0.181655, 0.193858, 0.211297,
        0.231651, 0.260982, 0.334437, 0.378277, 0.392996, 0.567416,
        0.782946, 0.855932, 0.971154, 1.0],
    "encode_together": false,
    "text_field_embedder": {
        "token_embedders": {
            "elmo": {
                    "type": "elmo_token_embedder",
                    "options_file": "rsv_elmo/options.json",
                    "weight_file": "rsv_elmo/model.hdf5",
                    "do_layer_norm": false,
                    "dropout": 0.1
            },
            "token_characters": {
                "type": "character_encoding",
                "dropout": 0.1,
                "embedding": {
                    "embedding_dim": 20,
                    "padding_index": 0,
                    "vocab_namespace": "token_characters"
                },
                "encoder": {
                    "type": "lstm",
                    "input_size": $.model.text_field_embedder.token_embedders.token_characters.embedding.embedding_dim,
                    "hidden_size": LSTM_ENCODER_HIDDEN,
                    "num_layers": 1,
                    "bidirectional": true,
                    "dropout": 0.4
                },
            },
      }
    },
    "encoder": {
      "type": "lstm",
      "input_size": 1024+LSTM_ENCODER_HIDDEN+LSTM_ENCODER_HIDDEN,
      "hidden_size": 300,
      "num_layers": 1,
      "bidirectional": true
    },
    "matrix_attention": {"type": "dot_product"},
    "projection_feedforward": {
      "input_dim": 2400,
      "hidden_dims": 300,
      "num_layers": 1,
      "activations": "relu"
    },
    "inference_encoder": {
      "type": "lstm",
      "input_size": 300,
      "hidden_size": 300,
      "num_layers": 1,
      "bidirectional": true
    },
    "output_feedforward": {
      "input_dim": 2400,
      "num_layers": 1,
      "hidden_dims": 300,
      "activations": "relu",
      "dropout": 0.5
    },
    "output_logit": {
      "input_dim": 300,
      "num_layers": 1,
      "hidden_dims": 22,
      "activations": "linear"
    },
    "initializer": {
      "regexes": [
        [".*linear_layers.*weight", {"type": "xavier_normal"}],
        [".*linear_layers.*bias", {"type": "constant", "val": 0}],
        [".*weight_ih.*", {"type": "xavier_normal"}],
        [".*weight_hh.*", {"type": "orthogonal"}],
        [".*bias.*", {"type": "constant", "val": 0}],
        [".*matcher.*match_weights.*", {"type": "kaiming_normal"}]
      ]
    }
   },
  "data_loader": {
    "batch_sampler": {
        "type": "bucket",
        "batch_size": 20,
        "padding_noise": 0.0,
        "sorting_keys": ["premise"],
    },
  },
  "trainer": {
    "num_epochs": NUM_EPOCHS,
    "cuda_device": 1,
    "grad_clipping": 5.0,
    "validation_metric": "+f1_macro",
    "shuffle": true,
    "optimizer": {
      "type": "adam",
      "lr": LR
    },
    "learning_rate_scheduler": {
      "type": "reduce_on_plateau",
      "factor": 0.5,
      "mode": "max",
      "patience": 0
    }
  }
}
! cp -r $MODEL_PATH ../../../maintenance_rst/models/label_predictor_esim
! cp -r $MODEL_PATH/config_elmo.json ../../../maintenance_rst/models/label_predictor_esim/<jupyter_output><empty_output><jupyter_text>3. Scripts for training/prediction   Option 1. Directly from the config Train a model<jupyter_code>%%writefile models/train_label_predictor_esim.sh
# usage:
# $ cd models 
# $ sh train_label_predictor.sh {bert|elmo} result_30

export METHOD=${1}
export RESULT_DIR=${2}
export DEV_FILE_PATH="nlabel_cf_dev.tsv"
export TEST_FILE_PATH="nlabel_cf_test.tsv"

rm -r label_predictor_esim/${RESULT_DIR}/
allennlp train -s label_predictor_esim/${RESULT_DIR}/ label_predictor_esim/config_${METHOD}.json \
    --include-package bimpm_custom_package
allennlp predict --use-dataset-reader --silent \
    --output-file label_predictor_esim/${RESULT_DIR}/predictions_dev.json label_predictor_esim/${RESULT_DIR}/model.tar.gz label_predictor_esim/${DEV_FILE_PATH} \
    --include-package bimpm_custom_package \
    --predictor textual-entailment
allennlp predict --use-dataset-reader --silent \
    --output-file label_predictor_esim/${RESULT_DIR}/predictions_test.json label_predictor_esim/${RESULT_DIR}/model.tar.gz label_predictor_esim/${TEST_FILE_PATH} \
    --include-package bimpm_custom_package \
    --predictor textual-entailment
! cp models/train_label_predictor_esim.sh ../../../maintenance_rst/models/<jupyter_output><empty_output><jupyter_text>Predict on dev&test<jupyter_code>%%writefile models/eval_label_predictor_esim.sh
# usage:
# $ cd models 
# $ sh train_label_predictor.sh {bert|elmo} result_30

export METHOD=${1}
export RESULT_DIR=${2}
export DEV_FILE_PATH="nlabel_cf_dev.tsv"
export TEST_FILE_PATH="nlabel_cf_test.tsv"

allennlp predict --use-dataset-reader --silent \
    --output-file label_predictor_esim/${RESULT_DIR}/predictions_dev.json label_predictor_esim/${RESULT_DIR}/model.tar.gz label_predictor_esim/${DEV_FILE_PATH} \
    --include-package bimpm_custom_package \
    --predictor textual-entailment
allennlp predict --use-dataset-reader --silent \
    --output-file label_predictor_esim/${RESULT_DIR}/predictions_test.json label_predictor_esim/${RESULT_DIR}/model.tar.gz label_predictor_esim/${TEST_FILE_PATH} \
    --include-package bimpm_custom_package \
    --predictor textual-entailment
! cp models/eval_label_predictor_esim.sh ../../../maintenance_rst/models/<jupyter_output><empty_output><jupyter_text>(optional) predict on train<jupyter_code>%%writefile models/eval_label_predictor_train.sh
# usage:
# $ cd models 
# $ sh eval_label_predictor_train.sh {bert|elmo} result_30

export METHOD=${1}
export RESULT_DIR=${2}
export TEST_FILE_PATH="nlabel_cf_train.tsv"

allennlp predict --use-dataset-reader --silent \
    --output-file label_predictor_bimpm/${RESULT_DIR}/predictions_train.json label_predictor_bimpm/${RESULT_DIR}/model.tar.gz label_predictor_bimpm/${TEST_FILE_PATH} \
    --include-package customization_package \
    --predictor textual-entailment<jupyter_output><empty_output><jupyter_text>Option 2. Using wandb for parameters adjustment<jupyter_code>%%writefile ../../../maintenance_rst/models/wandb_label_predictor_esim.yaml

name: label_predictor_esim
program: wandb_allennlp # this is a wrapper console script around allennlp commands. It is part of wandb-allennlp
method: bayes
## Do not for get to use the command keyword to specify the following command structure
command:
  - ${program} #omit the interpreter as we use allennlp train command directly
  - "--subcommand=train"
  - "--include-package=customization_package" # add all packages containing your registered classes here
  - "--config_file=label_predictor_esim/config_elmo.json"
  - ${args}
metric:
    name: best_f1_macro
    goal: maximize
parameters:
    model.encode_together:
        values: ["true", ]    
    iterator.batch_size:
        values: [8,]
    trainer.optimizer.lr:
        values: [0.001,]
    model.dropout:
        values: [0.5]<jupyter_output><empty_output><jupyter_text>3. Run training ``wandb sweep wandb_label_predictor_esim.yaml``(returns %sweepname1)``wandb sweep wandb_label_predictor2.yaml``(returns %sweepname2)``wandb agent --count 1 %sweepname1 && wandb agent --count 1 %sweepname2`` Move the best model in label_predictor_bimpm<jupyter_code>! ls -laht models/wandb
! cp -r models/wandb/run-20201218_123424-kcphaqhi/training_dumps models/label_predictor_esim/esim_elmo<jupyter_output><empty_output><jupyter_text>**Or** load from wandb by %sweepname<jupyter_code>import wandb
api = wandb.Api()
run = api.run("tchewik/tmp/7hum4oom")
for file in run.files():
    file.download(replace=True)
! cp -r training_dumps models/label_predictor_bimpm/toasty-sweep-1<jupyter_output><empty_output><jupyter_text>And run evaluation from shell``sh eval_label_predictor_esim.sh {elmo|elmo_fasttext} toasty-sweep-1``  4. Evaluate classifier<jupyter_code>def load_predictions(path):
    result = []
    vocab = []
    
    with open(path, 'r') as file:
        for line in file.readlines():
            line = json.loads(line)
            if line.get("label"):
                result.append(line.get("label"))
            elif line.get("label_probs"):
                if not vocab:
                    vocab = open(path[:path.rfind('/')] + '/vocabulary/labels.txt', 'r').readlines()
                    vocab = [label.strip() for label in vocab]
                
                result.append(vocab[np.argmax(line.get("label_probs"))])
            
    print('length of result:', len(result))
    return result
RESULT_DIR = 'esim_elmo'
! mkdir models/label_predictor_esim/$RESULT_DIR
! cp -r ../../../maintenance_rst/models/label_predictor_esim/$RESULT_DIR/*.json models/label_predictor_esim/$RESULT_DIR/<jupyter_output><empty_output><jupyter_text>On dev set<jupyter_code>import pandas as pd
import json

true = pd.read_csv(DEV_FILE_PATH, sep='\t', header=None)[0].values.tolist()
pred = load_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_dev.json')
from sklearn.metrics import classification_report

print(classification_report(true[:len(pred)], pred, digits=4))
test_metrics = classification_report(true[:len(pred)], pred, digits=4, output_dict=True)
test_f1 = np.array(
    [test_metrics[label].get('f1-score') for label in test_metrics if type(test_metrics[label]) == dict]) * 100

test_f1
len(true)
from sklearn.metrics import f1_score, precision_score, recall_score

print('f1: %.2f'%(f1_score(true[:len(pred)], pred, average='macro')*100))
print('pr: %.2f'%(precision_score(true[:len(pred)], pred, average='macro')*100))
print('re: %.2f'%(recall_score(true[:len(pred)], pred, average='macro')*100))
from utils.plot_confusion_matrix import plot_confusion_matrix
from sklearn.metrics import confusion_matrix

labels = list(set(true))
labels.sort()
plot_confusion_matrix(confusion_matrix(true[:len(pred)], pred, labels), target_names=labels, normalize=True)
top_classes = [
    'attribution_NS',
    'attribution_SN',
    'purpose_NS',
    'purpose_SN',
    'condition_SN',
    'contrast_NN',
    'condition_NS',
    'joint_NN',
    'concession_NS',
    'same-unit_NN',
    'elaboration_NS',
    'cause-effect_NS',
]

class_mapper = {weird_class: 'other' + weird_class[-3:] for weird_class in labels if not weird_class in top_classes}
import numpy as np

true = [class_mapper.get(value) if class_mapper.get(value) else value for value in true]
pred = [class_mapper.get(value) if class_mapper.get(value) else value for value in pred]

pred_mapper = {
    'other_NN': 'joint_NN',
    'other_NS': 'joint_NN',
    'other_SN': 'joint_NN'
}
pred = [pred_mapper.get(value) if pred_mapper.get(value) else value for value in pred]

_to_stay = (np.array(true) != 'other_NN') & (np.array(true) != 'other_SN') & (np.array(true) != 'other_NS')

_true = np.array(true)[_to_stay]
_pred = np.array(pred)[_to_stay[:len(pred)]]
labels = list(set(_true))
from sklearn.metrics import f1_score, precision_score, recall_score

print('f1: %.2f'%(f1_score(true[:len(pred)], pred, average='macro')*100))
print('pr: %.2f'%(precision_score(true[:len(pred)], pred, average='macro')*100))
print('re: %.2f'%(recall_score(true[:len(pred)], pred, average='macro')*100))
labels.sort()
plot_confusion_matrix(confusion_matrix(_true[:len(_pred)], _pred), target_names=labels, normalize=True)
import numpy as np

for rel in np.unique(_true):
    print(rel)<jupyter_output><empty_output><jupyter_text>On train set (optional)<jupyter_code>import pandas as pd
import json

true = pd.read_csv('models/label_predictor_bimpm/nlabel_cf_train.tsv', sep='\t', header=None)[0].values.tolist()
pred = load_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_train.json')

print(classification_report(true[:len(pred)], pred, digits=4))
file = 'models/label_predictor_lstm/nlabel_cf_train.tsv'
true_train = pd.read_csv(file, sep='\t', header=None)
true_train['predicted_relation'] = pred

print(true_train[true_train.relation != true_train.predicted_relation].shape)

true_train[true_train.relation != true_train.predicted_relation].to_csv('mispredicted_relations.csv', sep='\t')<jupyter_output><empty_output><jupyter_text>On test set<jupyter_code>import pandas as pd
import json

true = pd.read_csv(TEST_FILE_PATH, sep='\t', header=None)[0].values.tolist()
pred = load_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_test.json')

print(classification_report(true[:len(pred)], pred, digits=4))
test_metrics = classification_report(true[:len(pred)], pred, digits=4, output_dict=True)
test_f1 = np.array(
    [test_metrics[label].get('f1-score') for label in test_metrics if type(test_metrics[label]) == dict]) * 100

test_f1
from sklearn.metrics import f1_score, precision_score, recall_score

print('f1: %.2f'%(f1_score(true[:len(pred)], pred, average='macro')*100))
print('pr: %.2f'%(precision_score(true[:len(pred)], pred, average='macro')*100))
print('re: %.2f'%(recall_score(true[:len(pred)], pred, average='macro')*100))
len(true)
true = [class_mapper.get(value) if class_mapper.get(value) else value for value in true]
pred = [class_mapper.get(value) if class_mapper.get(value) else value for value in pred]
pred = [pred_mapper.get(value) if pred_mapper.get(value) else value for value in pred]

_to_stay = (np.array(true) != 'other_NN') & (np.array(true) != 'other_SN') & (np.array(true) != 'other_NS')

_true = np.array(true)[_to_stay]
_pred = np.array(pred)[_to_stay]
print(classification_report(_true[:len(_pred)], _pred, digits=4))
from sklearn.metrics import f1_score, precision_score, recall_score

print('f1: %.2f'%(f1_score(_true[:len(_pred)], _pred, average='macro')*100))
print('pr: %.2f'%(precision_score(_true[:len(_pred)], _pred, average='macro')*100))
print('re: %.2f'%(recall_score(_true[:len(_pred)], _pred, average='macro')*100))<jupyter_output><empty_output><jupyter_text>Ensemble: (Logreg+Catboost) + ESIM<jupyter_code>! ls models/label_predictor_esim
import json

model_vocab = open(MODEL_PATH + '/' + RESULT_DIR + '/vocabulary/labels.txt', 'r').readlines()
model_vocab = [label.strip() for label in model_vocab]

catboost_vocab = [
   'attribution_NS', 'attribution_SN', 'background_NS',
   'cause-effect_NS', 'cause-effect_SN', 'comparison_NN',
   'concession_NS', 'condition_NS', 'condition_SN', 'contrast_NN',
   'elaboration_NS', 'evidence_NS', 'interpretation-evaluation_NS',
   'interpretation-evaluation_SN', 'joint_NN', 'preparation_SN',
   'purpose_NS', 'purpose_SN', 'restatement_NN', 'same-unit_NN',
   'sequence_NN', 'solutionhood_SN']

def load_neural_predictions(path):
    result = []
    
    with open(path, 'r') as file:
        for line in file.readlines():
            line = json.loads(line)
            if line.get('probs'):
                probs = line.get('probs')
            elif line.get('label_probs'):
                probs = line.get('label_probs')
            probs = {model_vocab[i]: probs[i] for i in range(len(model_vocab))}
            result.append(probs)
            
    return result

def load_scikit_predictions(model, X):
    result = []
    predictions = model.predict_proba(X)
    
    for prediction in predictions:
        probs = {catboost_vocab[j]: prediction[j] for j in range(len(catboost_vocab))}
        result.append(probs)
    
    return result

def vote_predictions(predictions, soft=True, weights=[1., 1.]):
    for i in range(1, len(predictions)):
        assert len(predictions[i-1]) == len(predictions[i])
        
    if weights == [1., 1.]:
        weights = [1.,] * len(predictions)
    
    result = []
    
    for i in range(len(predictions[0])):
        sample_result = {}
        for key in predictions[0][i].keys():
            if soft:
                sample_result[key] = 0
                for j, prediction in enumerate(predictions):
                    sample_result[key] += prediction[i][key] * weights[j]
            else:
                sample_result[key] = max([pred[i][key] * weights[j] for j, pred in enumerate(predictions)])

        
        result.append(sample_result)
    
    return result

def probs_to_classes(pred):
    result = []
    
    for sample in pred:
        best_class = ''
        best_prob = 0.
        for key in sample.keys():
            if sample[key] > best_prob:
                best_prob = sample[key]
                best_class = key
        
        result.append(best_class)
    
    return result
! pip install catboost
import pickle

fs_catboost_plus_logreg = pickle.load(open('models/relation_predictor_baseline/model.pkl', 'rb'))
lab_encoder = pickle.load(open('models/relation_predictor_baseline/label_encoder.pkl', 'rb'))
scaler = pickle.load(open('models/relation_predictor_baseline/scaler.pkl', 'rb'))
drop_columns = pickle.load(open('models/relation_predictor_baseline/drop_columns.pkl', 'rb'))<jupyter_output>/opt/.pyenv/versions/3.7.4/lib/python3.7/site-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator Pipeline from version 0.22.2.post1 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.
  UserWarning)
/opt/.pyenv/versions/3.7.4/lib/python3.7/site-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator LabelEncoder from version 0.22.2.post1 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.
  UserWarning)
/opt/.pyenv/versions/3.7.4/lib/python3.7/site-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator VotingClassifier from version 0.22.2.post1 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.
  UserWarning)
/opt/.pyenv/versions/3.7.4/lib/python3.7/site-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator StandardScaler from version 0.22.2.post1 when using version [...]<jupyter_text>On dev set<jupyter_code>from sklearn import metrics


TARGET = 'relation'

y_dev, X_dev = dev_samples['relation'].to_frame(), dev_samples.drop('relation', axis=1).drop(
    columns=drop_columns + ['category_id', 'index'])

X_scaled_np = scaler.transform(X_dev)
X_dev = pd.DataFrame(X_scaled_np, index=X_dev.index)

catboost_predictions = load_scikit_predictions(fs_catboost_plus_logreg, X_dev)
neural_predictions = load_neural_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_dev.json')

tmp = vote_predictions([neural_predictions, catboost_predictions], soft=True, weights=[1., 1.])
ensemble_pred = probs_to_classes(tmp)

print('weighted f1: ', metrics.f1_score(y_dev.values, ensemble_pred, average='weighted'))
print('macro f1: ', metrics.f1_score(y_dev.values, ensemble_pred, average='macro'))
print('accuracy: ', metrics.accuracy_score(y_dev.values, ensemble_pred))
print()
print(metrics.classification_report(y_dev, ensemble_pred, digits=4))<jupyter_output>weighted f1:  0.5413872373769657
macro f1:  0.5354738926873194
accuracy:  0.5389321468298109

                              precision    recall  f1-score   support

              attribution_NS     0.8409    0.9024    0.8706        82
              attribution_SN     0.8424    0.8564    0.8493       181
               background_NS     0.2167    0.1461    0.1745        89
             cause-effect_NS     0.6015    0.5128    0.5536       156
             cause-effect_SN     0.5096    0.4598    0.4834       174
               comparison_NN     0.1449    0.1923    0.1653        52
               concession_NS     0.9000    0.5625    0.6923        32
                condition_NS     0.6438    0.6620    0.6528        71
                condition_SN     0.6716    0.8333    0.7438       108
                 contrast_NN     0.7229    0.6231    0.6693       268
              elaboration_NS     0.3932    0.5776    0.4679       644
                 evidence_NS     0.1698    0.1698    0.1698      [...]<jupyter_text>On test set<jupyter_code>_test_samples = test_samples[:]
test_samples = _test_samples[:]
mask = test_samples.filename.str.contains('news')
test_samples = test_samples[test_samples['filename'].str.contains('news')]
mask.shape
test_samples.shape
def mask_predictions(predictions, mask):
    result = []
    mask = mask.values
    for i, prediction in enumerate(predictions):
        if mask[i]:
            result.append(prediction)
    return result
TARGET = 'relation'

y_test, X_test = test_samples[TARGET].to_frame(), test_samples.drop(TARGET, axis=1).drop(
    columns=drop_columns + ['category_id', 'index'])

X_scaled_np = scaler.transform(X_test)
X_test = pd.DataFrame(X_scaled_np, index=X_test.index)

catboost_predictions = load_scikit_predictions(fs_catboost_plus_logreg, X_test)
neural_predictions = load_neural_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_test.json')
# neural_predictions = mask_predictions(neural_predictions, mask)

tmp = vote_predictions([neural_predictions, catboost_predictions], soft=True, weights=[1., 2.])

ensemble_pred = probs_to_classes(tmp)

print('weighted f1: ', metrics.f1_score(y_test.values, ensemble_pred, average='weighted'))
print('macro f1: ', metrics.f1_score(y_test.values, ensemble_pred, average='macro'))
print('accuracy: ', metrics.accuracy_score(y_test.values, ensemble_pred))
print()
print(metrics.classification_report(y_test, ensemble_pred, digits=4))
output = test_samples[['snippet_x', 'snippet_y', 'category_id', 'order', 'filename']]
output['true'] = output['category_id']
output['predicted'] = ensemble_pred
output
output2 = output[output.true != output.predicted.map(lambda row: row.split('_')[0])]
output2.shape
output2
del output2['category_id']
output2.to_csv('mispredictions.csv')
test_metrics = metrics.classification_report(y_test, ensemble_pred, digits=4, output_dict=True)
test_f1 = np.array(
    [test_metrics[label].get('f1-score') for label in test_metrics if type(test_metrics[label]) == dict]) * 100

test_f1<jupyter_output><empty_output><jupyter_text>Ensemble: BiMPM + ESIM On dev set<jupyter_code>!ls models/label_predictor_bimpm/
from sklearn import metrics


TARGET = 'relation'

y_dev, X_dev = dev_samples['relation'].to_frame(), dev_samples.drop('relation', axis=1).drop(
    columns=drop_columns + ['category_id', 'index'])

X_scaled_np = scaler.transform(X_dev)
X_dev = pd.DataFrame(X_scaled_np, index=X_dev.index)

bimpm = load_neural_predictions(f'models/label_predictor_bimpm/winter-sweep-1/predictions_dev.json')
esim = load_neural_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_dev.json')
catboost_predictions = load_scikit_predictions(fs_catboost_plus_logreg, X_dev)

tmp = vote_predictions(bimpm, esim, soft=False, weights=[1., 1.])
tmp = vote_predictions(tmp, catboost_predictions, soft=True, weights=[1., 1.])
ensemble_pred = probs_to_classes(tmp)

print('weighted f1: ', metrics.f1_score(y_dev.values, ensemble_pred, average='weighted'))
print('macro f1: ', metrics.f1_score(y_dev.values, ensemble_pred, average='macro'))
print('accuracy: ', metrics.accuracy_score(y_dev.values, ensemble_pred))
print()
print(metrics.classification_report(y_dev, ensemble_pred, digits=4))<jupyter_output><empty_output><jupyter_text>On test set<jupyter_code>TARGET = 'relation'

y_test, X_test = test_samples[TARGET].to_frame(), test_samples.drop(TARGET, axis=1).drop(
    columns=drop_columns + ['category_id', 'index'])

X_scaled_np = scaler.transform(X_test)
X_test = pd.DataFrame(X_scaled_np, index=X_test.index)

bimpm = load_neural_predictions(f'models/label_predictor_bimpm/winter-sweep-1/predictions_test.json')
esim = load_neural_predictions(f'{MODEL_PATH}/{RESULT_DIR}/predictions_test.json')
catboost_predictions = load_scikit_predictions(fs_catboost_plus_logreg, X_test)

tmp = vote_predictions([bimpm, catboost_predictions, esim], soft=True, weights=[2., 1, 15.])

ensemble_pred = probs_to_classes(tmp)

print('weighted f1: ', metrics.f1_score(y_test.values, ensemble_pred, average='weighted'))
print('macro f1: ', metrics.f1_score(y_test.values, ensemble_pred, average='macro'))
print('accuracy: ', metrics.accuracy_score(y_test.values, ensemble_pred))
print()
print(metrics.classification_report(y_test, ensemble_pred, digits=4))<jupyter_output><empty_output>