Upload 16 files

app.py

@@ -0,0 +1,103 @@
+import streamlit as st
+from transformers import AutoModelForTokenClassification
+from annotated_text import annotated_text
+import numpy as np
+import os, joblib
+
+from utils import get_idxs_from_text
+
+model = AutoModelForTokenClassification.from_pretrained("CyberPeace-Institute/Cybersecurity-Knowledge-Graph", trust_remote_code=True)
+
+role_classifiers = {}
+folder_path = '/arg_role_models'
+for filename in os.listdir(os.getcwd() + folder_path):
+    if filename.endswith('.joblib'):
+        file_path = os.getcwd() + os.path.join(folder_path, filename)
+        clf = joblib.load(file_path)
+        arg = filename.split(".")[0]
+        role_classifiers[arg] = clf
+
+def annotate(name):
+    tokens = [item["token"] for item in output]
+    tokens = [token.replace(" ", "") for token in tokens]
+    text = model.tokenizer.decode([item["id"] for item in output])
+    idxs = get_idxs_from_text(text, tokens)
+    labels = [item[name] for item in output]
+
+    annotated_text_list = []
+    last_label = ""
+    cumulative_tokens = "" 
+    last_id = 0
+    for idx, label in zip(idxs, labels):
+        to_label = label
+        label_short = to_label.split("-")[1] if "-" in to_label else to_label
+        if last_label == label_short:
+            cumulative_tokens += text[last_id : idx["end_idx"]]
+            last_id = idx["end_idx"]
+        else:
+            if last_label != "":
+                if last_label == "O":
+                    annotated_text_list.append(cumulative_tokens)
+                else:
+                    annotated_text_list.append((cumulative_tokens, last_label))
+            last_label = label_short
+            cumulative_tokens = idx["word"]
+            last_id = idx["end_idx"]
+    if last_label == "O":
+        annotated_text_list.append(cumulative_tokens)
+    else:  
+        annotated_text_list.append((cumulative_tokens, last_label))
+    annotated_text(annotated_text_list)
+
+def get_arg_roles(output):
+    args = [(idx, item["argument"], item["token"]) for idx, item in enumerate(output) if item["argument"]!= "O"]
+        
+    entities = []
+    current_entity = None
+    for position, label, token in args:
+        if label.startswith('B-'):
+            if current_entity is not None:
+                entities.append(current_entity)
+            current_entity = {'label': label[2:], 'text': token.replace(" ", ""), 'start': position, 'end': position}
+        elif label.startswith('I-'):
+            if current_entity is not None:
+                current_entity['text'] += ' ' + token.replace(" ", "")
+                current_entity['end'] = position
+    for entity in entities:
+        context = model.tokenizer.decode([item["id"] for item in output[max(0, entity["start"] - 15) : min(len(output), entity["end"] + 15)]])
+        entity["context"] = context
+    
+    for entity in entities:
+        if len(model.arg_2_role[entity["label"]]) > 1:
+            sent_embed = model.embed_model.encode(entity["context"])
+            arg_embed = model.embed_model.encode(entity["text"])
+            embed = np.concatenate((sent_embed, arg_embed))
+            arg_clf = role_classifiers[entity["label"]]
+            role_id = arg_clf.predict(embed.reshape(1, -1))
+            role = model.arg_2_role[entity["label"]][role_id[0]]
+            entity["role"] = role
+        else:
+            entity["role"] = model.arg_2_role[entity["label"]][0]
+    
+    for item in output:
+        item["role"] = "O"
+    for entity in entities:
+        for i in range(entity["start"], entity["end"] + 1):
+            output[i]["role"] = entity["role"]
+    return output
+
+st.title("Create Knowledge Graphs from Cyber Incidents")
+
+text_input = st.text_area("Enter your text here", height=100)
+
+if text_input:
+    output = model(text_input)
+    st.subheader("Event Nuggets")
+    annotate("nugget")
+    st.subheader("Event Arguments")
+    annotate("argument")
+    st.subheader("Realis of Event Nuggets")
+    annotate("realis")
+    output = get_arg_roles(output)
+    st.subheader("Role of the Event Arguments")
+    annotate("role")

arg_role_models/Capabilities.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d542202e44436144702ed00865ebd91f9714f78e5cd05277aa69fc66c15479ba
+size 7728396

arg_role_models/Data.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6d04fd53240a8bc105b255c440c8d00354249cf268067ab4fbeef2fc94f73ed
+size 2430452

arg_role_models/Device.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56a3356645eb885604d651cb20d0f80887c710b1acedcbddddef5e11ddf92a2
+size 4236556

arg_role_models/File.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e0ea25e1f1f7c215e5335bf7a2ee2638820762de25e84d47c8ad90601b096e9
+size 3413236

arg_role_models/Money.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7481b7117ef96b4810ab7d1cba13544dd86497126a5db4546ee830d2649b8557
+size 2741604

arg_role_models/Number.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18f178926cb3ea67f32e9537841b3c71abcef8b66342721e9c287d624ac19a90
+size 3254948

arg_role_models/Organization.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:026de561dc01cd8f8f89ac52bbdd7c1cc8746e44627d2bb87de985273c03ad51
+size 17816780

arg_role_models/PII.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25c5e38a3e1cf918fe9cb929db156de4c180d1cf8dc0ddceaef16c8a15ab6557
+size 2911620

arg_role_models/Person.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60028d5e392e548f7cd8daf9681d9a181b5a8ea4c3f0a0a8f35d2d82314c3120
+size 14664508

arg_role_models/Software.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9be024f9c4d0ae43db13ed4b4b0140f209b5b0c43a9bb675855a4d484c12a7d
+size 4388264

arg_role_models/System.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d79586b50dec6bd6d838110ed568f66329e5266f86bc6350bcf522876d764309
+size 7239048

arg_role_models/Version.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3584898a0d7af2ba4f927e91805018565218f43806ca9572751370ae8c5fa07f
+size 3637220

arg_role_models/Website.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4c8e3d2f167d2c5bcef014051dd52c231ac8394d772b33343cbd1fe8424e4b7
+size 3654520

test.ipynb

@@ -0,0 +1,1032 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- configuration.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "047d95084e98403589b108942daf1a0c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)esolve/main/model.py:   0%|          | 0.00/6.66k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "840315351a8e489d9e65c4285a9511b2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)event_arg_predict.py:   0%|          | 0.00/12.8k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e211aa84be44465396acb1e1e1b5b683",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)/args_model_utils.py:   0%|          | 0.00/11.2k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- args_model_utils.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b4c2fe2b94884fa5bc8e245a874ad21a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)nt_nugget_predict.py:   0%|          | 0.00/10.7k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d192dbc95c6749b6a9c6a5c984c646fd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)ugget_model_utils.py:   0%|          | 0.00/6.59k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- nugget_model_utils.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0ca6f6c3195c4708bb1834a0c876e893",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)esolve/main/utils.py:   0%|          | 0.00/7.05k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- utils.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- event_nugget_predict.py\n",
+      "- nugget_model_utils.py\n",
+      "- utils.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- event_arg_predict.py\n",
+      "- args_model_utils.py\n",
+      "- event_nugget_predict.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2776f9469d414ed6aaa5b594fae0c6a8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)nt_realis_predict.py:   0%|          | 0.00/11.5k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "22265c1ceb7d4611a5118fbde096bc1d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)ealis_model_utils.py:   0%|          | 0.00/7.09k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- realis_model_utils.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- event_realis_predict.py\n",
+      "- realis_model_utils.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
+      "A new version of the following files was downloaded from https://huggingface.co/CyberPeace-Institute/Cybersecurity-Knowledge-Graph:\n",
+      "- model.py\n",
+      "- event_arg_predict.py\n",
+      "- event_realis_predict.py\n",
+      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
+      "Some weights of RobertaModel were not initialized from the model checkpoint at ehsanaghaei/SecureBERT and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "Some weights of RobertaModel were not initialized from the model checkpoint at ehsanaghaei/SecureBERT and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "Some weights of RobertaModel were not initialized from the model checkpoint at ehsanaghaei/SecureBERT and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load model directly\n",
+    "from transformers import AutoModelForTokenClassification\n",
+    "model = AutoModelForTokenClassification.from_pretrained(\"CyberPeace-Institute/Cybersecurity-Knowledge-Graph\", trust_remote_code=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e3e5818a4d6b4ba99243603a068622d0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/1 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f3a24eb19f414ae3ad974b1461fc3e64",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/1 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3c3dd376f5614f739e5c0780af8b3a9b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/1 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "20c9cd31ea854bb58208249c18840e14",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/1 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a668052410cf41968f97ebdcf187debc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/1 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'id': 1437, 'token': ' ', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 50118, 'token': '\\n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 287, 'token': ' As', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 9, 'token': ' of', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 502, 'token': ' June', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 336, 'token': ' 2016', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 55, 'token': ' more', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 87, 'token': ' than', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 3982, 'token': ' 150', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 153,\n",
+       "  'token': ' million',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2171,\n",
+       "  'token': ' active',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1434,\n",
+       "  'token': ' users',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 10754,\n",
+       "  'token': ' interact',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 19, 'token': ' with', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 65, 'token': ' one', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 277,\n",
+       "  'token': ' another',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1230,\n",
+       "  'token': ' daily',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1241, 'token': ' via', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11477,\n",
+       "  'token': ' Snapchat',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5763,\n",
+       "  'token': ' Others',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 32, 'token': ' are', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 4777,\n",
+       "  'token': ' drawn',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 30, 'token': ' by', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 544,\n",
+       "  'token': ' service',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 29, 'token': 's', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 55, 'token': ' more', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 485,\n",
+       "  'token': ' recent',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1575,\n",
+       "  'token': ' features',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2246,\n",
+       "  'token': ' Those',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 680,\n",
+       "  'token': ' include',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 14100,\n",
+       "  'token': ' Snap',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 20122, 'token': 'cash', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 10, 'token': ' a', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5448,\n",
+       "  'token': ' method',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2942,\n",
+       "  'token': ' introduced',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 13, 'token': ' for', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1434,\n",
+       "  'token': ' users',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2142, 'token': ' send', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1830,\n",
+       "  'token': ' mobile',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 3081,\n",
+       "  'token': ' payments',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 49, 'token': ' their', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 964,\n",
+       "  'token': ' friends',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1437, 'token': ' ', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 50118, 'token': '\\n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 6211,\n",
+       "  'token': ' Given',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1553, 'token': ' app', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 29, 'token': 's', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 7347,\n",
+       "  'token': ' popularity',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 24, 'token': ' it', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 29, 'token': 's', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 117, 'token': ' no', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5170,\n",
+       "  'token': ' wonder',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 804,\n",
+       "  'token': ' online',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 9177,\n",
+       "  'token': ' criminals',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 33, 'token': ' have', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 278, 'token': ' set', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 49, 'token': ' their', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 579, 'token': ' s', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 6183, 'token': 'ights', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 15, 'token': ' on', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11597,\n",
+       "  'token': ' hacking',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1434,\n",
+       "  'token': ' users',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11477,\n",
+       "  'token': ' Snapchat',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-PII',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2349,\n",
+       "  'token': ' accounts',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-PII',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 286, 'token': ' For', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 4327,\n",
+       "  'token': ' instance',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 124, 'token': ' back', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11, 'token': ' in', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 628,\n",
+       "  'token': ' late',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Time',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1014,\n",
+       "  'token': ' 2013',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Time',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 10,\n",
+       "  'token': ' a',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 333,\n",
+       "  'token': ' group',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 9,\n",
+       "  'token': ' of',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 11344,\n",
+       "  'token': ' hackers',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1027,\n",
+       "  'token': ' published',\n",
+       "  'nugget': 'B-Databreach',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 10, 'token': ' a', 'nugget': 'O', 'argument': 'B-Data', 'realis': 'O'},\n",
+       " {'id': 8503,\n",
+       "  'token': ' database',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Data',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 8200,\n",
+       "  'token': ' containing',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'B-PII', 'realis': 'O'},\n",
+       " {'id': 32200,\n",
+       "  'token': ' usernames',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-PII',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 8, 'token': ' and', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1028,\n",
+       "  'token': ' phone',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-PII',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1530,\n",
+       "  'token': ' numbers',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-PII',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 9, 'token': ' of', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2219,\n",
+       "  'token': ' approximately',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 204, 'token': ' 4', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 4, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 401, 'token': '6', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 153,\n",
+       "  'token': ' million',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 11477,\n",
+       "  'token': ' Snapchat',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1434,\n",
+       "  'token': ' users',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1437, 'token': ' ', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 50118, 'token': '\\n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 234, 'token': ' N', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 4550, 'token': 'ef', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 38313,\n",
+       "  'token': 'arious',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2172,\n",
+       "  'token': ' individuals',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 115, 'token': ' could', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 33, 'token': ' have', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 341, 'token': ' used', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 14, 'token': ' that', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 335,\n",
+       "  'token': ' information',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 4392,\n",
+       "  'token': ' profile',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 3247,\n",
+       "  'token': ' targets',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 420,\n",
+       "  'token': ' across',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1533,\n",
+       "  'token': ' multiple',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 3748, 'token': ' web', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2349,\n",
+       "  'token': ' accounts',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 166, 'token': ' We', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 67, 'token': ' also', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 6056, 'token': ' ca', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 295, 'token': ' n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 75, 'token': \"'t\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 4309,\n",
+       "  'token': ' forget',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 59, 'token': ' about', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 573,\n",
+       "  'token': ' security',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1160,\n",
+       "  'token': ' incident',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 14, 'token': ' that', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2756,\n",
+       "  'token': ' occurred',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 124, 'token': ' back', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11, 'token': ' in', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 902,\n",
+       "  'token': ' February',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Time',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 336,\n",
+       "  'token': ' 2016',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Time',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 96, 'token': ' In', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 14,\n",
+       "  'token': ' that',\n",
+       "  'nugget': 'B-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 908,\n",
+       "  'token': ' attack',\n",
+       "  'nugget': 'I-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 951,\n",
+       "  'token': ' someone',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 7444,\n",
+       "  'token': ' posed',\n",
+       "  'nugget': 'B-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 25,\n",
+       "  'token': ' as',\n",
+       "  'nugget': 'I-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 138,\n",
+       "  'token': ' company',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Organization',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 29, 'token': 's', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1324,\n",
+       "  'token': ' CEO',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 8, 'token': ' and', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 7013,\n",
+       "  'token': ' convinced',\n",
+       "  'nugget': 'B-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 10, 'token': ' a', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11477,\n",
+       "  'token': ' Snapchat',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 3200,\n",
+       "  'token': ' employee',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2142,\n",
+       "  'token': ' send',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Purpose',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 81,\n",
+       "  'token': ' over',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Purpose',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 10984,\n",
+       "  'token': ' payroll',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Purpose',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 335,\n",
+       "  'token': ' information',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'I-Purpose',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1437, 'token': ' ', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 50118, 'token': '\\n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 20,\n",
+       "  'token': ' The',\n",
+       "  'nugget': 'B-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 1800,\n",
+       "  'token': ' successful',\n",
+       "  'nugget': 'I-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 28237,\n",
+       "  'token': ' phish',\n",
+       "  'nugget': 'I-Phishing',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 3284,\n",
+       "  'token': ' ultimately',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 13969,\n",
+       "  'token': ' compromised',\n",
+       "  'nugget': 'B-Databreach',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'Actual'},\n",
+       " {'id': 4295,\n",
+       "  'token': ' dozens',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 9, 'token': ' of', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1321,\n",
+       "  'token': ' employees',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-Person',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 14875,\n",
+       "  'token': ' identities',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'B-PII',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 598, 'token': ' To', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 28, 'token': ' be', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2105, 'token': ' fair', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 10, 'token': ' a', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 14251,\n",
+       "  'token': ' mega',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 6999,\n",
+       "  'token': ' breach',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 15, 'token': ' on', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 3189,\n",
+       "  'token': ' scale',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 9, 'token': ' of', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 99, 'token': ' what', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2132,\n",
+       "  'token': ' affected',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 6582,\n",
+       "  'token': ' LinkedIn',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 9494,\n",
+       "  'token': ' Tumblr',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 8, 'token': ' and', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 10354,\n",
+       "  'token': ' Yahoo',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 34, 'token': ' has', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 648, 'token': ' yet', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2506,\n",
+       "  'token': ' strike',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11203,\n",
+       "  'token': ' messaging',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 1553, 'token': ' app', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 125, 'token': ' But', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 14, 'token': ' that', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 29, 'token': 's', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 45, 'token': ' not', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 224, 'token': ' say', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 9177,\n",
+       "  'token': ' criminals',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 32, 'token': ' are', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 295, 'token': ' n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 75, 'token': \"'t\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 667,\n",
+       "  'token': ' trying',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 465, 'token': ' find', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 10, 'token': ' a', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 169, 'token': ' way', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 88, 'token': ' into', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 82, 'token': ' people', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 128, 'token': \" '\", 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 29, 'token': 's', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2349,\n",
+       "  'token': ' accounts',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1437, 'token': ' ', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 50118, 'token': '\\n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 46450,\n",
+       "  'token': ' Hackers',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2563,\n",
+       "  'token': ' clearly',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 33, 'token': ' have', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 11477,\n",
+       "  'token': ' Snapchat',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 11, 'token': ' in', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 49, 'token': ' their', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 579, 'token': ' s', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 6183, 'token': 'ights', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2156, 'token': ',', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 61, 'token': ' which', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 16, 'token': ' is', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 596, 'token': ' why', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1434,\n",
+       "  'token': ' users',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 240, 'token': ' need', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1532,\n",
+       "  'token': ' learn',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 141, 'token': ' how', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 7, 'token': ' to', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1514, 'token': ' spot', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5, 'token': ' the', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2892,\n",
+       "  'token': ' warning',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 2434,\n",
+       "  'token': ' signs',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 9, 'token': ' of', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 10, 'token': ' a', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 14157,\n",
+       "  'token': ' hack',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 8, 'token': ' and', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 141, 'token': ' how', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 51, 'token': ' they', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 64, 'token': ' can', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 5312,\n",
+       "  'token': ' recover',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 49, 'token': ' their', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 2349,\n",
+       "  'token': ' accounts',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 114, 'token': ' if', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 951,\n",
+       "  'token': ' someone',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 30478,\n",
+       "  'token': ' compromises',\n",
+       "  'nugget': 'O',\n",
+       "  'argument': 'O',\n",
+       "  'realis': 'O'},\n",
+       " {'id': 106, 'token': ' them', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 479, 'token': '.', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 1437, 'token': ' ', 'nugget': 'O', 'argument': 'O', 'realis': 'O'},\n",
+       " {'id': 50118, 'token': '\\n', 'nugget': 'O', 'argument': 'O', 'realis': 'O'}]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "input = \"\"\"\n",
+    "As of June 2016, more than 150 million active users interact with one another daily via Snapchat. Others are drawn by the service's more recent features. Those include Snapcash, a method introduced for users to send mobile payments to their friends. \n",
+    "Given the app's popularity, it's no wonder online criminals have set their sights on hacking users' Snapchat accounts. For instance, back in late 2013, a group of hackers published a database containing the usernames and phone numbers of approximately 4.6 million Snapchat users. \n",
+    "Nefarious individuals could have used that information to profile targets across multiple web accounts. We also can't forget about the security incident that occurred back in February 2016. In that attack, someone posed as the company's CEO and convinced a Snapchat employee to send over payroll information. \n",
+    "The successful phish ultimately compromised dozens of employees' identities. To be fair, a mega breach on the scale of what affected LinkedIn, Tumblr, and Yahoo has yet to strike the messaging app. But that's not to say criminals aren't trying to find a way into people's accounts. \n",
+    "Hackers clearly have Snapchat in their sights, which is why users need to learn how to spot the warning signs of a hack and how they can recover their accounts if someone compromises them.\n",
+    "\"\"\"\n",
+    "\n",
+    "model(input)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.forward()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.15"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

utils.py

@@ -0,0 +1,196 @@
+list_of_pos_tags = [
+    "ADJ",
+    "ADP",
+    "ADV",
+    "AUX",
+    "CCONJ",
+    "DET",
+    "INTJ",
+    "NOUN",
+    "NUM",
+    "PART",
+    "PRON",
+    "PROPN",
+    "PUNCT",
+    "SCONJ",
+    "SYM",
+    "VERB",
+    "X"
+]
+
+realis_list = ["O",
+    "Generic", 
+    "Other", 
+    "Actual"
+]
+
+
+event_args_list = ['O',
+    'B-System',
+    'I-System',
+    'B-Organization',
+    'B-Money',
+    'I-Money',
+    'B-Device',
+    'B-Person',
+    'I-Person',
+    'B-Vulnerability',
+    'I-Vulnerability',
+    'B-Capabilities',
+    'I-Capabilities',
+    'I-Organization',
+    'B-PaymentMethod',
+    'I-PaymentMethod',
+    'B-Data',
+    'I-Data',
+    'B-Number',
+    'I-Number',
+    'B-Malware',
+    'I-Malware',
+    'B-PII',
+    'I-PII',
+    'B-CVE',
+    'I-CVE',
+    'B-Purpose',
+    'I-Purpose',
+    'B-File',
+    'I-File',
+    'I-Device',
+    'B-Time',
+    'I-Time',
+    'B-Software',
+    'I-Software',
+    'B-Patch',
+    'I-Patch',
+    'B-Version',
+    'I-Version',
+    'B-Website',
+    'I-Website',
+    'B-GPE',
+    'I-GPE'
+]
+
+event_nugget_list = ['O',
+ 'B-Ransom',
+ 'I-Ransom',
+ 'B-DiscoverVulnerability',
+ 'I-DiscoverVulnerability',
+ 'B-PatchVulnerability',
+ 'I-PatchVulnerability',
+ 'B-Databreach',
+ 'I-Databreach',
+ 'B-Phishing',
+ 'I-Phishing'
+]
+
+arg_2_role = {
+    "File" : ['Tool', 'Trusted-Entity'],
+    "Person" : ['Victim', 'Attacker', 'Discoverer', 'Releaser', 'Trusted-Entity', 'Vulnerable_System_Owner'],
+    "Capabilities" : ['Attack-Pattern', 'Capabilities', 'Issues-Addressed'],
+    "Purpose" : ['Purpose'],
+    "Time" : ['Time'],
+    "PII" : ['Compromised-Data', 'Trusted-Entity'],
+    "Data" : ['Compromised-Data', 'Trusted-Entity'],
+    "Organization" : ['Victim', 'Releaser', 'Discoverer', 'Attacker', 'Vulnerable_System_Owner', 'Trusted-Entity'],
+    "Patch" : ['Patch'],
+    "Software" : ['Vulnerable_System', 'Victim', 'Trusted-Entity', 'Supported_Platform'],
+    "Vulnerability" : ['Vulnerability'],
+    "Version" : ['Patch-Number', 'Vulnerable_System_Version'],
+    "Device" : ['Vulnerable_System', 'Victim', 'Supported_Platform'],
+    "CVE" : ['CVE'],
+    "Number" : ['Number-of-Data', 'Number-of-Victim'],
+    "System" : ['Victim', 'Supported_Platform', 'Vulnerable_System', 'Trusted-Entity'],
+    "Malware" : ['Tool'],
+    "Money" : ['Price', 'Damage-Amount'],
+    "PaymentMethod" : ['Payment-Method'],
+    "GPE" : ['Place'],
+    "Website" : ['Trusted-Entity', 'Tool', 'Vulnerable_System', 'Victim', 'Supported_Platform'],
+}
+
+def get_content(data):
+    return data["content"]
+
+def get_event_nugget(data):
+    return [
+        {"nugget" : event["nugget"], "type" : event["type"], "subtype" : event["subtype"], "realis" : event["realis"]}
+        for hopper in data["cyberevent"]["hopper"] for event in hopper["events"]
+    ]
+def get_event_args(data):
+    events = [event for hopper in data["cyberevent"]["hopper"] for event in hopper["events"]]
+    args = []
+    for event in events:
+        if "argument" in event.keys():
+            args.extend(event["argument"])
+    return args
+
+def get_idxs_from_text(text, text_tokenized):
+    rest_text = text
+    last_idx = 0
+    result_dict = []
+
+    for substring in text_tokenized:
+        index = rest_text.find(substring)
+        result_dict.append(
+            {
+                "word" : substring,
+                "start_idx" : last_idx + index,
+                "end_idx" : last_idx + index + len(substring)
+            }
+        )
+        rest_text = rest_text[index + len(substring) : ]
+        last_idx += index + len(substring)
+    return result_dict
+
+def get_entity_from_idx(start_idx, end_idx, event_nuggets):
+    event_nuggets_idxs = [(nugget["nugget"]["startOffset"], nugget["nugget"]["endOffset"]) for nugget in event_nuggets]
+    for idx, (nugget_start, nugget_end) in enumerate(event_nuggets_idxs):
+        if (start_idx == nugget_start and end_idx == nugget_end) or (start_idx == nugget_start and end_idx <= nugget_end) or (start_idx == nugget_start and end_idx > nugget_end) or (end_idx == nugget_end and start_idx < nugget_start) or (start_idx <= nugget_start and end_idx <= nugget_end and end_idx > nugget_start):
+            return "B-" + event_nuggets[idx]["subtype"]
+        elif (start_idx > nugget_start and end_idx <= nugget_end) or (start_idx > nugget_start and start_idx < nugget_end):
+            return "I-" + event_nuggets[idx]["subtype"]
+    return "O"
+
+def get_entity_and_realis_from_idx(start_idx, end_idx, event_nuggets):
+    event_nuggets_idxs = [(nugget["nugget"]["startOffset"], nugget["nugget"]["endOffset"]) for nugget in event_nuggets]
+    for idx, (nugget_start, nugget_end) in enumerate(event_nuggets_idxs):
+        if (start_idx == nugget_start and end_idx == nugget_end) or (start_idx == nugget_start and end_idx <= nugget_end) or (start_idx == nugget_start and end_idx > nugget_end) or (end_idx == nugget_end and start_idx < nugget_start) or (start_idx <= nugget_start and end_idx <= nugget_end and end_idx > nugget_start):
+            return "B-" + event_nuggets[idx]["subtype"], "B-" + event_nuggets[idx]["realis"]
+        elif (start_idx > nugget_start and end_idx <= nugget_end) or (start_idx > nugget_start and start_idx < nugget_end):
+            return "I-" + event_nuggets[idx]["subtype"], "I-" + event_nuggets[idx]["realis"]
+    return "O", "O"
+
+def get_args_entity_from_idx(start_idx, end_idx, event_args):
+    event_nuggets_idxs = [(nugget["startOffset"], nugget["endOffset"]) for nugget in event_args]
+    for idx, (nugget_start, nugget_end) in enumerate(event_nuggets_idxs):
+        if (start_idx == nugget_start and end_idx == nugget_end) or (start_idx == nugget_start and end_idx <= nugget_end) or (start_idx == nugget_start and end_idx > nugget_end) or (end_idx == nugget_end and start_idx < nugget_start) or (start_idx <= nugget_start and end_idx <= nugget_end and end_idx > nugget_start):
+            return "B-" + event_args[idx]["type"]
+        elif (start_idx > nugget_start and end_idx <= nugget_end) or (start_idx > nugget_start and start_idx < nugget_end):
+            return "I-" + event_args[idx]["type"]
+    return "O"
+
+def split_with_character(string, char):
+    result = []
+    start = 0
+    for i, c in enumerate(string):
+        if c == char:
+            result.append(string[start:i])
+            result.append(char)
+            start = i + 1
+    result.append(string[start:])
+    return [x for x in result if x != '']
+
+def extend_list_with_character(content_list, character):
+    content_as_words = []
+    for word in content_list:
+        if character in word:
+            split_list = split_with_character(word, character)
+            content_as_words.extend(split_list)
+        else:
+            content_as_words.append(word)
+    return content_as_words
+
+def find_dict_by_overlap(list_of_dicts, key_value_pairs):
+    for dictionary in list_of_dicts:
+        if max(dictionary["start"], dictionary["end"]) >= min(key_value_pairs["start"], key_value_pairs["end"]) and max(key_value_pairs["start"], key_value_pairs["end"]) >= min(dictionary["start"], dictionary["end"]):
+            return dictionary
+    return None