Upload 8 files

README.md

@@ -1,13 +1,9 @@
----
-title: Maliza Uhalifu Mtandaoni
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
-sdk: gradio
-sdk_version: 4.36.1
-app_file: app.py
-pinned: false
-license: unknown
----
-
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+# Hate Speech Classification System
+
+The model parses text data and returns boolean responses in relation to data provided.
+
+sdk: gradio
+sdk_version: 3.36.1
+app_file: app.py
+
+Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>

app.py

@@ -1,63 +1,30 @@
-import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
-
-
-if __name__ == "__main__":
-    demo.launch()
+import tensorflow as tf
+import gradio as gr
+from gradio.components import input
+import pandas as pd
+import pickle
+from keras.utils import pad_sequences
+
+max_len = 200
+
+# set the vocabulary mapping on a new TextVectorization layer
+with open('tokenizer.pickle', 'rb') as handle:
+    tokenizer = pickle.load(handle)
+model = tf.keras.models.load_model('toxic.h5')
+
+
+arr=["toxic","severe_toxic","obscene threat","insult","identity_hate"]
+def score_comment(comment):
+    sequences = tokenizer.texts_to_sequences([comment])
+    inp = pad_sequences(sequences,maxlen=max_len)
+    results = model.predict(inp)
+    text = ''
+    for i in range(len(arr)):
+        text += '{}: {}\n'.format(arr[i], results[0][i]>0.5)
+    return text
+
+interface = gr.Interface(fn=score_comment, 
+                         inputs=gr.inputs.Textbox(lines=2, placeholder='Comment to score'),
+                        outputs='text')
+
+interface.launch(debug=True,share=True)

requirements.txt

@@ -1 +1,5 @@
-huggingface_hub==0.22.2
+gradio==3.36.1
+keras==2.13.1
+pandas==2.0.3
+Pillow==10.0.0
+tensorflow==2.13.0

swahili.csv

@@ -0,0 +1,117 @@
+comment,toxic,severe_toxic,obscene threat,insult,identity_hate
+Huyu msee ni malaya,1,0,0,1,0
+Zakayo ni mbwa,0,0,0,1,0
+Chira ni shoga,1,1,0,1,0
+Akinirushia mamdomo mi ntamuua,1,1,1,0,0
+Haga sana,0,0,0,1,0
+Kuma ya mamako,1,1,0,1,0
+Wewe ni mjinga sana!,1,0,0,1,0
+Nataka nikuchape vibaya!,0,1,0,0,0
+Huo ni upuuzi mtupu!,1,0,0,0,0
+Wewe ni takataka!,0,0,0,0,1
+Nitakuua!,0,0,1,0,0
+Unakera sana!,1,0,0,1,0
+Unazungumza kama mpumbavu!,1,0,0,0,0
+Wewe ni mchafu!,0,0,0,1,0
+Nitaharibu maisha yako!,0,0,1,0,0
+Hiyo ni ujinga wa hali ya juu!,1,0,0,0,0
+Wewe ni nani kuzungumza hivyo?,0,0,0,1,0
+Mimi nitakumaliza!,0,1,1,0,0
+Una sura mbaya sana!,0,0,0,1,0
+Usinieleze mambo yako ya kijinga!,1,0,0,0,0
+Wewe ni kichekesho!,0,0,0,1,0
+Nitakupiga hadi utoe damu!,0,1,0,0,0
+Wewe ni taka taka!,1,0,0,1,0
+Ondoka hapa!,1,0,0,0,0
+Nitakuvunja miguu!,0,1,1,0,0
+Wewe ni mpuuzi tu!,1,0,0,1,0
+Nitakuchoma moto!,0,1,1,0,0
+Unafanya mambo ya kipumbavu!,1,0,0,0,0
+Nakuona ni kama mavi ya mbwa!,0,0,0,1,0
+Usiniletee upuuzi wako!,1,0,0,0,0
+Utakufa maskini!,0,0,0,1,0
+Wewe ni bure kabisa!,1,0,0,0,0
+Nitakufanya ujutie kuzaliwa kwako!,0,1,1,0,0
+Wewe ni kama paka wa mtaani!,0,0,0,1,0
+Nitaharibu uso wako!,0,1,1,0,0
+Unafikiri wewe ni nani?,1,0,0,0,0
+Hakuna mtu anayekupenda!,1,0,0,0,0
+Unapaswa kufa!,0,0,1,0,0
+Wewe ni mnyama tu!,1,0,0,0,0
+Wewe ni mjinga na kipumbavu!,1,0,0,1,0
+Nitakupiga vibaya hadi ulie!,0,1,0,0,0
+Usiniletee upumbavu wako!,1,0,0,0,0
+Wewe ni bure kama taka!,1,0,0,1,0
+Wewe ni kipande cha mbwa!,1,0,0,1,0
+Nitaharibu kila kitu kwako!,0,1,1,0,0
+Wewe ni mfano wa upumbavu!,1,0,0,0,0
+Unakera kama mbu!,1,0,0,1,0
+Wewe ni shetani!,1,0,0,0,0
+Nitakutoa roho!,0,1,1,0,0
+Hakuna mtu anataka kukuona!,1,0,0,0,0
+Wewe ni mbwa mwitu!,1,0,0,0,0
+Nitakupiga ngumi!,0,1,0,0,0
+Wewe ni sumu!,1,0,0,0,0
+Nitachoma nyumba yako!,0,1,1,0,0
+Wewe ni mfano wa ujinga!,1,0,0,0,0
+Wewe ni kama kinyesi!,1,0,0,1,0
+Nitaharibu maisha yako!,0,1,1,0,0
+Wewe ni bure kabisa!,1,0,0,1,0
+Wewe ni kama panya!,1,0,0,0,0
+Nakuona kama panya!,1,0,0,1,0
+Wewe ni kipande cha mavi!,1,0,0,1,0
+Wewe ni chizi!,1,0,0,0,0
+Wewe ni mshenzi!,1,0,0,1,0
+Nitakupiga teke!,0,1,0,0,0
+Wewe ni bure kama takataka!,1,0,0,1,0
+Wewe ni kama kuku wa mtaani!,1,0,0,0,0
+Nitakukata kichwa!,0,1,1,0,0
+Wewe ni bure kabisa kama takataka!,1,0,0,1,0
+Wewe ni mwehu!,1,0,0,0,0
+Nitakutupa jalalani!,0,1,1,0,0
+Wewe ni kipande cha taka!,1,0,0,1,0
+Wewe ni kama mbwa mwitu!,1,0,0,0,0
+Wewe ni mfano wa pumbavu!,1,0,0,0,0
+Nitakupiga teke hadi ufe!,0,1,0,0,0
+Wewe ni mfano wa upuuzi!,1,0,0,0,0
+Wewe ni kipande cha taka!,1,0,0,1,0
+Wewe ni bure kama kinyesi!,1,0,0,1,0
+Wewe ni mnyama!,1,0,0,0,0
+Nitakupiga hadi utoe machozi!,0,1,0,0,0
+Wewe ni kichekesho cha dunia!,1,0,0,1,0
+Wewe ni takataka ya mtaa!,1,0,0,1,0
+Nitakupiga na risasi!,0,1,1,0,0
+Wewe ni mfano wa upumbavu!,1,0,0,0,0
+Wewe ni kama mbwa!,1,0,0,0,0
+Nitaharibu uso wako!,0,1,1,0,0
+Wewe ni kama paka wa mtaani!,1,0,0,0,0
+Nitakufanya ujutie kuzaliwa kwako!,0,1,1,0,0
+Wewe ni kipande cha mbwa!,1,0,0,1,0
+Nitakupiga vibaya hadi ulie!,0,1,0,0,0
+Wewe ni kipande cha takataka!,1,0,0,1,0
+Wewe ni sumu kwa jamii!,1,0,0,0,0
+Nitakuchoma moto!,0,1,1,0,0
+Wewe ni kichekesho cha mtaa!,1,0,0,1,0
+Wewe ni kama kinyesi cha mbwa!,1,0,0,1,0
+Nitakuvunja miguu!,0,1,1,0,0
+Wewe ni takataka kabisa!,1,0,0,1,0
+Wewe ni kama mavi ya mbwa!,1,0,0,1,0
+Nitakupiga hadi utoe damu!,0,1,0,0,0
+Ukiniletea mamdomo mi ntakumada,1,1,1,1,1
+Madem wa Kenya ni malaya,1,1,1,1,1
+Uskii jamaa anatupa haga,1,1,0,1,0
+Sura kama punda,1,0,0,1,0
+Manze ngeus ni nyangumi,1,1,0,1,1
+Si unakaa poa aki,0,0,0,0,0
+Huyo chali ni mpoa,0,0,0,0,0
+Ngeus amenidrive crazy,0,0,0,0,0
+Johnte ni fala achana tu na yeye,1,0,1,1,1
+Nairobii kumejaa malaya,1,1,1,1,1
+Ngeus wake wake ni hoe,1,1,0,1,1
+Dem wake ni whore,1,1,1,1,1
+Manze fuck.,0,1,0,0,0
+Jamaa anapenda nginyo sana,1,1,0,1,0
+Jamaa anastream gwati sana,1,0,1,1,1
+Njoti yake ni kama maziwa mala,1,1,0,1,1
+Niaje bro,0,0,0,0,0
+Kamaa ako na ufala sana,1,0,0,1,0

swahili_dataset.csv

@@ -0,0 +1,117 @@
+comment,toxic,severe_toxic,obscene threat,insult,identity_hate
+Huyu msee ni malaya,1,0,0,1,0
+Zakayo ni mbwa,0,0,0,1,0
+Chira ni shoga,1,1,0,1,0
+Akinirushia mamdomo mi ntamuua,1,1,1,0,0
+Haga sana,0,0,0,1,0
+Kuma ya mamako,1,1,0,1,0
+Wewe ni mjinga sana!,1,0,0,1,0
+Nataka nikuchape vibaya!,0,1,0,0,0
+Huo ni upuuzi mtupu!,1,0,0,0,0
+Wewe ni takataka!,0,0,0,0,1
+Nitakuua!,0,0,1,0,0
+Unakera sana!,1,0,0,1,0
+
+Unazungumza kama mpumbavu!,1,0,0,0,0
+Wewe ni mchafu!,0,0,0,1,0
+Nitaharibu maisha yako!,0,0,1,0,0
+Hiyo ni ujinga wa hali ya juu!,1,0,0,0,0
+Wewe ni nani kuzungumza hivyo?,0,0,0,1,0
+Mimi nitakumaliza!,0,1,1,0,0
+Una sura mbaya sana!,0,0,0,1,0
+Usinieleze mambo yako ya kijinga!,1,0,0,0,0
+Wewe ni kichekesho!,0,0,0,1,0
+Nitakupiga hadi utoe damu!,0,1,0,0,0
+Wewe ni taka taka!,1,0,0,1,0
+Ondoka hapa!,1,0,0,0,0
+Nitakuvunja miguu!,0,1,1,0,0
+Wewe ni mpuuzi tu!,1,0,0,1,0
+Nitakuchoma moto!,0,1,1,0,0
+Unafanya mambo ya kipumbavu!,1,0,0,0,0
+Nakuona ni kama mavi ya mbwa!,0,0,0,1,0
+Usiniletee upuuzi wako!,1,0,0,0,0
+Utakufa maskini!,0,0,0,1,0
+Wewe ni bure kabisa!,1,0,0,0,0
+Nitakufanya ujutie kuzaliwa kwako!,0,1,1,0,0
+Wewe ni kama paka wa mtaani!,0,0,0,1,0
+Nitaharibu uso wako!,0,1,1,0,0
+Unafikiri wewe ni nani?,1,0,0,0,0
+Hakuna mtu anayekupenda!,1,0,0,0,0
+Unapaswa kufa!,0,0,1,0,0
+Wewe ni mnyama tu!,1,0,0,0,0
+Wewe ni mjinga na kipumbavu!,1,0,0,1,0
+Nitakupiga vibaya hadi ulie!,0,1,0,0,0
+Usiniletee upumbavu wako!,1,0,0,0,0
+Wewe ni bure kama taka!,1,0,0,1,0
+Wewe ni kipande cha mbwa!,1,0,0,1,0
+Nitaharibu kila kitu kwako!,0,1,1,0,0
+Wewe ni mfano wa upumbavu!,1,0,0,0,0
+Unakera kama mbu!,1,0,0,1,0
+Wewe ni shetani!,1,0,0,0,0
+Nitakutoa roho!,0,1,1,0,0
+Hakuna mtu anataka kukuona!,1,0,0,0,0
+Wewe ni mbwa mwitu!,1,0,0,0,0
+Nitakupiga ngumi!,0,1,0,0,0
+Wewe ni sumu!,1,0,0,0,0
+Nitachoma nyumba yako!,0,1,1,0,0
+Wewe ni mfano wa ujinga!,1,0,0,0,0
+Wewe ni kama kinyesi!,1,0,0,1,0
+Nitaharibu maisha yako!,0,1,1,0,0
+Wewe ni bure kabisa!,1,0,0,1,0
+Wewe ni kama panya!,1,0,0,0,0
+Nakuona kama panya!,1,0,0,1,0
+Wewe ni kipande cha mavi!,1,0,0,1,0
+Wewe ni chizi!,1,0,0,0,0
+Wewe ni mshenzi!,1,0,0,1,0
+Nitakupiga teke!,0,1,0,0,0
+Wewe ni bure kama takataka!,1,0,0,1,0
+Wewe ni kama kuku wa mtaani!,1,0,0,0,0
+Nitakukata kichwa!,0,1,1,0,0
+Wewe ni bure kabisa kama takataka!,1,0,0,1,0
+Wewe ni mwehu!,1,0,0,0,0
+Nitakutupa jalalani!,0,1,1,0,0
+Wewe ni kipande cha taka!,1,0,0,1,0
+Wewe ni kama mbwa mwitu!,1,0,0,0,0
+Wewe ni mfano wa pumbavu!,1,0,0,0,0
+Nitakupiga teke hadi ufe!,0,1,0,0,0
+Wewe ni mfano wa upuuzi!,1,0,0,0,0
+Wewe ni kipande cha taka!,1,0,0,1,0
+Wewe ni bure kama kinyesi!,1,0,0,1,0
+Wewe ni mnyama!,1,0,0,0,0
+Nitakupiga hadi utoe machozi!,0,1,0,0,0
+Wewe ni kichekesho cha dunia!,1,0,0,1,0
+Wewe ni takataka ya mtaa!,1,0,0,1,0
+Nitakupiga na risasi!,0,1,1,0,0
+Wewe ni mfano wa upumbavu!,1,0,0,0,0
+Wewe ni kama mbwa!,1,0,0,0,0
+Nitaharibu uso wako!,0,1,1,0,0
+Wewe ni kama paka wa mtaani!,1,0,0,0,0
+Nitakufanya ujutie kuzaliwa kwako!,0,1,1,0,0
+Wewe ni kipande cha mbwa!,1,0,0,1,0
+Nitakupiga vibaya hadi ulie!,0,1,0,0,0
+Wewe ni kipande cha takataka!,1,0,0,1,0
+Wewe ni sumu kwa jamii!,1,0,0,0,0
+Nitakuchoma moto!,0,1,1,0,0
+Wewe ni kichekesho cha mtaa!,1,0,0,1,0
+Wewe ni kama kinyesi cha mbwa!,1,0,0,1,0
+Nitakuvunja miguu!,0,1,1,0,0
+Wewe ni takataka kabisa!,1,0,0,1,0
+Wewe ni kama mavi ya mbwa!,1,0,0,1,0
+Nitakupiga hadi utoe damu!,0,1,0,0,0
+Ukiniletea mamdomo mi ntakumada,1,1,1,1,1
+Madem wa Kenya ni malaya,1,1,1,1,1
+Uskii jamaa anatupa haga,1,1,0,1,0
+Sura kama punda,1,0,0,1,0
+Manze ngeus ni nyangumi,1,1,0,1,1
+Si unakaa poa aki,0,0,0,0,0
+Huyo chali ni mpoa,0,0,0,0,0
+Ngeus amenidrive crazy,0,0,0,0,0
+Johnte ni fala achana tu na yeye,1,0,1,1,1
+Nairobii kumejaa malaya,1,1,1,1,1
+Ngeus wake wake ni hoe,1,1,0,1,1
+Dem wake ni whore,1,0,0,,1
+Manze fuck.,0,1,0,0,0
+Jamaa anapenda nginyo sana,1,1,0,1,0
+Jamaa anastream gwati sana,1,0,1,1,1
+Njoti yake ni kama maziwa mala,1,1,0,1,1
+

tokenizer.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d77d70fdcb351caea5ee6d9dfbd607f61ac419b4a04ec521d84605bbc9f41165
+size 7740158

toxic.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73b4cc477172e73a10c43cc26bc3c628a71b2a2a6c145b5edf6a8ce42d4905e8
+size 1816640

train.ipynb

@@ -0,0 +1,240 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Add Additional Datasets to The Model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "# from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
+    "import tensorflow as tf\n",
+    "import pickle"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>comment</th>\n",
+       "      <th>toxic</th>\n",
+       "      <th>severe_toxic</th>\n",
+       "      <th>obscene threat</th>\n",
+       "      <th>insult</th>\n",
+       "      <th>identity_hate</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Huyu msee ni malaya</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Zakayo ni mbwa</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Chira ni shoga</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Akinirushia mamdomo mi ntamuua</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Haga sana</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>Kuma ya mamako</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>Wewe ni mjinga sana!</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>Nataka nikuchape vibaya!</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>Huo ni upuuzi mtupu!</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>Wewe ni takataka!</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                          comment  toxic  severe_toxic  obscene threat  \\\n",
+       "0             Huyu msee ni malaya      1             0               0   \n",
+       "1                  Zakayo ni mbwa      0             0               0   \n",
+       "2                  Chira ni shoga      1             1               0   \n",
+       "3  Akinirushia mamdomo mi ntamuua      1             1               1   \n",
+       "4                       Haga sana      0             0               0   \n",
+       "5                  Kuma ya mamako      1             1               0   \n",
+       "6            Wewe ni mjinga sana!      1             0               0   \n",
+       "7        Nataka nikuchape vibaya!      0             1               0   \n",
+       "8            Huo ni upuuzi mtupu!      1             0               0   \n",
+       "9               Wewe ni takataka!      0             0               0   \n",
+       "\n",
+       "   insult  identity_hate  \n",
+       "0     1.0              0  \n",
+       "1     1.0              0  \n",
+       "2     1.0              0  \n",
+       "3     0.0              0  \n",
+       "4     1.0              0  \n",
+       "5     1.0              0  \n",
+       "6     1.0              0  \n",
+       "7     0.0              0  \n",
+       "8     0.0              0  \n",
+       "9     0.0              1  "
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataframe = pd.read_csv('swahili.csv')\n",
+    "\n",
+    "texts = dataframe['comment'].values\n",
+    "labels = dataframe[['toxic', 'severe_toxic', 'obscene threat', 'insult', 'identity_hate']].values\n",
+    "\n",
+    "dataframe.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Preprocess and Prepare Data for Training:\n",
+    "max_len = 200\n",
+    "\n",
+    "# Load the tokenizer\n",
+    "with open('tokenizer.pickle', 'rb') as handle:\n",
+    "    tokenizer = pickle.load(handle)\n",
+    "\n",
+    "# Pad & Tokenize Data\n",
+    "sequences = tokenizer.texts_to_sequences(texts)\n",
+    "padded_sequences = pad_sequences(sequences, maxlen=max_len)\n",
+    "\n",
+    "# Data spliting\n",
+    "X_train, X_val, y_train, y_val = train_test_split(padded_sequences, labels, test_size=0.2, random_state=42)\n",
+    "\n",
+    "# Create TensorFlow datasets\n",
+    "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)).batch(32)\n",
+    "val_dataset = tf.data.Dataset.from_tensor_slices((X_val, y_val)).batch(32)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}