p4 early warning signs q8a demo app on hf spaces

notebooks/zero-shot-classification-pbsp-q8a-azure-gpt35-eval-demo-hf.ipynb

@@ -0,0 +1,507 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f56cc5ad",
+   "metadata": {},
+   "source": [
+    "# NDIS Project - PBSP Scoring - Quality Marker Q8a - Early Warning Signs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a8d844ea",
+   "metadata": {
+    "hide_input": false
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from ipywidgets import interact\n",
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display, clear_output, Javascript, HTML\n",
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.ticker as mtick\n",
+    "import json\n",
+    "import spacy\n",
+    "from spacy import displacy\n",
+    "import nltk\n",
+    "from nltk import sent_tokenize\n",
+    "from sklearn.feature_extraction import text\n",
+    "from pprint import pprint\n",
+    "import re\n",
+    "from sentence_transformers import SentenceTransformer, util\n",
+    "import pandas as pd\n",
+    "import argilla as rg\n",
+    "from argilla.metrics.text_classification import f1\n",
+    "from setfit import SetFitModel\n",
+    "import joblib\n",
+    "import random\n",
+    "from typing import Dict\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "%matplotlib inline\n",
+    "from tqdm import tqdm\n",
+    "import time\n",
+    "for i in tqdm(range(15), disable=True):\n",
+    "    time.sleep(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96b83a1d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#initializations\n",
+    "model_name = \"setfit-zero-shot-classification-pbsp-q8a-azure-gpt35\"\n",
+    "model = SetFitModel.from_pretrained(f\"aammari/{model_name}\")\n",
+    "\n",
+    "# download nltk 'punkt' if not available\n",
+    "try:\n",
+    "    nltk.data.find('tokenizers/punkt')\n",
+    "except LookupError:\n",
+    "    nltk.download('punkt')\n",
+    "\n",
+    "# download nltk 'averaged_perceptron_tagger' if not available\n",
+    "try:\n",
+    "    nltk.data.find('taggers/averaged_perceptron_tagger')\n",
+    "except LookupError:\n",
+    "    nltk.download('averaged_perceptron_tagger')\n",
+    "    \n",
+    "#argilla\n",
+    "rg.init(\n",
+    "    api_url=os.environ[\"ARGILLA_API_URL\"],\n",
+    "    api_key=os.environ[\"ARGILLA_API_KEY\"]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72c2c6f9",
+   "metadata": {
+    "hide_input": false
+   },
+   "outputs": [],
+   "source": [
+    "#Text Preprocessing\n",
+    "try:\n",
+    "    nlp = spacy.load('en_core_web_sm')\n",
+    "except OSError:\n",
+    "    spacy.cli.download('en_core_web_sm')\n",
+    "    nlp = spacy.load('en_core_web_sm')\n",
+    "sw_lst = text.ENGLISH_STOP_WORDS\n",
+    "def preprocess(onto_lst):\n",
+    "    cleaned_onto_lst = []\n",
+    "    pattern = re.compile(r'^[a-z ]*$')\n",
+    "    for document in onto_lst:\n",
+    "        text = []\n",
+    "        doc = nlp(document)\n",
+    "        person_tokens = []\n",
+    "        for w in doc:\n",
+    "            if w.ent_type_ == 'PERSON':\n",
+    "                person_tokens.append(w.lemma_)\n",
+    "        for w in doc:\n",
+    "            if not w.is_stop and not w.is_punct and not w.like_num and not len(w.text.strip()) == 0 and not w.lemma_ in person_tokens:\n",
+    "                text.append(w.lemma_.lower())\n",
+    "        texts = [t for t in text if len(t) > 1 and pattern.search(t) is not None and t not in sw_lst]\n",
+    "        cleaned_onto_lst.append(\" \".join(texts))\n",
+    "    return cleaned_onto_lst"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40070313",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#sentence extraction\n",
+    "def extract_sentences(nltk_query):\n",
+    "    sentences = sent_tokenize(nltk_query)\n",
+    "    return sentences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1550437b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#query and get predicted topic\n",
+    "\n",
+    "def get_topic(sentences):\n",
+    "    preds = list(model(sentences))\n",
+    "    return preds\n",
+    "def get_topic_scores(sentences):\n",
+    "    preds = model.predict_proba(sentences)\n",
+    "    preds = [max(list(x)) for x in preds]\n",
+    "    return preds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e99e5afc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def convert_df(result_df):\n",
+    "    new_df = pd.DataFrame(columns=['text', 'prediction'])\n",
+    "    new_df['text'] = result_df['Answer']\n",
+    "    new_df['prediction'] = result_df.apply(lambda row: [[row['Topic'], row['Score']]], axis=1)\n",
+    "    return new_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd6f85fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def custom_f1(data: Dict[str, float], title: str):\n",
+    "    from plotly.subplots import make_subplots\n",
+    "    import plotly.colors\n",
+    "    import random\n",
+    "\n",
+    "    fig = make_subplots(\n",
+    "        rows=2,\n",
+    "        cols=1,\n",
+    "        subplot_titles=[        \"Overall Model Score\",        \"Model Score By Category\",    ],\n",
+    "    )\n",
+    "\n",
+    "    x = ['precision', 'recall', 'f1']\n",
+    "    macro_data = [v for k, v in data.items() if \"macro\" in k]\n",
+    "    fig.add_bar(\n",
+    "        x=x,\n",
+    "        y=macro_data,\n",
+    "        row=1,\n",
+    "        col=1,\n",
+    "    )\n",
+    "    per_label = {\n",
+    "        k: v\n",
+    "        for k, v in data.items()\n",
+    "        if all(key not in k for key in [\"macro\", \"micro\", \"support\"])\n",
+    "    }\n",
+    "\n",
+    "    num_labels = int(len(per_label.keys())/3)\n",
+    "    fixed_colors = [str(color) for color in plotly.colors.qualitative.Plotly]\n",
+    "    colors = random.sample(fixed_colors, num_labels)\n",
+    "\n",
+    "    fig.add_bar(\n",
+    "        x=[k for k, v in per_label.items()],\n",
+    "        y=[v for k, v in per_label.items()],\n",
+    "        row=2,\n",
+    "        col=1,\n",
+    "        marker_color=[colors[int(i/3)] for i in range(0, len(per_label.keys()))]\n",
+    "    )\n",
+    "    fig.update_layout(showlegend=False, title_text=title)\n",
+    "\n",
+    "    return fig"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e28f99b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_null_class_df(sentences, result_df):\n",
+    "    sents = result_df['Answer'].tolist()\n",
+    "    null_sents = [x for x in sentences if x not in sents]\n",
+    "    topics = ['NONE'] * len(null_sents)\n",
+    "    scores = [0.90] * len(null_sents)\n",
+    "    null_df = pd.DataFrame({'Answer': null_sents, 'Topic': topics, 'Score': scores})\n",
+    "    return null_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "02fda761",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# format output\n",
+    "ind_topic_dict = {\n",
+    "        0: 'NONE',\n",
+    "        1: 'EARLY WARNING'\n",
+    "    }\n",
+    "\n",
+    "topic_color_dict = {\n",
+    "        'NONE': '#FFB6C1',\n",
+    "        'EARLY WARNING': '#90EE90'\n",
+    "    }\n",
+    "\n",
+    "def color(df, color):\n",
+    "    return df.style.format({'Score': '{:,.2%}'.format}).bar(subset=['Score'], color=color)\n",
+    "\n",
+    "def annotate_query(highlights, query, topics):\n",
+    "    ents = []\n",
+    "    for h, t in zip(highlights, topics):\n",
+    "        ent_dict = {}\n",
+    "        for match in re.finditer(h, query):\n",
+    "            ent_dict = {\"start\": match.start(), \"end\": match.end(), \"label\": t}\n",
+    "            break\n",
+    "        if len(ent_dict.keys()) > 0:\n",
+    "            ents.append(ent_dict)\n",
+    "    return ents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "905eaf2a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def path_to_image_html(path):\n",
+    "    return '<img src=\"'+ path + '\" width=\"30\" height=\"15\" />'\n",
+    "\n",
+    "final_passing = 0.0\n",
+    "def display_final_df(agg_df):\n",
+    "    tags = []\n",
+    "    crits = [\n",
+    "            'EARLY WARNING'\n",
+    "        ]\n",
+    "    orig_crits = crits\n",
+    "    crits = [x for x in crits if x in agg_df.index.tolist()]\n",
+    "    bools = [agg_df.loc[crit, 'Final_Score'] > final_passing for crit in crits]\n",
+    "    paths = ['./thumbs_up.png' if x else './thumbs_down.png' for x in bools]\n",
+    "    df = pd.DataFrame({'Life Event': crits, 'USED': paths})\n",
+    "    rem_crits = [x for x in orig_crits if x not in crits]\n",
+    "    if len(rem_crits) > 0:\n",
+    "        df2 = pd.DataFrame({'Life Event': rem_crits, 'USED': ['./thumbs_down.png'] * len(rem_crits)})\n",
+    "        df = pd.concat([df, df2])\n",
+    "    df = df.set_index('Life Event')\n",
+    "    pd.set_option('display.max_colwidth', None)\n",
+    "    display(HTML('<div style=\"text-align: center;\">' + df.to_html(classes=[\"align-center\"], index=True, escape=False ,formatters=dict(USED=path_to_image_html)) + '</div>'))\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c6e9fe7",
+   "metadata": {},
+   "source": [
+    "### Question:\n",
+    "\n",
+    "#### Please outline the strategies that will be implemented to de-escalate the focus person’s target behaviour(s). \n",
+    "##### This section should focus on: Strategies to address early warning signs.\n",
+    "\n",
+    "### Quality Marker Q8a:\n",
+    "\n",
+    "#### A set of early warning signs for the target behaviour(s) are identified for the purposes of de-escalation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76dd8cab",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "#demo with Voila\n",
+    "\n",
+    "bhvr_label = widgets.Label(value='Please type your answer:')\n",
+    "bhvr_text_input = widgets.Textarea(\n",
+    "    value='',\n",
+    "    placeholder='Type your answer',\n",
+    "    description='',\n",
+    "    disabled=False,\n",
+    "    layout={'height': '300px', 'width': '90%'}\n",
+    ")\n",
+    "\n",
+    "bhvr_nlp_btn = widgets.Button(\n",
+    "    description='Score Answer',\n",
+    "    disabled=False,\n",
+    "    button_style='success', # 'success', 'info', 'warning', 'danger' or ''\n",
+    "    tooltip='Score Answer',\n",
+    "    icon='check',\n",
+    "    layout={'height': '70px', 'width': '250px'}\n",
+    ")\n",
+    "bhvr_agr_btn = widgets.Button(\n",
+    "    description='Validate Data',\n",
+    "    disabled=False,\n",
+    "    button_style='success', # 'success', 'info', 'warning', 'danger' or ''\n",
+    "    tooltip='Validate Data',\n",
+    "    icon='check',\n",
+    "    layout={'height': '70px', 'width': '250px'}\n",
+    ")\n",
+    "bhvr_eval_btn = widgets.Button(\n",
+    "    description='Evaluate Model',\n",
+    "    disabled=False,\n",
+    "    button_style='success', # 'success', 'info', 'warning', 'danger' or ''\n",
+    "    tooltip='Evaluate Model',\n",
+    "    icon='check',\n",
+    "    layout={'height': '70px', 'width': '250px'}\n",
+    ")\n",
+    "btn_box = widgets.HBox([bhvr_nlp_btn, bhvr_agr_btn, bhvr_eval_btn], \n",
+    "                       layout={'width': '100%', 'height': '160%'})\n",
+    "bhvr_outt = widgets.Output()\n",
+    "bhvr_outt.layout.height = '100%'\n",
+    "bhvr_outt.layout.width = '100%'\n",
+    "bhvr_box = widgets.VBox([bhvr_text_input, btn_box, bhvr_outt], \n",
+    "                   layout={'width': '100%', 'height': '160%'})\n",
+    "dataset_rg_name = 'pbsp-page4-q8a-argilla-ds'\n",
+    "agrilla_df = None\n",
+    "annotated = False\n",
+    "def on_bhvr_button_next(b):\n",
+    "    global agrilla_df\n",
+    "    with bhvr_outt:\n",
+    "        clear_output()\n",
+    "        query = bhvr_text_input.value\n",
+    "        fake_query = 'This is just a fake query to avoid ValueError'\n",
+    "        sentences = extract_sentences(query)\n",
+    "        cl_sentences = preprocess(sentences)\n",
+    "        topic_inds = get_topic([query, fake_query])\n",
+    "        topics = [ind_topic_dict[i] for i in topic_inds]\n",
+    "        scores = get_topic_scores([query, fake_query])\n",
+    "        result_df = pd.DataFrame({'Answer': [query, fake_query], 'Topic': topics, 'Score': scores})\n",
+    "        sub_result_df = result_df[result_df['Topic'] != 'NONE']\n",
+    "        sub_2_result_df = get_null_class_df([query, fake_query], sub_result_df)\n",
+    "        highlights = []\n",
+    "        if len(sub_result_df) > 0:\n",
+    "            highlights = sub_result_df['Answer'].tolist()\n",
+    "            highlight_topics = sub_result_df['Topic'].tolist()    \n",
+    "            ents = annotate_query(highlights, query, highlight_topics)\n",
+    "            colors = {}\n",
+    "            for ent, ht in zip(ents, highlight_topics):\n",
+    "                colors[ent['label']] = topic_color_dict[ht]\n",
+    "\n",
+    "            ex = [{\"text\": query,\n",
+    "                   \"ents\": ents,\n",
+    "                   \"title\": None}]\n",
+    "            title = \"Warning Sign Highlights\"\n",
+    "            display(HTML(f'<center><h1>{title}</h1></center>'))\n",
+    "            html = displacy.render(ex, style=\"ent\", manual=True, jupyter=True, options={'colors': colors})\n",
+    "            display(HTML(html))\n",
+    "            title = \"Warning Sign Classifications\"\n",
+    "            display(HTML(f'<center><h1>{title}</h1></center>'))\n",
+    "            for top in topic_color_dict.keys():\n",
+    "                top_result_df = sub_result_df[sub_result_df['Topic'] == top]\n",
+    "                if len(top_result_df) > 0:\n",
+    "                    top_result_df = top_result_df.sort_values(by='Score', ascending=False).reset_index(drop=True)\n",
+    "                    top_result_df = top_result_df.set_index('Answer')\n",
+    "                    top_result_df = top_result_df[['Score']]\n",
+    "                    display(HTML(\n",
+    "                        f'<left><h2 style=\"text-decoration: underline; text-decoration-color:{topic_color_dict[top]};\">{top}</h2></left>'))\n",
+    "                    display(color(top_result_df, topic_color_dict[top]))\n",
+    "            \n",
+    "            agg_df = sub_result_df.groupby('Topic')['Score'].sum()\n",
+    "            agg_df = agg_df.to_frame()\n",
+    "            agg_df.index.name = 'Topic'\n",
+    "            agg_df.columns = ['Total Score']\n",
+    "            agg_df = agg_df.assign(\n",
+    "                Final_Score=lambda x: x['Total Score'] / x['Total Score'].sum() * 100.00\n",
+    "            )\n",
+    "            agg_df = agg_df.sort_values(by='Final_Score', ascending=False)\n",
+    "            agg_df['Topic'] = agg_df.index\n",
+    "            rem_topics= [x for x in list(topic_color_dict.keys()) if not x in agg_df.Topic.tolist()]\n",
+    "            if len(rem_topics) > 0:\n",
+    "                rem_agg_df = pd.DataFrame({'Topic': rem_topics, 'Final_Score': 0.0, 'Total Score': 0.0})\n",
+    "                agg_df = pd.concat([agg_df, rem_agg_df])\n",
+    "            title = \"Final Scores\"\n",
+    "            display(HTML(f'<left><h1>{title}</h1></left>'))\n",
+    "            display_final_df(agg_df)\n",
+    "            if len(sub_2_result_df) > 0:\n",
+    "                sub_result_df = pd.concat([sub_result_df, sub_2_result_df]).reset_index(drop=True)\n",
+    "            agrilla_df = sub_result_df.copy()\n",
+    "        else:\n",
+    "            print(query)\n",
+    "\n",
+    "def on_agr_button_next(b):\n",
+    "    global agrilla_df, annotated\n",
+    "    with bhvr_outt:\n",
+    "        clear_output()\n",
+    "        if agrilla_df is not None:\n",
+    "            # convert the dataframe to the structure accepted by argilla\n",
+    "            converted_df = convert_df(agrilla_df)\n",
+    "            # convert pandas dataframe to DatasetForTextClassification\n",
+    "            dataset_rg = rg.DatasetForTextClassification.from_pandas(converted_df)\n",
+    "            # delete the old DatasetForTextClassification from the Argilla web app if exists\n",
+    "            rg.delete(dataset_rg_name, workspace=\"admin\")\n",
+    "            # load the new DatasetForTextClassification into the Argilla web app\n",
+    "            rg.log(dataset_rg, name=dataset_rg_name, workspace=\"admin\")\n",
+    "            # Make sure all classes are present for annotation\n",
+    "            rg_settings = rg.TextClassificationSettings(label_schema=list(topic_color_dict.keys()))\n",
+    "            rg.configure_dataset(name=dataset_rg_name, workspace=\"admin\", settings=rg_settings)\n",
+    "            annotated = True\n",
+    "        else:\n",
+    "            display(Markdown(\"<h2 style='color:red; text-align:center;'>Please score the answer first!</h2>\"))\n",
+    "            \n",
+    "def on_eval_button_next(b):\n",
+    "    global annotated\n",
+    "    with bhvr_outt:\n",
+    "        clear_output()\n",
+    "        if annotated:\n",
+    "            data = dict(f1(dataset_rg_name))['data']\n",
+    "            display(custom_f1(data, \"Model Evaluation Results\"))\n",
+    "        else:\n",
+    "            display(Markdown(\"<h2 style='color:red; text-align:center;'>Please score the answer and validate the data first!</h2>\"))\n",
+    "\n",
+    "bhvr_nlp_btn.on_click(on_bhvr_button_next)\n",
+    "bhvr_agr_btn.on_click(on_agr_button_next)\n",
+    "bhvr_eval_btn.on_click(on_eval_button_next)\n",
+    "\n",
+    "display(bhvr_label, bhvr_box)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ed551eba",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "hide_input": false,
+  "kernelspec": {
+   "display_name": "Python 3.9 (Argilla)",
+   "language": "python",
+   "name": "argilla"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": false,
+   "sideBar": true,
+   "skip_h1_title": true,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {
+    "height": "calc(100% - 180px)",
+    "left": "10px",
+    "top": "150px",
+    "width": "258.097px"
+   },
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}