app init

app.py

@@ -0,0 +1,52 @@
+import streamlit as st
+from beluga import load_model, process_emotions, generate_prompt
+from emodeepface import check_image_rotation, process_photo
+
+@st.cache_resource
+def load_cached_model():
+    return load_model()
+
+if 'model' not in st.session_state:
+    loading_message = st.empty()
+    loading_message.text("Loading model... Please wait.")
+    
+    st.session_state.model, st.session_state.tokenizer = load_cached_model()
+    loading_message.empty()
+
+st.title("Affective Journaling Assistant")
+
+st.write("""
+Welcome to the Affective Journaling Assistant!
+
+For a tailored journaling experience, we analyze your facial expressions to gauge your emotions.
+To proceed:
+
+1. Ensure the image is well-lit and of high quality.
+2. Your face should be fully visible without obstructions (e.g., no sunglasses or hats).
+3. By uploading, you acknowledge and consent to our data processing.
+
+Let's get started!
+""")
+
+file_name = st.file_uploader("Please upload your photo.")
+
+if file_name is not None:
+    image = check_image_rotation(file_name)
+    
+    processing_message = st.empty()
+    processing_message.text("Analyzing your image... Please wait.")
+    
+    emotion_predictions = process_photo(file_name) 
+    result = process_emotions(st.session_state.model, st.session_state.tokenizer, emotion_predictions)
+    processing_message.empty()
+    
+    prompt = generate_prompt(result)
+
+    # Create columns to place the image and the prompt side by side
+    col1, col2 = st.columns(2)
+
+    # Show image in the left column
+    col1.image(image, width=300)  
+
+    # Show generated prompt in the right column
+    col2.write(prompt)

beluga.py

@@ -0,0 +1,44 @@
+# import required libraries
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def load_model():
+    # define the tokenizer for preprocessing any inputs
+    tokenizer = AutoTokenizer.from_pretrained("stabilityai/StableBeluga-7B", use_fast=False)
+    # define the LLM model for the generating output
+    model = AutoModelForCausalLM.from_pretrained("stabilityai/StableBeluga-7B", torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
+    return (model, tokenizer)
+
+def process_emotions(model, tokenizer, emotion_data):
+    # ensure emotion_data is a dictionary
+    if not isinstance(emotion_data, dict):
+        raise ValueError("Expected emotion_data to be a dictionary.")
+    # convert emotion_data dictionary to string format
+    emotions = ', '.join(f"{k}: {v:.2f}%" for k, v in emotion_data.items())
+    # prompt-engineering instead of fine-tuning
+    system_prompt = """### System:
+    You are StableBeluga, an AI programmed to follow instructions with high accuracy. Your current task is to assist with enhancing human emotional regulation and intelligence. You will receive information about the emotional ratios of a client. Your task is to identify the one or two emotions with the highest values from the information provided, and create a one or two-sentence original journal prompt for the client's self-reflection and emotional exploration. Do not discuss the emotions or their scores, and do not include any other introductory or explanatory text. Only provide the journal prompt, and ensure it is based on the emotions with the highest values. Here are examples of the desired format:
+
+    Example 1: 'Reflect on a recent situation when you felt anger. What was the trigger and how did it affect your actions?'
+
+    Example 2: 'Write about a moment when you felt sad. What thoughts or events led to this emotion and how did it impact your day?'
+
+    Provide a prompt following this format, based on the emotions with the highest values from the data provided.
+    """
+    # user input will come from deepface FER results
+    prompt = f"{system_prompt}### User: {emotions}\n\n### Assistant:\n"
+    # preprocess the inputs
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+    output = model.generate(**inputs, do_sample=True, top_p=0.95, top_k=0, max_new_tokens=256)
+    result = tokenizer.decode(output[0], skip_special_tokens=True)
+    # return decoded output
+    return result
+
+def generate_prompt(result):
+    # split the text by the "Assistant:" label
+    response = result.split("### Assistant:")[1].strip()
+    # check if response includes uneeded prompt indifier (e.g., Journal prompt: ..., etc.)
+    if ":" in response:
+        split_response = response.split(":", 1)
+        response = split_response[1].strip()
+    return response

emodeepface.py

@@ -0,0 +1,55 @@
+from deepface import DeepFace
+import numpy as np
+from PIL import Image, ExifTags
+
+# used to raise custom exceptions
+class ImageProcessingError(Exception):
+    pass
+
+# used to ensure upright photo orientation for face detection
+def check_image_rotation(image_path):
+    try:
+        image = Image.open(image_path)
+        for orientation in ExifTags.TAGS.keys():
+            if ExifTags.TAGS[orientation] == 'Orientation':
+                break
+        exif = image._getexif()
+        if exif is not None:
+            if orientation in exif:
+                if exif[orientation] == 3:
+                    image = image.rotate(180, expand=True)
+                elif exif[orientation] == 6:
+                    image = image.rotate(270, expand=True)
+                elif exif[orientation] == 8:
+                    image = image.rotate(90, expand=True)
+        return image
+    except (AttributeError, KeyError, IndexError):
+        # If the orientation tag is not found or any other error occurs, return the original image
+        return Image.open(image_path)
+
+# used to process photo through deepface emotion model
+def process_photo(file_name):
+    backends = ['opencv', 'mtcnn', 'retinaface', 'mediapipe', 'ssd']
+    attempt = 0
+
+    image = check_image_rotation(file_name)
+    image_data = np.array(image)
+
+    while attempt < len(backends):
+        try:
+            predictions = DeepFace.analyze(image_data, actions=['emotion'], detector_backend=backends[attempt])
+            print("detector backend:", backends[attempt])
+            if len(predictions) > 1:
+                faces = [(face, face['region']['w'] * face['region']['h']) for face in predictions]
+                new_predictions = sorted(faces, key=lambda x: x[1], reverse=True)[0][0]
+                emotion_dict = new_predictions['emotion']
+                return emotion_dict
+            return predictions[0]['emotion']
+        except Exception as e:
+            if attempt == len(backends) - 1:
+                error_message = f"Failed to analyze image after attempting all detector backends available. Please upload a new image."
+                raise ImageProcessingError(error_message)
+            else:
+                # log the error message for each failed backend here:
+                print(f"Retrying with backend `{backends[attempt+1]}` due to error: {str(e)}")
+                attempt += 1

requirements.txt

@@ -0,0 +1,6 @@
+torch
+transformers
+sentencepiece
+accelerate
+deepface
+mediapipe