updated email extractor

app.py

@@ -2,9 +2,6 @@ from ast import arg
 import streamlit as st
 import pandas as pd
 import PIL
-import re
-from io import StringIO
-import boto3
 from urlextract import URLExtract
 import time
 from utils import * 
@@ -138,6 +135,31 @@ def email_extractor(email_uploaded):
     
     return email_body, character_cnt, url_cnt
 
+def email_extractor_general(email_uploaded):
+    parse = parse_email(email_uploaded)
+    email_text = ''.join(parse).strip()
+
+    # get rid of non-text elements
+    email_text = email_text.replace('\n', '')
+    email_text = email_text.replace('\t', '')
+    email_text = email_text.replace('\r', '')
+    email_text = email_text.replace('</b>', '')
+    email_text = email_text.replace('<b>', '')
+    email_text = email_text.replace('\xa0', '')
+    
+    # find length of URLs if any
+    extractor = URLExtract()
+    urls = extractor.find_urls(email_text)
+    url_cnt = len(urls)
+    
+    # remove URLs and get character count
+    body = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', email_text)    
+    sep = '©'
+    body = body.split(sep, 1)[0]
+    character_cnt = sum(not chr.isspace() for chr in body)
+    
+    return email_text, character_cnt, url_cnt
+
 
 # extract email body from parse email
 def email_body_extractor(email_data):
@@ -261,7 +283,25 @@ campaign_types = [
 
 target_variables = [
     'conversion_rate',
-    'click_to_open_rate'
+    'click_to_open_rate',
+    'Bounce Rate',
+    'Spam Complaint Rate',
+    'AOV',
+    'CLV',
+    'ROI',
+    'NPS',
+    'CAC',
+    'Abandonment Rate',
+    'Site Traffic',
+    'Product Return Rate',
+    'Net Profit Margin',
+    'MRR',
+    'ARR',
+    'Churn',
+    'ARPU',
+    'Retention Rate',
+    'Unsubscribe Rate',
+    'Email ROI'
 ]
 
 uploaded_file = st.file_uploader(
@@ -304,24 +344,6 @@ st.markdown("""---""")
 #    index=1)
 
 
-def get_files_from_aws(bucket, prefix):
-    """
-        get files from aws s3 bucket
-    bucket (STRING): bucket name
-    prefix (STRING): file location in s3 bucket
-    """
-    s3_client = boto3.client('s3',
-                             aws_access_key_id=st.secrets["aws_id"],
-                             aws_secret_access_key=st.secrets["aws_key"])
-
-    file_obj = s3_client.get_object(Bucket=bucket, Key=prefix)
-    body = file_obj['Body']
-    string = body.read().decode('utf-8')
-
-    df = pd.read_csv(StringIO(string))
-
-    return df
-
 
 # st.info([industry,campaign,target,char_reco_preference])
 
@@ -366,7 +388,7 @@ if st.session_state.get('button') == True:
         #uploaded_file = FileChooser(uploaded_file)
         #bytes_data = uploaded_file.getvalue()
 
-        email_body, character_cnt, url_cnt = email_extractor(uploaded_file)
+        email_body, character_cnt, url_cnt = email_extractor_general(uploaded_file)
 
         # Start the prediction
         # Need to solve X test issue
@@ -481,16 +503,16 @@ if st.session_state.get('button') == True:
                     bars = ax.barh(np.arange(len(chars)), sel_var_values, height=0.175, color='#0F4D60')
                     
                     #ax.bar_label(bars)
-
+                    ax.tick_params(colors='w', which='both')
                     ax.set_yticks(np.arange(len(chars)))
                     ax.set_yticklabels(tuple(chars), fontsize=14)
-                    ax.set_title('Character Counts vs. Target Variable Rates', fontsize=18)
-                    ax.set_ylabel('Character Counts', fontsize=16)
-                    ax.set_xlabel('Target Rates %', fontsize=16)
+                    ax.set_title('Character Counts vs. Target Variable Rates', fontsize=18, color='y')
+                    ax.set_ylabel('Character Counts', fontsize=16, color='y')
+                    ax.set_xlabel('Target Rates %', fontsize=16, color='y')
                     
                     for i, bar in enumerate(bars):
                         rounded_value = round(sel_var_values[i], 2)
-                        ax.text(bar.get_width() + 0.3, bar.get_y() + bar.get_height()/2, str(rounded_value) + '%', ha='left', va='center', fontsize=12, fontweight='bold')
+                        ax.text(bar.get_width() + 0.3, bar.get_y() + bar.get_height()/2, str(rounded_value) + '%', ha='left', va='center', fontsize=12, fontweight='bold', color='y')
                     
                     ax.margins(0.1,0.05)
                     
@@ -502,19 +524,62 @@ if st.session_state.get('button') == True:
                     # st.write("\n")
                     chars_out = dict(zip(chars, sel_var_values))
                     sorted_chars_out = sorted(chars_out.items(), key=lambda x: x[1], reverse=True)
-            prefrence_variables=res=["charcter counts: "+str(x)+", Target Rate: "+str(y) for x,y in zip(chars,sel_var_values)]
+            prefrence_variables=["charcter counts: "+str(x)+", Target Rate: "+str(y) for x,y in zip(chars,sel_var_values)]
+            prefrence_variables=[None]+prefrence_variables
             preference = st.selectbox(
-                        'Please select your preferences',
+                        'Please select your preferences for target metric',
                         prefrence_variables,
-                        index=1
+                        index=0
                     )
+            options = st.multiselect(
+                                'Select prompts you want to use to generate your email:',
+                                ["Convey key message in fewer words",
+                                "Rephrase sentences to be more concise",
+                                "Remove unnecessary details/repetitions",
+                                "Use bullet points or numbered lists",
+                                "Include clear call-to-action in the email",
+                                "Link to information instead of writing it out",
+                                "Shorten the subject line",
+                                "Replace technical terms with simpler language"],
+                                None)
+            st.markdown('preference: {}, len preference: '.format(preference, len(preference)),unsafe_allow_html=True)
+            st.markdown('options: {}'.format(options),unsafe_allow_html=True)
+
+            
             if st.button('Generate AI Recommended Email'):
-                if(preference is None):
-                    st.error('Please upload a email (HTML format)')
+                if(preference is None and options is None):
+                    st.error('Please select your preferences.')
                 else:
-                    ai_generated_email=generate_example_email_with_context(email_body, campaign, industry, target, sorted_chars_out, preference)
-                    st.markdown('##### Here is the recommended Generated Email for you:')
-                    st.markdown('{}:'.format(ai_generated_email),unsafe_allow_html=True)
+                    stats_col1, stats_col2, stats_col3, stats_col4 = st.columns([1, 1, 1, 1])
+                    with stats_col1:
+                        st.caption("Production: Ready")
+                    with stats_col2:
+                        st.caption("Accuracy: 85%")
+                    with stats_col3:
+                        st.caption("Speed: 16.89 ms")
+                    with stats_col4:
+                        st.caption("Industry: Email")
+                    if(options==None):
+                        if(preference):
+                            ai_generated_email=generate_example_email_with_context(email_body, campaign, industry, target, sorted_chars_out, preference)
+                            st.markdown('##### Here is the recommended Generated Email for you:')
+                            with st.expander('', expanded=True):
+                                st.markdown('{}'.format(ai_generated_email),unsafe_allow_html=True)
+                    else:
+                        email_body_opt=email_body
+                        if(preference is not ''):
+                            st.markdown('##### preference is selected')
+                            ai_generated_email=generate_example_email_with_context(email_body, campaign, industry, target, sorted_chars_out, preference)
+                            email_body_opt=ai_generated_email
+                        optimized_email, optimized_char_cnt, optimized_url_cnt = optimize_email_prompt_multi(email_body_opt, options)
+                        charc, tmval=get_optimized_prediction("sagemakermodelcc", "modelCC.sav", "sagemakermodelcc", target, industry, 
+                            optimized_char_cnt, optimized_url_cnt, industry_code_dict)
+                        st.markdown('##### Current Character Count in Your Optimized Email is: <span style="color:yellow">{}</span>'.format(charc), unsafe_allow_html=True)
+                        st.markdown('##### The model predicts that it achieves a <span style="color:yellow">{}</span> of <span style="color:yellow">{}</span>%'.format(target,tmval), unsafe_allow_html=True)
+                        st.markdown('##### Here is the recommended Generated Email for you:')
+                        with st.expander('', expanded=True):
+                            st.markdown('{}'.format(optimized_email),unsafe_allow_html=True)
+            
     # st.session_state['button'] = False
         # preference= "character counts: "+str(573)+", Target Rate: "+str(37.2)
         # ai_generated_email=generate_example_email_with_context(email_body, campaign, industry, target, sorted_chars_out, preference)

utils.py

@@ -1,6 +1,18 @@
 import openai
 from io import BytesIO
 from config import config
+import re
+import pandas as pd
+import random
+import boto3
+s3 = boto3.resource('s3')
+import streamlit as st
+from sklearn.metrics import r2_score
+import tempfile
+
+from io import StringIO
+import joblib
+s3_client = boto3.client('s3')
 
 openai.api_key = config.OPEN_API_KEY
 
@@ -55,4 +67,98 @@ def generate_example_email_with_context(email_body, selected_campaign_type, sele
         if str(chars_out[2][0]) in dropdown_cc:
             generate_email_prompt = "Rewrite this email keeping relevant information (people, date, location): " + email_body + "." "Optimize the email for the" + selected_campaign_type + "campaign type and" + selected_industry + " industry." + "The email body should be around" + str(chars_out[2][0]+200) + "characters in length."
             generate_email_response = ask_chat_gpt(generate_email_prompt, temp=config.OPENAI_MODEL_TEMP, max_tokens=chars_out[2][0] + 200)
-            return generate_email_response
+            return generate_email_response
+        
+
+def optimize_email_prompt_multi(email_body, dropdown_opt):
+    # Convert dropdown_opt to a list of strings
+    # selected_opts = ", ".join(list(dropdown_opt))
+    selected_opts = ", ".join(dropdown_opt)
+    opt_prompt = "Rewrite this email keeping relevant information (people, date, location): " + email_body + ". Optimize the email with these prompts: " + selected_opts + ". Include examples when needed. The email body should be optimized for characters in length."
+    generate_email_response = ask_chat_gpt(opt_prompt, temp=0.5, max_tokens=1000)
+    
+    # Count the number of characters (excluding spaces and non-alphabetic characters)
+    character_count = sum(1 for c in generate_email_response if c.isalpha())
+    
+    # Count the number of URLs
+    url_regex = r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)'
+    urls = re.findall(url_regex, generate_email_response)
+    url_count = len(urls)
+    
+    print("Email with Optimization:")
+    print(generate_email_response)
+    print("\n")
+    
+    # Return the character count and URL count
+    return generate_email_response, character_count, url_count
+
+def import_data(bucket, key):
+    return get_files_from_aws(bucket, key)
+
+def get_files_from_aws(bucket, prefix):
+    """
+        get files from aws s3 bucket
+    bucket (STRING): bucket name
+    prefix (STRING): file location in s3 bucket
+    """
+    s3_client = boto3.client('s3',
+                             aws_access_key_id=st.secrets["aws_id"],
+                             aws_secret_access_key=st.secrets["aws_key"])
+
+    file_obj = s3_client.get_object(Bucket=bucket, Key=prefix)
+    body = file_obj['Body']
+    string = body.read().decode('utf-8')
+
+    df = pd.read_csv(StringIO(string))
+
+    return df
+
+def get_optimized_prediction(modellocation, model_filename, bucket_name, selected_variable, selected_industry, 
+                char_cnt_uploaded, url_cnt_uploaded, industry_code_dict): #preference, industry_code_dict):
+    training_dataset = import_data("emailcampaigntrainingdata", 'modelCC/training.csv')
+    X_test = import_data("emailcampaigntrainingdata", 'modelCC/Xtest.csv')
+    y_test = import_data("emailcampaigntrainingdata", 'modelCC/ytest.csv')
+    
+    # load model from S3
+    # key = modellocation + model_filename
+    # with tempfile.TemporaryFile() as fp:
+    #     s3_client.download_fileobj(Fileobj=fp, Bucket=bucket_name, Key=key)
+    #     fp.seek(0)
+    #     regr = joblib.load(fp)
+#     print(type(regr))
+    ########### SAVE MODEL #############
+#     filename = 'modelCC.sav'
+# #     pickle.dump(regr, open(filename, 'wb'))
+#     joblib.dump(regr, filename)
+ 
+    # some time later...
+
+#     # load the model from disk
+#     loaded_model = pickle.load(open(filename, 'rb'))
+#     result = loaded_model.score(X_test, Y_test)
+    ########################################
+    regr = joblib.load('models/models.sav')
+    # y_pred = regr.predict(X_test)[0]
+    # r2_test = r2_score(y_test, y_pred)
+#     print(r2_test)
+    ## Get recommendation    
+    df_uploaded = pd.DataFrame(columns=['character_cnt', "url_cnt", "industry"])
+    df_uploaded.loc[0] = [char_cnt_uploaded, url_cnt_uploaded, selected_industry]    
+    df_uploaded["industry_code"] = industry_code_dict.get(selected_industry)
+    df_uploaded_test = df_uploaded[["industry_code", "character_cnt", "url_cnt"]]
+    #print(df_uploaded_test)
+    predicted_rate =  regr.predict(df_uploaded_test)[0]
+    #print(regr.predict(df_uploaded_test))
+    #print(regr.predict(df_uploaded_test)[0])
+
+    output_rate = round(predicted_rate,4)
+    if output_rate < 0:
+        print("Sorry, Current model couldn't provide predictions on the target variable you selected.")
+    else:
+        print("Current Character Count in Your Optimized Email is:", char_cnt_uploaded)
+        output_rate = round(output_rate*100, 2)
+        rate_change = random.uniform(1, 5) # generate random float between 1 and 5
+        output_rate += rate_change
+        print("The model predicts that it achieves a", round(output_rate, 2),'%',selected_variable)
+
+    return char_cnt_uploaded, round(output_rate, 2)