update

app.py

@@ -34,10 +34,9 @@ from googleapiclient.http import MediaIoBaseUpload
 
 from educational_material import EducationalMaterial
 from storage_service import GoogleCloudStorage
-from google.cloud import aiplatform
-from vertexai.preview.generative_models import GenerativeModel
 from google.oauth2.service_account import Credentials
-
+import vertexai
+from vertexai.generative_models import GenerativeModel, Part
 
 import boto3
 
@@ -102,12 +101,11 @@ GOOGPE_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
 google_creds = Credentials.from_service_account_info(
     google_service_account_info_dict, scopes=GOOGPE_SCOPES
 )
-aiplatform.init(
+vertexai.init(
     project="junyiacademy",
     service_account=google_service_account_info_dict,
     credentials=google_creds,
 )
-GEMINI_MODEL = GenerativeModel("gemini-pro")
 
 # CLIENTS CONFIG
 GBQ_CLIENT = bigquery.Client.from_service_account_info(json.loads(GBQ_KEY))
@@ -789,7 +787,9 @@ def generate_content_by_open_ai(sys_content, user_content, response_format=None,
 def generate_content_by_gemini(sys_content, user_content, response_format=None, model_name=None):
     print("generate_content_by_gemini")
     print(f"LLM using: {model_name}")
-    model_response = GEMINI_MODEL.generate_content(
+
+    gemini_model = GenerativeModel(model_name=model_name)
+    model_response = gemini_model.generate_content(
         f"{sys_content}, {user_content}"
     )
     content = model_response.candidates[0].content.parts[0].text
@@ -799,8 +799,9 @@ def generate_content_by_gemini(sys_content, user_content, response_format=None,
 def generate_content_by_LLM(sys_content, user_content, response_format=None, LLM_model=None, model_name=None):
     # 使用 OpenAI 生成基于上传数据的问题
 
-    if LLM_model == "gemini-pro":
+    if LLM_model in ["gemini-1.5-pro","gemini-1.5-flash"]:
         print(f"LLM: {LLM_model}")
+        model_name = LLM_model
         content = generate_content_by_gemini(sys_content, user_content, response_format, model_name=model_name)
     # elif LLM_model == "anthropic-claude-3-sonnet":
     #     print(f"LLM: {LLM_model}")
@@ -2175,7 +2176,7 @@ def download_exam_result(content):
     return word_path
 
 # ---- Chatbot ----
-def get_instructions(content_subject, content_grade, key_moments, socratic_mode=True):
+def get_instructions(content_subject, content_grade, transcript_text, key_moments, socratic_mode=True):
     if socratic_mode:
         method = "Socratic style, guide thinking, no direct answers. this is very important, please be seriously following."
     else:
@@ -2185,18 +2186,59 @@ def get_instructions(content_subject, content_grade, key_moments, socratic_mode=
         subject: {content_subject}
         grade: {content_grade}
         context: {key_moments}
+        transcript_text: {transcript_text}
         Assistant Role: you are a {content_subject} assistant. you can call yourself as  {content_subject} 學伴
         User Role: {content_grade} th-grade student.
         Method: {method}
         Language: Traditional Chinese ZH-TW (it's very important), suitable for {content_grade} th-grade level.
+        Strategy:
+        - You are a professional tutor, and you will use the following teaching strategies based on the textbook content.
+            # General Strategies
+            Needs Analysis:
+            The tutor/assistant teacher should be able to conduct dynamic needs analysis based on the student's responses. Use questions to understand the student's needs and difficulties.
+            Example questions: "What do you want to learn today?" or "What difficulties are you encountering in this part of the content?"
+            
+            Dynamic Goal Setting:
+            Set learning goals based on student feedback, which can be short-term or long-term. The tutor/assistant teacher can adjust the plan automatically according to the student's progress.
+            Example questions: "What is our goal for this week?" or "What tasks do you hope to complete today?"
+            
+            Flexible Teaching Methods:
+            Provide different teaching methods and resources based on the student's age and learning style. The tutor/assistant teacher can adjust teaching strategies based on student feedback.
+            Example questions: "Do you prefer learning through videos or reading materials?" or "We can understand this problem through examples, what do you think?"
+            
+            Patience and Encouragement:
+            Provide positive feedback and encouragement, especially when students encounter difficulties. The tutor/assistant teacher should be able to detect the student's emotions and provide appropriate support.
+            Example questions: "Don't worry, let's try again." or "You did well, keep it up!"
+            
+            Regular Feedback and Evaluation:
+            Regularly evaluate the student's learning progress and provide feedback. The tutor/assistant teacher can use tests and practice questions to assess the student's understanding.
+            Example questions: "Let's check your progress." or "How do you feel about your learning progress during this period?"
+            
+            Good Communication Skills:
+            Maintain good communication with students, responding to their questions and needs in a timely manner. The tutor/assistant teacher should be able to identify and solve students' problems.
+            Example questions: "Is there any problem that you need my help with?" or "Is this part clear to you?"
+            
+            Maintaining Professionalism:
+            Continue learning and improving teaching skills, and maintain punctuality and responsibility. The tutor/assistant teacher should provide accurate and up-to-date information.
+            Example questions: "What is our learning goal for today?" or "Remember to study a little bit every day, and gradually accumulate knowledge."
+            
+            Creating a Positive Learning Environment:
+            Create a positive, supportive, and motivating learning atmosphere. The tutor/assistant teacher should suggest students take breaks and relax at appropriate times.
+            Example questions: "Let's take a break and continue studying afterward." or "How do you feel about this learning environment? Do we need any adjustments?"
+            
+            # Specific Applications
+            The tutor/assistant teacher can automatically adjust the depth and complexity of the questions based on these general strategies by grade. For example:
+            - Kindergarten and Elementary School Students: Use simple vocabulary and concrete examples, with more pictures and gamified content.
+            - Middle School Students: Use interactive and practical methods, such as quizzes and group discussions.
+            - High School Students: Use deep learning and critical thinking exercises, such as project research and discussions.
+            - Adult Learners: Emphasize practical applications and work-related content, such as case studies and workshops.
+        
         Response: 
         - if user say hi or hello or any greeting, just say hi back and introduce yourself. Then tell user to ask question in context.
-        - Single question, under 100 characters
         - include math symbols (use LaTeX $ to cover before and after, ex: $x^2$)
         - hint with video timestamp which format 【參考：00:00:00】. 
-        - Sometimes encourage user by Taiwanese style with relaxing atmosphere.
-        - if user ask questions not include in context, 
-        - just tell them to ask the question in context and give them example question.
+        - Sometimes encourage user with relaxing atmosphere.
+        - if user ask questions not include in context, just tell them to ask the question in context and give them example question.
         Restrictions: 
         - Answer within video content, no external references
         - don't repeat user's question, guide them to think more.
@@ -2309,8 +2351,19 @@ def chat_with_any_ai(ai_type, password, video_id, user_data, transcript_state, k
                 moment.pop('images', None)
                 moment.pop('end', None)
                 moment.pop('transcript', None)
+                moment.pop('suggested_images', None)
+            
+            if isinstance(transcript_state, str):
+                transcript_state_json = json.loads(transcript_state)
+            else:
+                transcript_state_json = transcript_state
+            # remain only text
+            transcript_text = ""
+            for content in transcript_state_json:
+                transcript_text += content["text"] + ","
+            
             key_moments_text = json.dumps(key_moments_json, ensure_ascii=False)
-            instructions = get_instructions(content_subject, content_grade, key_moments_text, socratic_mode)
+            instructions = get_instructions(content_subject, content_grade, transcript_text, key_moments_text, socratic_mode)
             print(f"=== instructions:{instructions} ===")
             user_message_note = "/n 請嚴格遵循instructions，擔任一位蘇格拉底家教，絕對不要重複 user 的問句，請用引導的方式指引方向，請一定要用繁體中文回答 zh-TW，並用台灣人的禮貌口語表達，回答時不要特別說明這是台灣人的語氣，請在回答的最後標註【參考：（時）:（分）:（秒）】，（如果是反問學生，就只問一個問題，請幫助學生更好的理解資料，字數在100字以內，回答時如果講到數學專有名詞，請用數學符號代替文字（Latex 用 $ 字號 render, ex: $x^2$)"
             user_content = user_message + user_message_note
@@ -2383,9 +2436,19 @@ def get_chatbot_config(ai_name, transcript_state, key_moments, content_subject,
         moment.pop('images', None)
         moment.pop('end', None)
         moment.pop('transcript', None)
+        moment.pop('suggested_images', None)
     key_moments_text = json.dumps(key_moments_json, ensure_ascii=False)
 
-    instructions = get_instructions(content_subject, content_grade, key_moments_text, socratic_mode)
+    if isinstance(transcript_state, str):
+        transcript_state_json = json.loads(transcript_state)
+    else:
+        transcript_state_json = transcript_state
+    # remain only text
+    transcript_text = ""
+    for content in transcript_state_json:
+        transcript_text += content["text"] + ","
+
+    instructions = get_instructions(content_subject, content_grade, transcript_text, key_moments_text, socratic_mode)
 
     chatbot_config = {
         "video_id": video_id,
@@ -2689,7 +2752,7 @@ def poll_run_status(run_id, thread_id, timeout=600, poll_interval=5):
     
     return run.status
 
-def chat_with_opan_ai_assistant_streaming(user_message, chat_history, password, video_id, user_data, thread_id, trascript, key_moments, content_subject, content_grade, socratic_mode=True):
+def chat_with_opan_ai_assistant_streaming(user_message, chat_history, password, video_id, user_data, thread_id, transcript_state, key_moments, content_subject, content_grade, socratic_mode=True):
     verify_password(password)
 
     print("=====user_data=====")
@@ -2741,9 +2804,19 @@ def chat_with_opan_ai_assistant_streaming(user_message, chat_history, password,
                 moment.pop('images', None)
                 moment.pop('end', None)
                 moment.pop('transcript', None)
+                moment.pop('suggested_images', None)
             key_moments_text = json.dumps(key_moments_json, ensure_ascii=False)
 
-            instructions = get_instructions(content_subject, content_grade, key_moments_text, socratic_mode)
+            if isinstance(transcript_state, str):
+                transcript_state_json = json.loads(transcript_state)
+            else:
+                transcript_state_json = transcript_state
+            # remain only text
+            transcript_text = ""
+            for content in transcript_state_json:
+                transcript_text += content["text"] + ","
+
+            instructions = get_instructions(content_subject, content_grade, transcript_text, key_moments_text, socratic_mode)
             # 创建线程
             if not thread_id:
                 thread = client.beta.threads.create()
@@ -2997,7 +3070,7 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue=gr.themes.colors.orange, seconda
             )
             ai_chatbot_ai_type = gr.Textbox(value="chat_completions", visible=True)
             ai_chatbot_thread_id = gr.Textbox(label="thread_id", visible=True)
-            ai_chatbot_socratic_mode_btn = gr.Checkbox(label="蘇格拉底家教助理模式", value=True, visible=True)
+            ai_chatbot_socratic_mode_btn = gr.Checkbox(label="蘇格拉底家教助理模式", value=False, visible=True)
             latex_delimiters = [{"left": "$", "right": "$", "display": False}]
         with gr.Accordion("選擇 AI 小精靈", elem_id="chatbot_select_accordion") as chatbot_select_accordion:
             with gr.Row():
@@ -3227,7 +3300,7 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue=gr.themes.colors.orange, seconda
     with gr.Accordion("See Details", open=False) as see_details:
         with gr.Row():
             is_env_prod = gr.Checkbox(value=False, label="is_env_prod")
-            LLM_model = gr.Dropdown(label="LLM Model", choices=["open-ai-gpt-4o", "anthropic-claude-3-sonnet", "gemini-pro"], value="open-ai-gpt-4o", visible=True, interactive=True)
+            LLM_model = gr.Dropdown(label="LLM Model", choices=["open-ai-gpt-4o", "anthropic-claude-3-sonnet", "gemini-1.5-pro", "gemini-1.5-flash"], value="open-ai-gpt-4o", visible=True, interactive=True)
         with gr.Tab("逐字稿本文"):
             with gr.Row() as transcript_admmin:
                 transcript_kind = gr.Textbox(value="transcript", show_label=False)

requirements.txt

@@ -12,11 +12,11 @@ google-auth-httplib2
 google-auth-oauthlib
 google-cloud-storage
 google-cloud-bigquery
-google-cloud-aiplatform
+google-generativeai
 groq
 yt_dlp
 uuid
 gtts
-boto3
+boto3==1.28.57
 pydub
 vertexai