Upload 4 files

src/group_classification.py

@@ -7,6 +7,7 @@ from langchain_community.vectorstores import AzureSearch
 from langchain_community.document_loaders import PyMuPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from openai import AzureOpenAI
+from openai import OpenAIError
 from azure.search.documents.indexes import SearchIndexClient
 from azure.core.credentials import AzureKeyCredential
 import time
@@ -86,26 +87,39 @@ def main(PDF):
     message = [
             {"role": "system",
              "content":
-"""You are an assistant designed to analyze provided text and answer questions regarding CO2 emissions and reduction targets.
-When given a prompt, you should extract the relevant information from the provided text and answer in the form (排出量, 削減目標).
-- 排出量 (CO2 Emission): Answer 1 if there is any mention of CO2 emissions, otherwise answer 0.
-- 削減目標 (Reduction Target): Answer 1 if there is any mention of a CO2 emission reduction target, otherwise answer 0.
-Example:
-- If the text mentions CO2 emissions but does not mention any reduction targets, your response should be (1, 0).
-- If the text mentions neither CO2 emissions nor reduction targets, your response should be (0, 0)."""
-            },
+             """You are an assistant designed to analyze provided text and answer questions regarding CO2 emissions and reduction targets.
+             When given a prompt, you should extract the relevant information from the provided text and answer in the form (排出量, 削減目標).
+             - 排出量 (CO2 Emission): Answer 1 if there is any mention of CO2 emissions, otherwise answer 0.
+             - 削減目標 (Reduction Target): Answer 1 if there is any mention of a CO2 emission reduction target, otherwise answer 0.
+             Example:
+             - If the text mentions CO2 emissions but does not mention any reduction targets, your response should be (1, 0).
+             - If the text mentions neither CO2 emissions nor reduction targets, your response should be (0, 0)."""
+             },
             {"role": "user",
-            "content": f"""Using the provided information: {texts}, please answer the following question in the given form (排出量, 削減目標): {query}."""}
+             "content": f"""Using the provided information: {texts}, please answer the following question in the given form (排出量, 削減目標): {query}."""}
         ]
-
-    response = client.chat.completions.create(
-        model=api_type,
-        messages=message,
-        temperature=0
-    )
-    # print(response.choices[0].message.content)
-    time.sleep(1)
-    return response.choices[0].message.content
+    retries = 0
+    max_retries = 100
+    delay = 5
+    while retries < max_retries:
+        try:
+            response = client.chat.completions.create(
+                model=api_type,
+                messages=message,
+                temperature=0
+            )
+            # print(response.choices[0].message.content)
+            time.sleep(1)
+            return response.choices[0].message.content
+        except OpenAIError as e:
+            print(f"Error occurred: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+        except Exception as e:
+            print(f"Unexpected error: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+    raise RuntimeError("Maximum retries exceeded. Could not get a valid response.")
 
 
 if __name__ == '__main__':

src/research_html_scoring.py

@@ -13,6 +13,7 @@ import chromadb
 from rank_bm25 import BM25Okapi
 from janome.tokenizer import Tokenizer
 from openai import OpenAI
+from openai import OpenAIError
 from src.myLogger import set_logger
 import time
 
@@ -50,31 +51,42 @@ def normalize_text(s, sep_token = " \n "):
     return s
 
 
-def generate_answer_(reference, system_prompt, json_schema):
+def generate_answer_(reference, system_prompt, json_schema, max_retries=100, delay=5):
     api_key = os.getenv("OPENAI_API_KEY")
     client = OpenAI(
         api_key=api_key,
     )
-    response = client.chat.completions.create(
-        model="gpt-3.5-turbo",
-        messages=[
-            {
-                "role": "system",
-                "content": system_prompt,
-            },
-            {
-                "role": "user",
-                "content": reference,
-            },
-        ],
-        functions=[{"name": "generate_queries", "parameters": json_schema}],
-        function_call={"name": "generate_queries"},
-        temperature=0.0,
-        top_p=0.0,
-    )
-    output = response.choices[0].message.function_call.arguments
-    time.sleep(1)
-    return output
+    retries = 0
+    while retries < max_retries:
+        try:
+            response = client.chat.completions.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": system_prompt,
+                    },
+                    {
+                        "role": "user",
+                        "content": reference,
+                    },
+                ],
+                functions=[{"name": "generate_queries", "parameters": json_schema}],
+                function_call={"name": "generate_queries"},
+                temperature=0.0,
+                top_p=0.0,
+            )
+            output = response.choices[0].message.function_call.arguments
+            return output  # Return successfully if no exception occurs
+        except OpenAIError as e:
+            print(f"Error occurred: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+        except Exception as e:
+            print(f"Unexpected error: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+    raise RuntimeError("Maximum retries exceeded. Could not get a valid response.")
 
 
 def find_context(pdf_url):

src/research_pdf_scoring.py

@@ -4,6 +4,7 @@ from langchain.document_loaders import PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Chroma
 from langchain_openai import OpenAIEmbeddings
+from openai import OpenAIError
 import os
 import re
 import json
@@ -42,33 +43,44 @@ def normalize_text(s, sep_token = " \n "):
     return s
 
 
-def generate_answer_(reference, system_prompt, json_schema):
+def generate_answer_(reference, system_prompt, json_schema, max_retries=100, delay=5):
     api_key = os.getenv("OPENAI_API_KEY")
-    print("completion start")
     client = OpenAI(
         api_key=api_key,
     )
-    response = client.chat.completions.create(
-        model="gpt-3.5-turbo",
-        messages=[
-            {
-                "role": "system",
-                "content": system_prompt,
-            },
-            {
-                "role": "user",
-                "content": reference,
-            },
-        ],
-        functions=[{"name": "generate_queries", "parameters": json_schema}],
-        function_call={"name": "generate_queries"},
-        temperature=0.0,
-        top_p=0.0,
-    )
-    print("completion end")
-    output = response.choices[0].message.function_call.arguments
-    time.sleep(1)
-    return output
+    retries = 0
+    while retries < max_retries:
+        try:
+            response = client.chat.completions.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": system_prompt,
+                    },
+                    {
+                        "role": "user",
+                        "content": reference,
+                    },
+                ],
+                functions=[{"name": "generate_queries", "parameters": json_schema}],
+                function_call={"name": "generate_queries"},
+                temperature=0.0,
+                top_p=0.0,
+            )
+            print("completion end")
+            output = response.choices[0].message.function_call.arguments
+            time.sleep(1)
+            return output
+        except OpenAIError as e:
+            print(f"Error occurred: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+        except Exception as e:
+            print(f"Unexpected error: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+    raise RuntimeError("Maximum retries exceeded. Could not get a valid response.")
 
 
 def find_context(pdf_url):

src/scoring_utils.py

@@ -1,6 +1,7 @@
 import json
 import os
 import pandas as pd
+import time
 import gspread
 from google.oauth2.service_account import Credentials
 from src.research_html_scoring import research_html_hybrid, group1_html
@@ -8,6 +9,7 @@ from src.research_pdf_scoring import research_pdf_hybrid, group1_pdf
 from utils_groupclassification.check_openai import co
 from src.myLogger import set_logger
 from openai import AzureOpenAI
+from openai import OpenAIError
 from dotenv import load_dotenv
 
 logger = set_logger("my_app", level="INFO")
@@ -61,12 +63,28 @@ def summarize(sentence, user_prompt):
         {"role": "user",
          "content": user_prompt}
         ]
-    response = client.chat.completions.create(
-            messages=messages,
-            model=api_type,
-            temperature=0,
-    )
-    return response.choices[0].message.content
+
+    retries = 0
+    max_retries = 100
+    delay = 5
+
+    while retries < max_retries:
+        try:
+            response = client.chat.completions.create(
+                    messages=messages,
+                    model=api_type,
+                    temperature=0,
+            )
+            return response.choices[0].message.content
+        except OpenAIError as e:
+            print(f"Error occurred: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+        except Exception as e:
+            print(f"Unexpected error: {e}. Retrying in {delay} seconds...")
+            retries += 1
+            time.sleep(delay)
+    raise RuntimeError("Maximum retries exceeded. Could not get a valid response.")
 
 
 def summarize5(sentence):