Update app.py

app.py

@@ -235,8 +235,7 @@ global_data = {
 model_configs = [
     {"repo_id": "Hjgugugjhuhjggg/testing_semifinal-Q2_K-GGUF", "filename": "testing_semifinal-q2_k.gguf", "name": "testing"},
     {"repo_id": "bartowski/Llama-3.2-3B-Instruct-uncensored-GGUF", "filename": "Llama-3.2-3B-Instruct-uncensored-Q2_K.gguf", "name": "Llama-3.2-3B-Instruct"},
-    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B", "filename": "meta-llama-3.1-70B", "name": "Meta-Llama-3.1-70B"},
-    {"repo_id": "Hhhbvvkgh/Heidi-Llama-v4-Q2_K-GGUF", "filename": "heidi-llama-v4-q2_k.gguf", "name": "Heidi-Llama-V4"}
+    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-13B", "filename": "Meta-Llama-3.1-13B-Q2_K.gguf", "name": "Meta-Llama-3.1-13B"}
 ]
 
 def normalize_input(input_text):
@@ -246,12 +245,19 @@ def normalize_input(input_text):
     return " ".join(filtered_words)
 
 async def load_models():
+    tasks = []
     for model in model_configs:
         model_path = os.path.join("models", model["filename"])
         if not os.path.exists(model_path):
             url = f"https://huggingface.co/{model['repo_id']}/resolve/main/{model['filename']}"
-            wget.download(url, model_path)
-        global_data['models'][model['name']] = Llama(model_path)
+            tasks.append(download_model(url, model_path))
+    await asyncio.gather(*tasks)
+    for model in model_configs:
+        model_path = os.path.join("models", model["filename"])
+        global_data['models'][model["name"]] = Llama(model_path)
+
+async def download_model(url, model_path):
+    wget.download(url, model_path)
 
 async def generate_model_response(model, inputs):
     try:
@@ -263,7 +269,9 @@ async def generate_model_response(model, inputs):
 def get_best_response(responses):
     if not responses:
         return {"error": "No valid responses from models."}
-    return max(responses, key=lambda x: x['score'])
+    scores = [response['score'] for response in responses]
+    best_score_index = scores.index(max(scores))
+    return responses[best_score_index]
 
 async def process_message(message):
     inputs = normalize_input(message)
@@ -272,17 +280,6 @@ async def process_message(message):
     best_response = get_best_response(responses)
     return best_response
 
-app = FastAPI()
-
-@app.post("/generate")
-async def generate(request: Request):
-    try:
-        body = await request.json()
-        response = await process_message(body['message'])
-        return JSONResponse(content={"response": response})
-    except Exception as e:
-        return JSONResponse(content={"error": str(e)})
-
 def run_uvicorn():
     uvicorn.run(app, host="0.0.0.0", port=7860)