Update main.py

main.py

@@ -144,11 +144,76 @@ async def get_models():
         raise HTTPException(status_code=404, detail="models.json not found")
     except json.JSONDecodeError:
         raise HTTPException(status_code=500, detail="Error decoding models.json")
-
+@app.get("api/v1/models")
 @app.get("/models")
 async def fetch_models():
     return await get_models()
 server_status = True
+import asyncio
+import datetime
+import hashlib
+import json
+from functools import lru_cache
+from typing import Dict, Any
+
+# Create a simple in-memory cache with LRU (Least Recently Used) mechanism
+class RequestCache:
+    def __init__(self, max_size=100, expires_after=300):  # 100 entries, expires after 5 minutes
+        self.cache: Dict[str, Dict[str, Any]] = {}
+        self.max_size = max_size
+        self.expires_after = expires_after
+
+    def _generate_cache_key(self, payload_dict: Dict[str, Any]) -> str:
+        """
+        Generate a unique cache key based on the payload.
+        Exclude any time-sensitive or dynamic fields.
+        """
+        # Create a deep copy to avoid modifying the original payload
+        payload_for_hash = payload_dict.copy()
+        
+        # Remove fields that might change between identical requests
+        payload_for_hash.pop('request_id', None)
+        payload_for_hash.pop('timestamp', None)
+        
+        # Convert to a sorted, stable JSON string for hashing
+        payload_json = json.dumps(payload_for_hash, sort_keys=True)
+        
+        # Create a hash of the payload
+        return hashlib.md5(payload_json.encode()).hexdigest()
+
+    def get(self, payload_dict: Dict[str, Any]) -> Any:
+        """
+        Retrieve cached response if it exists and is not expired.
+        """
+        cache_key = self._generate_cache_key(payload_dict)
+        
+        # Check if cache entry exists and is not expired
+        if (cache_key in self.cache and 
+            (datetime.datetime.now() - self.cache[cache_key]['timestamp']).total_seconds() < self.expires_after):
+            return self.cache[cache_key]['response']
+        
+        return None
+
+    def set(self, payload_dict: Dict[str, Any], response: Any):
+        """
+        Store response in cache, managing cache size.
+        """
+        cache_key = self._generate_cache_key(payload_dict)
+        
+        # Remove oldest entry if cache is full
+        if len(self.cache) >= self.max_size:
+            oldest_key = min(self.cache, key=lambda k: self.cache[k]['timestamp'])
+            del self.cache[oldest_key]
+        
+        # Store the response with a timestamp
+        self.cache[cache_key] = {
+            'response': response,
+            'timestamp': datetime.datetime.now()
+        }
+
+# Global cache instance
+request_cache = RequestCache()
+
 @app.post("/chat/completions")
 @app.post("api/v1/chat/completions")
 async def get_completion(payload: Payload, request: Request):
@@ -167,12 +232,20 @@ async def get_completion(payload: Payload, request: Request):
             status_code=400,
             detail=f"Model '{model_to_use}' is not available. Check /models for the available model list."
         )
-        
-    usage_tracker.record_request(model=model_to_use, endpoint="/chat/completions")
-        
-    # Prepare payload
+    
+    # Convert payload to dictionary for caching
     payload_dict = payload.dict()
     payload_dict["model"] = model_to_use
+    
+    # Check cache first
+    cached_response = request_cache.get(payload_dict)
+    if cached_response:
+        return StreamingResponse(
+            (line for line in cached_response), 
+            media_type="application/json"
+        )
+    
+    usage_tracker.record_request(model=model_to_use, endpoint="/chat/completions")
         
     # Select the appropriate endpoint
     endpoint = secret_api_endpoint_2 if model_to_use in alternate_models else secret_api_endpoint
@@ -181,18 +254,18 @@ async def get_completion(payload: Payload, request: Request):
     current_time = (datetime.datetime.utcnow() + datetime.timedelta(hours=5, minutes=30)).strftime("%Y-%m-%d %I:%M:%S %p")
     aaip = request.client.host
     print(f"Time: {current_time}, {aaip}")
-    # print(payload_dict) 
+    
     scraper = cloudscraper.create_scraper()    
     async def stream_generator(payload_dict):
-        
         # Prepare custom headers
         custom_headers = {
             'DNT': '1',
-            # 'Origin': ENDPOINT_ORIGIN,
             'Priority': 'u=1, i',
-            # 'Referer': ENDPOINT_ORIGIN
         }
         
+        # Collect response lines to cache
+        response_lines = []
+        
         try:
             # Send POST request using CloudScraper with custom headers
             response = scraper.post(
@@ -214,10 +287,16 @@ async def get_completion(payload: Payload, request: Request):
             elif response.status_code >= 500:
                 raise HTTPException(status_code=500, detail="Server error. Try again later.")
             
-            # Stream response lines to the client
+            # Stream response lines to the client and collect for caching
             for line in response.iter_lines():
                 if line:
-                    yield line.decode('utf-8') + "\n"
+                    decoded_line = line.decode('utf-8') + "\n"
+                    response_lines.append(decoded_line)
+                    yield decoded_line
+            
+            # Cache the entire response after successful streaming
+            request_cache.set(payload_dict, response_lines)
+            
         except requests.exceptions.RequestException as req_err:
             # Handle request-specific errors
             print(response.text)