Debugging subprocess to get the space to not need restart after a day

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-import random
+import psutil
 import subprocess
 import time
 
@@ -13,36 +13,53 @@ def generate_response(user_message): #generate_response_token_by_token
     ]
 
     process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1)
+    process_monitor = psutil.Process(process.pid)
+
 
     start_time = time.time()
     alltokens = ""
     token_buffer = ''
-    while True:
-        # Read one character at a time
-        char = process.stdout.read(1)
-        if char == '' and process.poll() is not None:
-            break
-        if char != '':
-            token_buffer += char
-            if char == ' ' or char == '\n':  # Token delimiters
-                elapsed_time = time.time() - start_time  # Calculate elapsed time
-                alltokens += token_buffer
-                yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds]"
-                token_buffer = ''  # Reset token buffer
+    try:
+        while True:
+            # Read one character at a time
+            char = process.stdout.read(1)
+            if char == '' and process.poll() is not None:
+                break
+            if char != '':
+                token_buffer += char
+                if char == ' ' or char == '\n':  # Token delimiters
+                    elapsed_time = time.time() - start_time  # Calculate elapsed time
+                    alltokens += token_buffer
+                    yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds]"
+                    token_buffer = ''  # Reset token buffer
+            # Log resource usage every minute
+            if time.time() - start_time > 60:
+                cpu_usage = process_monitor.cpu_percent()
+                memory_usage = process_monitor.memory_info().rss  # in bytes 
+                print(f"Subprocess CPU Usage: {cpu_usage}%, Memory Usage: {memory_usage / 1024 ** 2} MB")
+                start_time = time.time()  # Reset the timer
     
-    # Yield the last token if there is any
-    if token_buffer:
-        elapsed_time = time.time() - start_time  # Calculate elapsed time
-        alltokens += token_buffer
-        yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds]"
-
-    # Wait for the subprocess to finish if it hasn't already
-    process.wait()
+        # Yield the last token if there is any
+        if token_buffer:
+            elapsed_time = time.time() - start_time  # Calculate elapsed time
+            alltokens += token_buffer
+            yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds]"
+    finally:
+        try:
+            # Wait for the process to complete, with a timeout
+            process.wait(timeout=60)  # Timeout in seconds
+        except subprocess.TimeoutExpired:
+            print("Process didn't complete within the timeout. Killing it.")
+            process.kill()
+            process.wait()  # Ensure proper cleanup
+        # Wait for the subprocess to finish if it hasn't already
+        process.stdout.close()
+        process.stderr.close()
 
-    # Check for any errors
-    if process.returncode != 0:
-        error_message = process.stderr.read()
-        print(f"Error: {error_message}")
+        # Check for any errors
+        if process.returncode != 0:
+            error_message = process.stderr.read()
+            print(f"Error: {error_message}")
 
 
 def custom_generate_response(cust_user_message):