app.py fixed

dashboard/app.py

@@ -276,98 +276,110 @@ def run(from_results_dir, datasource, port):
 
 
 @click.command()
-@click.option('--from-results-dir', default=None, help='Load inference-benchmarker results from a directory')
-@click.option('--datasource', default='file://benchmarks.parquet', help='Load a Parquet file already generated')
+@click.option('--from-results-dir', 'cli_from_results_dir', default=None, help='Load inference-benchmarker results from this directory. Overrides DASHBOARD_FROM_RESULTS_DIR.')
+@click.option('--datasource', 'cli_datasource', default='file://benchmarks.parquet', help='Load this Parquet file directly if not building from a results directory.')
 @click.option('--port', default=7860, help='Port to run the dashboard')
-def main(from_results_dir, datasource, port):
+def main(cli_from_results_dir, cli_datasource, port):
     print("===== Starting Application =====")
-    print(f"Environment variables: {os.environ}")
+    # print(f"Environment variables: {os.environ}") # Already in user's code or logs
+
+    # Determine the directory from which to process JSON results
+    # Priority: 1. CLI option, 2. Env Var, 3. Default to 'results' dir
+    processing_dir = cli_from_results_dir
+
+    if processing_dir is None:
+        env_var_value = os.environ.get('DASHBOARD_FROM_RESULTS_DIR')
+        if env_var_value:
+            print(f"Using environment variable DASHBOARD_FROM_RESULTS_DIR='{env_var_value}' for processing.")
+            processing_dir = env_var_value
+        elif os.path.exists('results') and os.path.isdir('results'):
+            print(f"No --from-results-dir option or DASHBOARD_FROM_RESULTS_DIR env var. Defaulting to 'results' directory for processing as it exists.")
+            processing_dir = 'results'
+        else:
+            print(f"No directory specified for processing (no --from-results-dir, no DASHBOARD_FROM_RESULTS_DIR env var, and 'results' dir not found).")
+            # processing_dir remains None
 
-    if from_results_dir is not None:
-        # If from_results_dir is specified, results are built into 'benchmarks.parquet'
-        # within that directory.
+    path_to_load_by_run_function = None # This will be the path to the .parquet file
+
+    if processing_dir:
+        # A directory for processing JSONs has been determined.
+        # Use the existing logic to build/fallback and generate benchmarks.parquet.
         output_filename = 'benchmarks.parquet'
-        print(f"💡 Debug - Building results from directory: {from_results_dir}")
+        abs_processing_dir = os.path.abspath(processing_dir)
 
-        # Check if results directory exists
-        check_file_exists(from_results_dir, "Results directory")
+        print(f"💡 Debug - Will process JSONs from directory: {abs_processing_dir}")
+        check_file_exists(abs_processing_dir, "Source directory for JSONs")
 
-        # Create absolute path for results directory
-        abs_results_dir = os.path.abspath(from_results_dir)
-        print(f"💡 Debug - Absolute results directory: {abs_results_dir}")
+        # Ensure the directory exists (it might be 'results' or user-provided)
+        # build_results might expect the output directory to exist.
+        if not os.path.exists(abs_processing_dir):
+            print(f"💡 Debug - Creating directory for processing/output: {abs_processing_dir}")
+            os.makedirs(abs_processing_dir, exist_ok=True)
 
-        # Create the results directory if it doesn't exist
-        if not os.path.exists(abs_results_dir):
-            print(f"💡 Debug - Creating results directory: {abs_results_dir}")
-            os.makedirs(abs_results_dir, exist_ok=True)
-
-        # Call build_results with absolute paths
-        full_output_path = os.path.join(abs_results_dir, output_filename)
-        print(f"💡 Debug - Expected output path: {full_output_path}")
+        # The generated parquet file will be placed inside the abs_processing_dir
+        generated_parquet_filepath = os.path.join(abs_processing_dir, output_filename)
+        print(f"💡 Debug - Expected path for generated parquet file: {generated_parquet_filepath}")
 
         try:
-            build_results(abs_results_dir, output_filename, None)
-            print("✅ Build results completed")
-        except Exception as e:
-            print(f"❌ Error in build_results: {str(e)}")
+            build_results(abs_processing_dir, output_filename, None) # output_filename is relative to abs_processing_dir
+            print("✅ Build results completed using build_results.")
+        except Exception as e_build:
+            print(f"❌ Error in build_results: {str(e_build)}")
             print(f"Stack trace: {traceback.format_exc()}")
             print("⚠️ Attempting fallback method: direct JSON processing")
-
-            # Fallback: Directly process the JSON files
             try:
-                # Find all JSON files in the results directory
-                json_files = glob.glob(os.path.join(abs_results_dir, "*.json"))
-                print(f"Found {len(json_files)} JSON files: {json_files}")
-
+                json_files = glob.glob(os.path.join(abs_processing_dir, "*.json"))
+                print(f"Found {len(json_files)} JSON files for fallback: {json_files}")
                 if not json_files:
-                    raise FileNotFoundError("No JSON files found in results directory")
+                    raise FileNotFoundError("Fallback: No JSON files found in results directory")
 
-                # Create a combined DataFrame from the JSON files
                 combined_data = []
                 for json_file in json_files:
                     try:
                         with open(json_file, 'r') as f:
                             data = json.load(f)
-
-                        # Extract model name from filename
                         filename = os.path.basename(json_file)
-                        model_name = filename.split('_')[0] + '_' + filename.split('_')[1]
+                        model_name_parts = filename.split('_')
+                        model_name = f"{model_name_parts[0]}_{model_name_parts[1]}" if len(model_name_parts) > 1 else model_name_parts[0]
 
-                        # Process the benchmarks - basic structure expected in the JSON files
                         if 'benchmarks' in data:
                             for benchmark in data['benchmarks']:
                                 benchmark['model'] = model_name
                                 benchmark['run_id'] = os.path.splitext(filename)[0]
                                 combined_data.append(benchmark)
                         else:
-                            print(f"⚠️ No 'benchmarks' key in {json_file}")
+                            print(f"⚠️ Fallback: No 'benchmarks' key in {json_file}")
                     except Exception as json_err:
-                        print(f"❌ Error processing {json_file}: {str(json_err)}")
+                        print(f"❌ Fallback: Error processing {json_file}: {str(json_err)}")
 
                 if combined_data:
                     df_direct = pd.DataFrame(combined_data)
-                    # Save the DataFrame to parquet
-                    df_direct.to_parquet(full_output_path)
-                    print(f"✅ Created parquet file via fallback method: {full_output_path}")
+                    df_direct.to_parquet(generated_parquet_filepath)
+                    print(f"✅ Created parquet file via fallback method: {generated_parquet_filepath}")
                 else:
-                    raise ValueError("No data could be extracted from JSON files")
-
-            except Exception as fallback_err:
-                print(f"❌ Fallback method failed: {str(fallback_err)}")
+                    raise ValueError("Fallback: No data could be extracted from JSON files")
+            except Exception as e_fallback:
+                print(f"❌ Fallback method failed: {str(e_fallback)}")
                 print(f"Stack trace: {traceback.format_exc()}")
 
-        # Check if the file was created
-        check_file_exists(full_output_path, "Generated parquet")
-
-        # The file to load is now in from_results_dir/output_filename
-        parquet_file_to_load = full_output_path
+        # After attempting to build/generate, check if the file exists
+        check_file_exists(generated_parquet_filepath, "Parquet file after build/fallback attempts")
+        if os.path.exists(generated_parquet_filepath):
+            path_to_load_by_run_function = generated_parquet_filepath
+        else:
+            print(f"❌ CRITICAL: Failed to generate or find parquet file at '{generated_parquet_filepath}' after all attempts.")
+            # path_to_load_by_run_function remains None.
+            # The run() function's error handling will create an empty DataFrame.
     else:
-        # If not building from results_dir, use the provided datasource directly.
-        parquet_file_to_load = datasource
+        # No directory for processing JSONs was determined. Use the cli_datasource directly.
+        print(f"Not processing a results directory. Using --datasource option directly: '{cli_datasource}'")
+        path_to_load_by_run_function = cli_datasource
 
-    print(f"Final parameters: from_results_dir={from_results_dir}, datasource={datasource}, port={port}")
+    print(f"💡 Final path to be loaded by run() function: '{path_to_load_by_run_function}'")
 
-    run(None, parquet_file_to_load, port)
+    # Call run(). The first argument (from_results_dir for run()) is None because main handles processing.
+    # The second argument (datasource for run()) is the actual file path to load.
+    run(None, path_to_load_by_run_function, port)
 
 
 if __name__ == '__main__':