//! Loading benchmark results from disk for consolidation //! //! This module provides `load_run_results` which recursively loads benchmark //! result JSON files from a directory tree, tagging them with batch mode info //! inferred from directory names. use crate::types::BenchmarkResult; use crate::{Error, Result}; use std::fs; use std::path::Path; /// Load benchmark results from `results.json` files in a directory. /// /// Recursively walks the given directory, loading any `results.json` files found. /// For directories whose name ends with `-batch`, the framework name in each result /// is suffixed with `-batch` so that the aggregation layer can distinguish single- /// vs batch-mode results. /// /// # Errors /// /// Returns [`Error::Io`] if the directory cannot be read, or [`Error::Benchmark`] /// if a `results.json` file contains invalid JSON or fails validation. pub fn load_run_results(dir: &Path) -> Result> { let mut results = Vec::new(); for entry in fs::read_dir(dir).map_err(Error::Io)? { let entry = entry.map_err(Error::Io)?; let path = entry.path(); if path.is_file() && path.file_name().is_some_and(|n| n == "results.json") { eprintln!("Loading results from {}", path.display()); let json_content = fs::read_to_string(&path).map_err(Error::Io)?; let mut run_results: Vec = serde_json::from_str(&json_content) .map_err(|e| Error::Benchmark(format!("Failed to parse {}: {}", path.display(), e)))?; // Infer benchmark mode from the parent directory name. // The runner outputs to `benchmark-results/{FRAMEWORK}-{MODE}/results.json` // where MODE is "batch" or "single-file". The framework field inside // results.json does NOT include the mode, so we tag it here to allow // the aggregation to distinguish single vs batch results. let dir_name = dir.file_name().and_then(|n| n.to_str()).unwrap_or(""); let is_batch = dir_name.ends_with("-batch"); if is_batch { for result in &mut run_results { if !result.framework.ends_with("-batch") { result.framework = format!("{}-batch", result.framework); } } } // Validate loaded results for result in &run_results { crate::output::validate_result(result) .map_err(|e| Error::Benchmark(format!("Invalid result in {}: {}", path.display(), e)))?; } results.extend(run_results); } else if path.is_dir() { match load_run_results(&path) { Ok(mut run_results) => results.append(&mut run_results), Err(e) => eprintln!("Warning: Failed to load results from {}: {}", path.display(), e), } } } Ok(results) } #[cfg(test)] mod tests { use super::*; use crate::types::{ErrorKind, FrameworkCapabilities, OutputFormat, PerformanceMetrics}; use std::time::Duration; /// Build a minimal valid `BenchmarkResult` for testing. fn make_result(framework: &str) -> BenchmarkResult { BenchmarkResult { framework: framework.to_string(), file_path: std::path::PathBuf::from("test.pdf"), file_size: 1024, success: true, error_message: None, error_kind: ErrorKind::None, duration: Duration::from_millis(100), extraction_duration: None, subprocess_overhead: None, metrics: PerformanceMetrics { peak_memory_bytes: 1_000_000, avg_cpu_percent: 50.0, throughput_bytes_per_sec: 10_240.0, p50_memory_bytes: 900_000, p95_memory_bytes: 950_000, p99_memory_bytes: 990_000, }, quality: None, iterations: vec![], statistics: None, cold_start_duration: None, file_extension: "pdf".to_string(), framework_capabilities: FrameworkCapabilities::default(), pdf_metadata: None, ocr_status: Default::default(), extracted_text: None, output_format: OutputFormat::Markdown, } } #[test] fn test_load_single_results_file() { let dir = tempfile::tempdir().expect("create temp dir"); let results = vec![make_result("kreuzberg-rust")]; let json = serde_json::to_string(&results).expect("serialize"); fs::write(dir.path().join("results.json"), &json).expect("write"); let loaded = load_run_results(dir.path()).expect("load"); assert_eq!(loaded.len(), 1); assert_eq!(loaded[0].framework, "kreuzberg-rust"); } #[test] fn test_batch_directory_tags_framework_name() { let dir = tempfile::tempdir().expect("create temp dir"); let batch_dir = dir.path().join("kreuzberg-rust-batch"); fs::create_dir_all(&batch_dir).expect("create subdir"); let results = vec![make_result("kreuzberg-rust")]; let json = serde_json::to_string(&results).expect("serialize"); fs::write(batch_dir.join("results.json"), &json).expect("write"); let loaded = load_run_results(dir.path()).expect("load"); assert_eq!(loaded.len(), 1); assert_eq!(loaded[0].framework, "kreuzberg-rust-batch"); } #[test] fn test_batch_suffix_not_doubled() { let dir = tempfile::tempdir().expect("create temp dir"); let batch_dir = dir.path().join("kreuzberg-rust-batch"); fs::create_dir_all(&batch_dir).expect("create subdir"); let results = vec![make_result("kreuzberg-rust-batch")]; let json = serde_json::to_string(&results).expect("serialize"); fs::write(batch_dir.join("results.json"), &json).expect("write"); let loaded = load_run_results(dir.path()).expect("load"); assert_eq!(loaded.len(), 1); assert_eq!(loaded[0].framework, "kreuzberg-rust-batch"); } #[test] fn test_recursive_loading() { let dir = tempfile::tempdir().expect("create temp dir"); let sub1 = dir.path().join("framework-a"); let sub2 = dir.path().join("framework-b"); fs::create_dir_all(&sub1).expect("create subdir 1"); fs::create_dir_all(&sub2).expect("create subdir 2"); fs::write( sub1.join("results.json"), serde_json::to_string(&vec![make_result("framework-a")]).expect("serialize"), ) .expect("write a"); fs::write( sub2.join("results.json"), serde_json::to_string(&vec![make_result("framework-b")]).expect("serialize"), ) .expect("write b"); let loaded = load_run_results(dir.path()).expect("load"); assert_eq!(loaded.len(), 2); let names: Vec<&str> = loaded.iter().map(|r| r.framework.as_str()).collect(); assert!(names.contains(&"framework-a")); assert!(names.contains(&"framework-b")); } #[test] fn test_malformed_json_returns_error() { let dir = tempfile::tempdir().expect("create temp dir"); fs::write(dir.path().join("results.json"), "NOT VALID JSON").expect("write"); let result = load_run_results(dir.path()); assert!(result.is_err()); let err_msg = format!("{}", result.unwrap_err()); assert!(err_msg.contains("Failed to parse")); } #[test] fn test_empty_directory_returns_empty_vec() { let dir = tempfile::tempdir().expect("create temp dir"); let loaded = load_run_results(dir.path()).expect("load"); assert!(loaded.is_empty()); } #[test] fn test_nonexistent_directory_returns_error() { let result = load_run_results(Path::new("/tmp/nonexistent_benchmark_dir_12345")); assert!(result.is_err()); } }