foundry_bench/
results.rs

1use crate::RepoConfig;
2use eyre::Result;
3use serde::{Deserialize, Serialize};
4use std::{collections::HashMap, process::Command, thread};
5
6/// Hyperfine benchmark result
7#[derive(Debug, Deserialize, Serialize)]
8pub struct HyperfineResult {
9    pub command: String,
10    pub mean: f64,
11    pub stddev: Option<f64>,
12    pub median: f64,
13    pub user: f64,
14    pub system: f64,
15    pub min: f64,
16    pub max: f64,
17    pub times: Vec<f64>,
18    #[serde(skip_serializing_if = "Option::is_none")]
19    pub exit_codes: Option<Vec<i32>>,
20    #[serde(skip_serializing_if = "Option::is_none")]
21    pub parameters: Option<HashMap<String, serde_json::Value>>,
22}
23
24/// Hyperfine JSON output format
25#[derive(Debug, Deserialize, Serialize)]
26pub struct HyperfineOutput {
27    pub results: Vec<HyperfineResult>,
28}
29
30/// Aggregated benchmark results
31#[derive(Debug, Default)]
32pub struct BenchmarkResults {
33    /// Map of benchmark_name -> version -> repo -> result
34    pub data: HashMap<String, HashMap<String, HashMap<String, HyperfineResult>>>,
35    /// Track the baseline version for comparison
36    pub baseline_version: Option<String>,
37    /// Map of version name -> full version details
38    pub version_details: HashMap<String, String>,
39}
40
41impl BenchmarkResults {
42    pub fn new() -> Self {
43        Self::default()
44    }
45
46    pub fn set_baseline_version(&mut self, version: String) {
47        self.baseline_version = Some(version);
48    }
49
50    pub fn add_result(
51        &mut self,
52        benchmark: &str,
53        version: &str,
54        repo: &str,
55        result: HyperfineResult,
56    ) {
57        self.data
58            .entry(benchmark.to_string())
59            .or_default()
60            .entry(version.to_string())
61            .or_default()
62            .insert(repo.to_string(), result);
63    }
64
65    pub fn add_version_details(&mut self, version: &str, details: String) {
66        self.version_details.insert(version.to_string(), details);
67    }
68
69    /// Generate a flat JSON summary mapping `"benchmark/repo" -> mean_seconds`.
70    ///
71    /// Used by the nightly regression comparison script.
72    pub fn generate_json_summary(&self, versions: &[String]) -> HashMap<String, f64> {
73        let mut summary = HashMap::new();
74        for (benchmark_name, version_data) in &self.data {
75            for version in versions {
76                if let Some(repo_data) = version_data.get(version) {
77                    for (repo_name, result) in repo_data {
78                        let key = format!("{benchmark_name}/{repo_name}");
79                        let rounded = (result.mean * 10_000.0).round() / 10_000.0;
80                        summary.insert(key, rounded);
81                    }
82                }
83            }
84        }
85        summary
86    }
87
88    pub fn generate_markdown(&self, versions: &[String], repos: &[RepoConfig]) -> String {
89        let mut output = String::new();
90
91        // Header
92        output.push_str("# Foundry Benchmark Results\n\n");
93        output.push_str(&format!(
94            "**Date**: {}\n\n",
95            chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
96        ));
97
98        // Summary
99        output.push_str("## Summary\n\n");
100        // Count actual repos that have results
101        let mut repos_with_results = std::collections::HashSet::new();
102        for version_data in self.data.values() {
103            for repo_data in version_data.values() {
104                for repo_name in repo_data.keys() {
105                    repos_with_results.insert(repo_name.clone());
106                }
107            }
108        }
109        output.push_str(&format!(
110            "Benchmarked {} Foundry versions across {} repositories.\n\n",
111            versions.len(),
112            repos_with_results.len()
113        ));
114
115        // Repositories tested
116        output.push_str("### Repositories Tested\n\n");
117        for (i, repo) in repos.iter().enumerate() {
118            output.push_str(&format!(
119                "{}. [{}/{}](https://github.com/{}/{})\n",
120                i + 1,
121                repo.org,
122                repo.repo,
123                repo.org,
124                repo.repo
125            ));
126        }
127        output.push('\n');
128
129        // Versions tested
130        output.push_str("### Foundry Versions\n\n");
131        for version in versions {
132            if let Some(details) = self.version_details.get(version) {
133                output.push_str(&format!("- **{version}**: {}\n", details.trim()));
134            } else {
135                output.push_str(&format!("- {version}\n"));
136            }
137        }
138        output.push('\n');
139
140        // Results for each benchmark type
141        for (benchmark_name, version_data) in &self.data {
142            output.push_str(&self.generate_benchmark_table(
143                benchmark_name,
144                version_data,
145                versions,
146                repos,
147            ));
148        }
149
150        // System info
151        output.push_str("## System Information\n\n");
152        output.push_str(&format!("- **OS**: {}\n", std::env::consts::OS));
153        output.push_str(&format!(
154            "- **CPU**: {}\n",
155            thread::available_parallelism().map_or(1, |n| n.get())
156        ));
157        output.push_str(&format!(
158            "- **Rustc**: {}\n",
159            get_rustc_version().unwrap_or_else(|_| "unknown".to_string())
160        ));
161
162        output
163    }
164
165    /// Generate a complete markdown table for a single benchmark type
166    ///
167    /// This includes the section header, table header, separator, and all rows
168    fn generate_benchmark_table(
169        &self,
170        benchmark_name: &str,
171        version_data: &HashMap<String, HashMap<String, HyperfineResult>>,
172        versions: &[String],
173        repos: &[RepoConfig],
174    ) -> String {
175        let mut output = String::new();
176
177        // Section header
178        output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name)));
179
180        // Create table header
181        output.push_str("| Repository |");
182        for version in versions {
183            output.push_str(&format!(" {version} |"));
184        }
185        output.push('\n');
186
187        // Table separator
188        output.push_str("|------------|");
189        for _ in versions {
190            output.push_str("----------|");
191        }
192        output.push('\n');
193
194        // Table rows
195        output.push_str(&generate_table_rows(version_data, versions, repos));
196        output.push('\n');
197
198        output
199    }
200}
201
202/// Generate table rows for benchmark results
203///
204/// This function creates the markdown table rows for each repository,
205/// showing the benchmark results for each version.
206fn generate_table_rows(
207    version_data: &HashMap<String, HashMap<String, HyperfineResult>>,
208    versions: &[String],
209    repos: &[RepoConfig],
210) -> String {
211    let mut output = String::new();
212
213    for repo in repos {
214        output.push_str(&format!("| {} |", repo.name));
215
216        for version in versions {
217            let cell_content = get_benchmark_cell_content(version_data, version, &repo.name);
218            output.push_str(&format!(" {cell_content} |"));
219        }
220
221        output.push('\n');
222    }
223
224    output
225}
226
227/// Get the content for a single benchmark table cell
228///
229/// Returns the formatted duration or "N/A" if no data is available.
230/// The nested if-let statements handle the following cases:
231/// 1. Check if version data exists
232/// 2. Check if repository data exists for this version
233fn get_benchmark_cell_content(
234    version_data: &HashMap<String, HashMap<String, HyperfineResult>>,
235    version: &str,
236    repo_name: &str,
237) -> String {
238    // Check if we have data for this version
239    if let Some(repo_data) = version_data.get(version) &&
240    // Check if we have data for this repository
241        let Some(result) = repo_data.get(repo_name)
242    {
243        return format_duration_seconds(result.mean);
244    }
245
246    "N/A".to_string()
247}
248
249pub fn format_benchmark_name(name: &str) -> String {
250    match name {
251        "forge_test" => "Forge Test",
252        "forge_build_no_cache" => "Forge Build (No Cache)",
253        "forge_build_with_cache" => "Forge Build (With Cache)",
254        "forge_fuzz_test" => "Forge Fuzz Test",
255        "forge_coverage" => "Forge Coverage",
256        "forge_isolate_test" => "Forge Test (Isolated)",
257        _ => name,
258    }
259    .to_string()
260}
261
262pub fn format_duration_seconds(seconds: f64) -> String {
263    if seconds < 0.001 {
264        format!("{:.2} ms", seconds * 1000.0)
265    } else if seconds < 1.0 {
266        format!("{seconds:.3} s")
267    } else if seconds < 60.0 {
268        format!("{seconds:.2} s")
269    } else {
270        let minutes = (seconds / 60.0).floor();
271        let remaining_seconds = seconds % 60.0;
272        format!("{minutes:.0}m {remaining_seconds:.1}s")
273    }
274}
275
276pub fn get_rustc_version() -> Result<String> {
277    let output = Command::new("rustc").arg("--version").output()?;
278
279    Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
280}
foundry_bench/results.rs

foundry_bench/
results.rs
