foundry_bench/
lib.rs

1//! Foundry benchmark runner.
2
3use crate::results::{HyperfineOutput, HyperfineResult};
4use eyre::{Result, WrapErr};
5use foundry_common::{sh_eprintln, sh_println};
6use foundry_compilers::project_util::TempProject;
7use foundry_test_utils::util::clone_remote;
8use once_cell::sync::Lazy;
9use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
10use std::{
11    env,
12    path::{Path, PathBuf},
13    process::Command,
14    str::FromStr,
15};
16
17pub mod results;
18
19/// Default number of runs for benchmarks
20pub const RUNS: u32 = 5;
21
22/// Configuration for repositories to benchmark
23#[derive(Debug, Clone)]
24pub struct RepoConfig {
25    pub name: String,
26    pub org: String,
27    pub repo: String,
28    pub rev: String,
29}
30
31impl FromStr for RepoConfig {
32    type Err = eyre::Error;
33
34    fn from_str(spec: &str) -> Result<Self> {
35        // Split by ':' first to separate repo path from optional rev
36        let parts: Vec<&str> = spec.splitn(2, ':').collect();
37        let repo_path = parts[0];
38        let custom_rev = parts.get(1).copied();
39
40        // Now split the repo path by '/'
41        let path_parts: Vec<&str> = repo_path.split('/').collect();
42        if path_parts.len() != 2 {
43            eyre::bail!("Invalid repo format '{}'. Expected 'org/repo' or 'org/repo:rev'", spec);
44        }
45
46        let org = path_parts[0];
47        let repo = path_parts[1];
48
49        // Try to find this repo in BENCHMARK_REPOS to get the full config
50        let existing_config = BENCHMARK_REPOS.iter().find(|r| r.org == org && r.repo == repo);
51
52        let config = if let Some(existing) = existing_config {
53            // Use existing config but allow custom rev to override
54            let mut config = existing.clone();
55            if let Some(rev) = custom_rev {
56                config.rev = rev.to_string();
57            }
58            config
59        } else {
60            // Create new config with custom rev or default
61            // Name should follow the format: org-repo (with hyphen)
62            Self {
63                name: format!("{org}-{repo}"),
64                org: org.to_string(),
65                repo: repo.to_string(),
66                rev: custom_rev.unwrap_or("main").to_string(),
67            }
68        };
69
70        let _ = sh_println!("Parsed repo spec '{spec}' -> {config:?}");
71        Ok(config)
72    }
73}
74
75/// Available repositories for benchmarking
76pub fn default_benchmark_repos() -> Vec<RepoConfig> {
77    vec![
78        RepoConfig {
79            name: "ithacaxyz-account".to_string(),
80            org: "ithacaxyz".to_string(),
81            repo: "account".to_string(),
82            rev: "main".to_string(),
83        },
84        RepoConfig {
85            name: "solady".to_string(),
86            org: "Vectorized".to_string(),
87            repo: "solady".to_string(),
88            rev: "main".to_string(),
89        },
90    ]
91}
92
93// Keep a lazy static for compatibility
94pub static BENCHMARK_REPOS: Lazy<Vec<RepoConfig>> = Lazy::new(default_benchmark_repos);
95
96/// Foundry versions to benchmark
97///
98/// To add more versions for comparison, install them first:
99/// ```bash
100/// foundryup --install stable
101/// foundryup --install nightly
102/// foundryup --install v0.2.0  # Example specific version
103/// ```
104///
105/// Then add the version strings to this array. Supported formats:
106/// - "stable" - Latest stable release
107/// - "nightly" - Latest nightly build
108/// - "v0.2.0" - Specific version tag
109/// - "commit-hash" - Specific commit hash
110/// - "nightly-rev" - Nightly build with specific revision
111pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"];
112
113/// A benchmark project that represents a cloned repository ready for testing
114pub struct BenchmarkProject {
115    pub name: String,
116    pub temp_project: TempProject,
117    pub root_path: PathBuf,
118}
119
120impl BenchmarkProject {
121    /// Set up a benchmark project by cloning the repository
122    #[allow(unused_must_use)]
123    pub fn setup(config: &RepoConfig) -> Result<Self> {
124        let temp_project =
125            TempProject::dapptools().wrap_err("Failed to create temporary project")?;
126
127        // Get root path before clearing
128        let root_path = temp_project.root().to_path_buf();
129        let root = root_path.to_str().unwrap();
130
131        // Remove all files in the directory
132        for entry in std::fs::read_dir(&root_path)? {
133            let entry = entry?;
134            let path = entry.path();
135            if path.is_dir() {
136                std::fs::remove_dir_all(&path).ok();
137            } else {
138                std::fs::remove_file(&path).ok();
139            }
140        }
141
142        // Clone the repository
143        let repo_url = format!("https://github.com/{}/{}.git", config.org, config.repo);
144        clone_remote(&repo_url, root, true);
145
146        // Checkout specific revision if provided
147        if !config.rev.is_empty() && config.rev != "main" && config.rev != "master" {
148            let status = Command::new("git")
149                .current_dir(root)
150                .args(["checkout", &config.rev])
151                .status()
152                .wrap_err("Failed to checkout revision")?;
153
154            if !status.success() {
155                eyre::bail!("Git checkout failed for {}", config.name);
156            }
157        }
158
159        // Git submodules are already cloned via --recursive flag
160        // But npm dependencies still need to be installed
161        Self::install_npm_dependencies(&root_path)?;
162
163        sh_println!("  ✅ Project {} setup complete at {}", config.name, root);
164        Ok(Self { name: config.name.to_string(), root_path, temp_project })
165    }
166
167    /// Install npm dependencies if package.json exists
168    #[allow(unused_must_use)]
169    fn install_npm_dependencies(root: &Path) -> Result<()> {
170        if root.join("package.json").exists() {
171            sh_println!("  📦 Running npm install...");
172            let status = Command::new("npm")
173                .current_dir(root)
174                .args(["install"])
175                .stdout(std::process::Stdio::inherit())
176                .stderr(std::process::Stdio::inherit())
177                .status()
178                .wrap_err("Failed to run npm install")?;
179
180            if !status.success() {
181                sh_println!(
182                    "  ⚠️  Warning: npm install failed with exit code: {:?}",
183                    status.code()
184                );
185            } else {
186                sh_println!("  ✅ npm install completed successfully");
187            }
188        }
189        Ok(())
190    }
191
192    /// Run a command with hyperfine and return the results
193    ///
194    /// # Arguments
195    /// * `benchmark_name` - Name of the benchmark for organizing output
196    /// * `version` - Foundry version being benchmarked
197    /// * `command` - The command to benchmark
198    /// * `runs` - Number of runs to perform
199    /// * `setup` - Optional setup command to run before the benchmark series (e.g., "forge build")
200    /// * `prepare` - Optional prepare command to run before each timing run (e.g., "forge clean")
201    /// * `conclude` - Optional conclude command to run after each timing run (e.g., cleanup)
202    /// * `verbose` - Whether to show command output
203    ///
204    /// # Hyperfine flags used:
205    /// * `--runs` - Number of timing runs
206    /// * `--setup` - Execute before the benchmark series (not before each run)
207    /// * `--prepare` - Execute before each timing run
208    /// * `--conclude` - Execute after each timing run
209    /// * `--export-json` - Export results to JSON for parsing
210    /// * `--shell=bash` - Use bash for shell command execution
211    /// * `--show-output` - Show command output (when verbose)
212    #[allow(clippy::too_many_arguments)]
213    fn hyperfine(
214        &self,
215        benchmark_name: &str,
216        version: &str,
217        command: &str,
218        runs: u32,
219        setup: Option<&str>,
220        prepare: Option<&str>,
221        conclude: Option<&str>,
222        verbose: bool,
223    ) -> Result<HyperfineResult> {
224        // Create structured temp directory for JSON output
225        // Format: <temp_dir>/<benchmark_name>/<version>/<repo_name>/<benchmark_name>.json
226        let temp_dir = std::env::temp_dir();
227        let json_dir =
228            temp_dir.join("foundry-bench").join(benchmark_name).join(version).join(&self.name);
229        std::fs::create_dir_all(&json_dir)?;
230
231        let json_path = json_dir.join(format!("{benchmark_name}.json"));
232
233        // Build hyperfine command
234        let mut hyperfine_cmd = Command::new("hyperfine");
235        hyperfine_cmd
236            .current_dir(&self.root_path)
237            .arg("--runs")
238            .arg(runs.to_string())
239            .arg("--export-json")
240            .arg(&json_path)
241            .arg("--shell=bash");
242
243        // Add optional setup command
244        if let Some(setup_cmd) = setup {
245            hyperfine_cmd.arg("--setup").arg(setup_cmd);
246        }
247
248        // Add optional prepare command
249        if let Some(prepare_cmd) = prepare {
250            hyperfine_cmd.arg("--prepare").arg(prepare_cmd);
251        }
252
253        // Add optional conclude command
254        if let Some(conclude_cmd) = conclude {
255            hyperfine_cmd.arg("--conclude").arg(conclude_cmd);
256        }
257
258        if verbose {
259            hyperfine_cmd.arg("--show-output");
260            hyperfine_cmd.stderr(std::process::Stdio::inherit());
261            hyperfine_cmd.stdout(std::process::Stdio::inherit());
262        }
263
264        // Add the benchmark command last
265        hyperfine_cmd.arg(command);
266
267        let status = hyperfine_cmd.status().wrap_err("Failed to run hyperfine")?;
268        if !status.success() {
269            eyre::bail!("Hyperfine failed for command: {}", command);
270        }
271
272        // Read and parse the JSON output
273        let json_content = std::fs::read_to_string(json_path)?;
274        let output: HyperfineOutput = serde_json::from_str(&json_content)?;
275
276        // Extract the first result (we only run one command at a time)
277        output.results.into_iter().next().ok_or_else(|| eyre::eyre!("No results from hyperfine"))
278    }
279
280    /// Benchmark forge test
281    pub fn bench_forge_test(
282        &self,
283        version: &str,
284        runs: u32,
285        verbose: bool,
286    ) -> Result<HyperfineResult> {
287        // Build before running tests
288        self.hyperfine(
289            "forge_test",
290            version,
291            "forge test",
292            runs,
293            Some("forge build"),
294            None,
295            None,
296            verbose,
297        )
298    }
299
300    /// Benchmark forge build with cache
301    pub fn bench_forge_build_with_cache(
302        &self,
303        version: &str,
304        runs: u32,
305        verbose: bool,
306    ) -> Result<HyperfineResult> {
307        self.hyperfine(
308            "forge_build_with_cache",
309            version,
310            "FOUNDRY_LINT_LINT_ON_BUILD=false forge build",
311            runs,
312            None,
313            Some("forge build"),
314            None,
315            verbose,
316        )
317    }
318
319    /// Benchmark forge build without cache
320    pub fn bench_forge_build_no_cache(
321        &self,
322        version: &str,
323        runs: u32,
324        verbose: bool,
325    ) -> Result<HyperfineResult> {
326        // Clean before each timing run
327        self.hyperfine(
328            "forge_build_no_cache",
329            version,
330            "FOUNDRY_LINT_LINT_ON_BUILD=false forge build",
331            runs,
332            Some("forge clean"),
333            None,
334            Some("forge clean"),
335            verbose,
336        )
337    }
338
339    /// Benchmark forge fuzz tests
340    pub fn bench_forge_fuzz_test(
341        &self,
342        version: &str,
343        runs: u32,
344        verbose: bool,
345    ) -> Result<HyperfineResult> {
346        // Build before running fuzz tests
347        self.hyperfine(
348            "forge_fuzz_test",
349            version,
350            r#"forge test --match-test "test[^(]*\([^)]+\)""#,
351            runs,
352            Some("forge build"),
353            None,
354            None,
355            verbose,
356        )
357    }
358
359    /// Benchmark forge coverage
360    pub fn bench_forge_coverage(
361        &self,
362        version: &str,
363        runs: u32,
364        verbose: bool,
365    ) -> Result<HyperfineResult> {
366        // No setup needed, forge coverage builds internally
367        // Use --ir-minimum to avoid "Stack too deep" errors
368        self.hyperfine(
369            "forge_coverage",
370            version,
371            "forge coverage --ir-minimum",
372            runs,
373            None,
374            None,
375            None,
376            verbose,
377        )
378    }
379
380    /// Benchmark forge test with --isolate flag
381    pub fn bench_forge_isolate_test(
382        &self,
383        version: &str,
384        runs: u32,
385        verbose: bool,
386    ) -> Result<HyperfineResult> {
387        // Build before running tests
388        self.hyperfine(
389            "forge_isolate_test",
390            version,
391            "forge test --isolate",
392            runs,
393            Some("forge build"),
394            None,
395            None,
396            verbose,
397        )
398    }
399
400    /// Get the root path of the project
401    pub fn root(&self) -> &Path {
402        &self.root_path
403    }
404
405    /// Run a specific benchmark by name
406    pub fn run(
407        &self,
408        benchmark: &str,
409        version: &str,
410        runs: u32,
411        verbose: bool,
412    ) -> Result<HyperfineResult> {
413        match benchmark {
414            "forge_test" => self.bench_forge_test(version, runs, verbose),
415            "forge_build_no_cache" => self.bench_forge_build_no_cache(version, runs, verbose),
416            "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose),
417            "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose),
418            "forge_coverage" => self.bench_forge_coverage(version, runs, verbose),
419            "forge_isolate_test" => self.bench_forge_isolate_test(version, runs, verbose),
420            _ => eyre::bail!("Unknown benchmark: {}", benchmark),
421        }
422    }
423}
424
425/// Switch to a specific foundry version
426#[allow(unused_must_use)]
427pub fn switch_foundry_version(version: &str) -> Result<()> {
428    let output = Command::new("foundryup")
429        .args(["--use", version])
430        .output()
431        .wrap_err("Failed to run foundryup")?;
432
433    // Check if the error is about forge --version failing
434    let stderr = String::from_utf8_lossy(&output.stderr);
435    if stderr.contains("command failed") && stderr.contains("forge --version") {
436        eyre::bail!(
437            "Foundry binaries maybe corrupted. Please reinstall by running `foundryup --install <version>`"
438        );
439    }
440
441    if !output.status.success() {
442        sh_eprintln!("foundryup stderr: {stderr}");
443        eyre::bail!("Failed to switch to foundry version: {}", version);
444    }
445
446    sh_println!("  Successfully switched to version: {version}");
447    Ok(())
448}
449
450/// Get the current forge version
451pub fn get_forge_version() -> Result<String> {
452    let output = Command::new("forge")
453        .args(["--version"])
454        .output()
455        .wrap_err("Failed to get forge version")?;
456
457    if !output.status.success() {
458        eyre::bail!("forge --version failed");
459    }
460
461    let version =
462        String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?;
463
464    Ok(version.lines().next().unwrap_or("unknown").to_string())
465}
466
467/// Get the full forge version details including commit hash and date
468pub fn get_forge_version_details() -> Result<String> {
469    let output = Command::new("forge")
470        .args(["--version"])
471        .output()
472        .wrap_err("Failed to get forge version")?;
473
474    if !output.status.success() {
475        eyre::bail!("forge --version failed");
476    }
477
478    let full_output =
479        String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?;
480
481    // Extract relevant lines and format them
482    let lines: Vec<&str> = full_output.lines().collect();
483    if lines.len() >= 3 {
484        // Extract version, commit, and timestamp
485        let version = lines[0].trim();
486        let commit = lines[1].trim().replace("Commit SHA: ", "");
487        let timestamp = lines[2].trim().replace("Build Timestamp: ", "");
488
489        // Format as: "forge 1.2.3-nightly (51650ea 2025-06-27)"
490        let short_commit = &commit[..7]; // First 7 chars of commit hash
491        let date = timestamp.split('T').next().unwrap_or(&timestamp);
492
493        Ok(format!("{version} ({short_commit} {date})"))
494    } else {
495        // Fallback to just the first line if format is unexpected
496        Ok(lines.first().unwrap_or(&"unknown").to_string())
497    }
498}
499
500/// Get Foundry versions to benchmark from environment variable or default
501///
502/// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set,
503/// otherwise returns the default versions from FOUNDRY_VERSIONS constant.
504///
505/// The environment variable should be a comma-separated list of versions,
506/// e.g., "stable,nightly,v1.2.0"
507pub fn get_benchmark_versions() -> Vec<String> {
508    if let Ok(versions_env) = env::var("FOUNDRY_BENCH_VERSIONS") {
509        versions_env.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()
510    } else {
511        FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect()
512    }
513}
514
515/// Setup Repositories for benchmarking
516pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> {
517    // Check for FOUNDRY_BENCH_REPOS environment variable
518    let repos = if let Ok(repos_env) = env::var("FOUNDRY_BENCH_REPOS") {
519        // Parse repo specs from the environment variable
520        // Format should be: "org1/repo1,org2/repo2"
521        repos_env
522            .split(',')
523            .map(|s| s.trim())
524            .filter(|s| !s.is_empty())
525            .map(|s| s.parse::<RepoConfig>())
526            .collect::<Result<Vec<_>>>()
527            .expect("Failed to parse FOUNDRY_BENCH_REPOS")
528    } else {
529        BENCHMARK_REPOS.clone()
530    };
531
532    repos
533        .par_iter()
534        .map(|repo_config| {
535            let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project");
536            (repo_config.clone(), project)
537        })
538        .collect()
539}
foundry_bench/lib.rs

foundry_bench/
lib.rs
