foundry_bench/
lib.rs

Search
1use crate::results::{HyperfineOutput, HyperfineResult};
2use eyre::{Result, WrapErr};
3use foundry_common::{sh_eprintln, sh_println};
4use foundry_compilers::project_util::TempProject;
5use foundry_test_utils::util::clone_remote;
6use once_cell::sync::Lazy;
7use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
8use std::{
9    env,
10    path::{Path, PathBuf},
11    process::Command,
12    str::FromStr,
13};
14
15pub mod results;
16
17/// Default number of runs for benchmarks
18pub const RUNS: u32 = 5;
19
20/// Configuration for repositories to benchmark
21#[derive(Debug, Clone)]
22pub struct RepoConfig {
23    pub name: String,
24    pub org: String,
25    pub repo: String,
26    pub rev: String,
27}
28
29impl FromStr for RepoConfig {
30    type Err = eyre::Error;
31
32    fn from_str(spec: &str) -> Result<Self> {
33        // Split by ':' first to separate repo path from optional rev
34        let parts: Vec<&str> = spec.splitn(2, ':').collect();
35        let repo_path = parts[0];
36        let custom_rev = parts.get(1).copied();
37
38        // Now split the repo path by '/'
39        let path_parts: Vec<&str> = repo_path.split('/').collect();
40        if path_parts.len() != 2 {
41            eyre::bail!("Invalid repo format '{}'. Expected 'org/repo' or 'org/repo:rev'", spec);
42        }
43
44        let org = path_parts[0];
45        let repo = path_parts[1];
46
47        // Try to find this repo in BENCHMARK_REPOS to get the full config
48        let existing_config = BENCHMARK_REPOS.iter().find(|r| r.org == org && r.repo == repo);
49
50        let config = if let Some(existing) = existing_config {
51            // Use existing config but allow custom rev to override
52            let mut config = existing.clone();
53            if let Some(rev) = custom_rev {
54                config.rev = rev.to_string();
55            }
56            config
57        } else {
58            // Create new config with custom rev or default
59            // Name should follow the format: org-repo (with hyphen)
60            RepoConfig {
61                name: format!("{org}-{repo}"),
62                org: org.to_string(),
63                repo: repo.to_string(),
64                rev: custom_rev.unwrap_or("main").to_string(),
65            }
66        };
67
68        let _ = sh_println!("Parsed repo spec '{spec}' -> {config:?}");
69        Ok(config)
70    }
71}
72
73/// Available repositories for benchmarking
74pub fn default_benchmark_repos() -> Vec<RepoConfig> {
75    vec![
76        RepoConfig {
77            name: "ithacaxyz-account".to_string(),
78            org: "ithacaxyz".to_string(),
79            repo: "account".to_string(),
80            rev: "main".to_string(),
81        },
82        RepoConfig {
83            name: "solady".to_string(),
84            org: "Vectorized".to_string(),
85            repo: "solady".to_string(),
86            rev: "main".to_string(),
87        },
88    ]
89}
90
91// Keep a lazy static for compatibility
92pub static BENCHMARK_REPOS: Lazy<Vec<RepoConfig>> = Lazy::new(default_benchmark_repos);
93
94/// Foundry versions to benchmark
95///
96/// To add more versions for comparison, install them first:
97/// ```bash
98/// foundryup --install stable
99/// foundryup --install nightly
100/// foundryup --install v0.2.0  # Example specific version
101/// ```
102///
103/// Then add the version strings to this array. Supported formats:
104/// - "stable" - Latest stable release
105/// - "nightly" - Latest nightly build
106/// - "v0.2.0" - Specific version tag
107/// - "commit-hash" - Specific commit hash
108/// - "nightly-rev" - Nightly build with specific revision
109pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"];
110
111/// A benchmark project that represents a cloned repository ready for testing
112pub struct BenchmarkProject {
113    pub name: String,
114    pub temp_project: TempProject,
115    pub root_path: PathBuf,
116}
117
118impl BenchmarkProject {
119    /// Set up a benchmark project by cloning the repository
120    #[allow(unused_must_use)]
121    pub fn setup(config: &RepoConfig) -> Result<Self> {
122        let temp_project =
123            TempProject::dapptools().wrap_err("Failed to create temporary project")?;
124
125        // Get root path before clearing
126        let root_path = temp_project.root().to_path_buf();
127        let root = root_path.to_str().unwrap();
128
129        // Remove all files in the directory
130        for entry in std::fs::read_dir(&root_path)? {
131            let entry = entry?;
132            let path = entry.path();
133            if path.is_dir() {
134                std::fs::remove_dir_all(&path).ok();
135            } else {
136                std::fs::remove_file(&path).ok();
137            }
138        }
139
140        // Clone the repository
141        let repo_url = format!("https://github.com/{}/{}.git", config.org, config.repo);
142        clone_remote(&repo_url, root);
143
144        // Checkout specific revision if provided
145        if !config.rev.is_empty() && config.rev != "main" && config.rev != "master" {
146            let status = Command::new("git")
147                .current_dir(root)
148                .args(["checkout", &config.rev])
149                .status()
150                .wrap_err("Failed to checkout revision")?;
151
152            if !status.success() {
153                eyre::bail!("Git checkout failed for {}", config.name);
154            }
155        }
156
157        // Git submodules are already cloned via --recursive flag
158        // But npm dependencies still need to be installed
159        Self::install_npm_dependencies(&root_path)?;
160
161        sh_println!("  ✅ Project {} setup complete at {}", config.name, root);
162        Ok(BenchmarkProject { name: config.name.to_string(), root_path, temp_project })
163    }
164
165    /// Install npm dependencies if package.json exists
166    #[allow(unused_must_use)]
167    fn install_npm_dependencies(root: &Path) -> Result<()> {
168        if root.join("package.json").exists() {
169            sh_println!("  📦 Running npm install...");
170            let status = Command::new("npm")
171                .current_dir(root)
172                .args(["install"])
173                .stdout(std::process::Stdio::inherit())
174                .stderr(std::process::Stdio::inherit())
175                .status()
176                .wrap_err("Failed to run npm install")?;
177
178            if !status.success() {
179                sh_println!(
180                    "  ⚠️  Warning: npm install failed with exit code: {:?}",
181                    status.code()
182                );
183            } else {
184                sh_println!("  ✅ npm install completed successfully");
185            }
186        }
187        Ok(())
188    }
189
190    /// Run a command with hyperfine and return the results
191    ///
192    /// # Arguments
193    /// * `benchmark_name` - Name of the benchmark for organizing output
194    /// * `version` - Foundry version being benchmarked
195    /// * `command` - The command to benchmark
196    /// * `runs` - Number of runs to perform
197    /// * `setup` - Optional setup command to run before the benchmark series (e.g., "forge build")
198    /// * `prepare` - Optional prepare command to run before each timing run (e.g., "forge clean")
199    /// * `conclude` - Optional conclude command to run after each timing run (e.g., cleanup)
200    /// * `verbose` - Whether to show command output
201    ///
202    /// # Hyperfine flags used:
203    /// * `--runs` - Number of timing runs
204    /// * `--setup` - Execute before the benchmark series (not before each run)
205    /// * `--prepare` - Execute before each timing run
206    /// * `--conclude` - Execute after each timing run
207    /// * `--export-json` - Export results to JSON for parsing
208    /// * `--shell=bash` - Use bash for shell command execution
209    /// * `--show-output` - Show command output (when verbose)
210    #[allow(clippy::too_many_arguments)]
211    fn hyperfine(
212        &self,
213        benchmark_name: &str,
214        version: &str,
215        command: &str,
216        runs: u32,
217        setup: Option<&str>,
218        prepare: Option<&str>,
219        conclude: Option<&str>,
220        verbose: bool,
221    ) -> Result<HyperfineResult> {
222        // Create structured temp directory for JSON output
223        // Format: <temp_dir>/<benchmark_name>/<version>/<repo_name>/<benchmark_name>.json
224        let temp_dir = std::env::temp_dir();
225        let json_dir =
226            temp_dir.join("foundry-bench").join(benchmark_name).join(version).join(&self.name);
227        std::fs::create_dir_all(&json_dir)?;
228
229        let json_path = json_dir.join(format!("{benchmark_name}.json"));
230
231        // Build hyperfine command
232        let mut hyperfine_cmd = Command::new("hyperfine");
233        hyperfine_cmd
234            .current_dir(&self.root_path)
235            .arg("--runs")
236            .arg(runs.to_string())
237            .arg("--export-json")
238            .arg(&json_path)
239            .arg("--shell=bash");
240
241        // Add optional setup command
242        if let Some(setup_cmd) = setup {
243            hyperfine_cmd.arg("--setup").arg(setup_cmd);
244        }
245
246        // Add optional prepare command
247        if let Some(prepare_cmd) = prepare {
248            hyperfine_cmd.arg("--prepare").arg(prepare_cmd);
249        }
250
251        // Add optional conclude command
252        if let Some(conclude_cmd) = conclude {
253            hyperfine_cmd.arg("--conclude").arg(conclude_cmd);
254        }
255
256        if verbose {
257            hyperfine_cmd.arg("--show-output");
258            hyperfine_cmd.stderr(std::process::Stdio::inherit());
259            hyperfine_cmd.stdout(std::process::Stdio::inherit());
260        }
261
262        // Add the benchmark command last
263        hyperfine_cmd.arg(command);
264
265        let status = hyperfine_cmd.status().wrap_err("Failed to run hyperfine")?;
266        if !status.success() {
267            eyre::bail!("Hyperfine failed for command: {}", command);
268        }
269
270        // Read and parse the JSON output
271        let json_content = std::fs::read_to_string(json_path)?;
272        let output: HyperfineOutput = serde_json::from_str(&json_content)?;
273
274        // Extract the first result (we only run one command at a time)
275        output.results.into_iter().next().ok_or_else(|| eyre::eyre!("No results from hyperfine"))
276    }
277
278    /// Benchmark forge test
279    pub fn bench_forge_test(
280        &self,
281        version: &str,
282        runs: u32,
283        verbose: bool,
284    ) -> Result<HyperfineResult> {
285        // Build before running tests
286        self.hyperfine(
287            "forge_test",
288            version,
289            "forge test",
290            runs,
291            Some("forge build"),
292            None,
293            None,
294            verbose,
295        )
296    }
297
298    /// Benchmark forge build with cache
299    pub fn bench_forge_build_with_cache(
300        &self,
301        version: &str,
302        runs: u32,
303        verbose: bool,
304    ) -> Result<HyperfineResult> {
305        self.hyperfine(
306            "forge_build_with_cache",
307            version,
308            "FOUNDRY_LINT_LINT_ON_BUILD=false forge build",
309            runs,
310            None,
311            Some("forge build"),
312            None,
313            verbose,
314        )
315    }
316
317    /// Benchmark forge build without cache
318    pub fn bench_forge_build_no_cache(
319        &self,
320        version: &str,
321        runs: u32,
322        verbose: bool,
323    ) -> Result<HyperfineResult> {
324        // Clean before each timing run
325        self.hyperfine(
326            "forge_build_no_cache",
327            version,
328            "FOUNDRY_LINT_LINT_ON_BUILD=false forge build",
329            runs,
330            Some("forge clean"),
331            None,
332            Some("forge clean"),
333            verbose,
334        )
335    }
336
337    /// Benchmark forge fuzz tests
338    pub fn bench_forge_fuzz_test(
339        &self,
340        version: &str,
341        runs: u32,
342        verbose: bool,
343    ) -> Result<HyperfineResult> {
344        // Build before running fuzz tests
345        self.hyperfine(
346            "forge_fuzz_test",
347            version,
348            r#"forge test --match-test "test[^(]*\([^)]+\)""#,
349            runs,
350            Some("forge build"),
351            None,
352            None,
353            verbose,
354        )
355    }
356
357    /// Benchmark forge coverage
358    pub fn bench_forge_coverage(
359        &self,
360        version: &str,
361        runs: u32,
362        verbose: bool,
363    ) -> Result<HyperfineResult> {
364        // No setup needed, forge coverage builds internally
365        // Use --ir-minimum to avoid "Stack too deep" errors
366        self.hyperfine(
367            "forge_coverage",
368            version,
369            "forge coverage --ir-minimum",
370            runs,
371            None,
372            None,
373            None,
374            verbose,
375        )
376    }
377
378    /// Benchmark forge test with --isolate flag
379    pub fn bench_forge_isolate_test(
380        &self,
381        version: &str,
382        runs: u32,
383        verbose: bool,
384    ) -> Result<HyperfineResult> {
385        // Build before running tests
386        self.hyperfine(
387            "forge_isolate_test",
388            version,
389            "forge test --isolate",
390            runs,
391            Some("forge build"),
392            None,
393            None,
394            verbose,
395        )
396    }
397
398    /// Get the root path of the project
399    pub fn root(&self) -> &Path {
400        &self.root_path
401    }
402
403    /// Run a specific benchmark by name
404    pub fn run(
405        &self,
406        benchmark: &str,
407        version: &str,
408        runs: u32,
409        verbose: bool,
410    ) -> Result<HyperfineResult> {
411        match benchmark {
412            "forge_test" => self.bench_forge_test(version, runs, verbose),
413            "forge_build_no_cache" => self.bench_forge_build_no_cache(version, runs, verbose),
414            "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose),
415            "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose),
416            "forge_coverage" => self.bench_forge_coverage(version, runs, verbose),
417            "forge_isolate_test" => self.bench_forge_isolate_test(version, runs, verbose),
418            _ => eyre::bail!("Unknown benchmark: {}", benchmark),
419        }
420    }
421}
422
423/// Switch to a specific foundry version
424#[allow(unused_must_use)]
425pub fn switch_foundry_version(version: &str) -> Result<()> {
426    let output = Command::new("foundryup")
427        .args(["--use", version])
428        .output()
429        .wrap_err("Failed to run foundryup")?;
430
431    // Check if the error is about forge --version failing
432    let stderr = String::from_utf8_lossy(&output.stderr);
433    if stderr.contains("command failed") && stderr.contains("forge --version") {
434        eyre::bail!(
435            "Foundry binaries maybe corrupted. Please reinstall by running `foundryup --install <version>`"
436        );
437    }
438
439    if !output.status.success() {
440        sh_eprintln!("foundryup stderr: {stderr}");
441        eyre::bail!("Failed to switch to foundry version: {}", version);
442    }
443
444    sh_println!("  Successfully switched to version: {version}");
445    Ok(())
446}
447
448/// Get the current forge version
449pub fn get_forge_version() -> Result<String> {
450    let output = Command::new("forge")
451        .args(["--version"])
452        .output()
453        .wrap_err("Failed to get forge version")?;
454
455    if !output.status.success() {
456        eyre::bail!("forge --version failed");
457    }
458
459    let version =
460        String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?;
461
462    Ok(version.lines().next().unwrap_or("unknown").to_string())
463}
464
465/// Get the full forge version details including commit hash and date
466pub fn get_forge_version_details() -> Result<String> {
467    let output = Command::new("forge")
468        .args(["--version"])
469        .output()
470        .wrap_err("Failed to get forge version")?;
471
472    if !output.status.success() {
473        eyre::bail!("forge --version failed");
474    }
475
476    let full_output =
477        String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?;
478
479    // Extract relevant lines and format them
480    let lines: Vec<&str> = full_output.lines().collect();
481    if lines.len() >= 3 {
482        // Extract version, commit, and timestamp
483        let version = lines[0].trim();
484        let commit = lines[1].trim().replace("Commit SHA: ", "");
485        let timestamp = lines[2].trim().replace("Build Timestamp: ", "");
486
487        // Format as: "forge 1.2.3-nightly (51650ea 2025-06-27)"
488        let short_commit = &commit[..7]; // First 7 chars of commit hash
489        let date = timestamp.split('T').next().unwrap_or(&timestamp);
490
491        Ok(format!("{version} ({short_commit} {date})"))
492    } else {
493        // Fallback to just the first line if format is unexpected
494        Ok(lines.first().unwrap_or(&"unknown").to_string())
495    }
496}
497
498/// Get Foundry versions to benchmark from environment variable or default
499///
500/// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set,
501/// otherwise returns the default versions from FOUNDRY_VERSIONS constant.
502///
503/// The environment variable should be a comma-separated list of versions,
504/// e.g., "stable,nightly,v1.2.0"
505pub fn get_benchmark_versions() -> Vec<String> {
506    if let Ok(versions_env) = env::var("FOUNDRY_BENCH_VERSIONS") {
507        versions_env.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()
508    } else {
509        FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect()
510    }
511}
512
513/// Setup Repositories for benchmarking
514pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> {
515    // Check for FOUNDRY_BENCH_REPOS environment variable
516    let repos = if let Ok(repos_env) = env::var("FOUNDRY_BENCH_REPOS") {
517        // Parse repo specs from the environment variable
518        // Format should be: "org1/repo1,org2/repo2"
519        repos_env
520            .split(',')
521            .map(|s| s.trim())
522            .filter(|s| !s.is_empty())
523            .map(|s| s.parse::<RepoConfig>())
524            .collect::<Result<Vec<_>>>()
525            .expect("Failed to parse FOUNDRY_BENCH_REPOS")
526    } else {
527        BENCHMARK_REPOS.clone()
528    };
529
530    repos
531        .par_iter()
532        .map(|repo_config| {
533            let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project");
534            (repo_config.clone(), project)
535        })
536        .collect()
537}
foundry_bench/lib.rs

foundry_bench/
lib.rs
