Skip to main content

foundry_bench/
symbolic.rs

1//! Formal symbolic benchmark suite definitions and result parsing.
2
3use crate::results::SymbolicBenchmarkSummary;
4use eyre::{Result, WrapErr};
5use serde::Serialize;
6use serde_json::Value;
7use std::{
8    fs,
9    path::{Path, PathBuf},
10};
11
12const FARCASTER_PATH: &str = "test/FarcasterNativeSymbolic.t.sol";
13const FARCASTER_TEST: &str = include_str!("../fixtures/symbolic/FarcasterNativeSymbolic.t.sol");
14const PREFIX: &str =
15    "FOUNDRY_DYNAMIC_TEST_LINKING=false FOUNDRY_LINT_LINT_ON_BUILD=false FOUNDRY_ISOLATE=false";
16const SOLADY_MATCH: &str = "check_(SaturatingAddEquivalence|SaturatingMulEquivalence|HasDuplicateHashmapCapacityTrickEquivalence|IsPermit2AndValueIsNotInfinityTrickEquivalence|IsNotUint256MaxTrickEquivalence|DelayRestriction|OperationStateDifferentialTrick|CarryBoundsTrick|SafeCastInt256ToIntTrickEquivalence|P256Normalized|AuxPackEquivalence|EcrecoverTrickEquivalence|EcrecoverLoopTrick)";
17
18#[derive(Clone, Copy, Debug, Serialize)]
19#[serde(rename_all = "snake_case")]
20pub enum Fixture {
21    Solady,
22    Angstrom,
23    Farcaster,
24    FoundryXor,
25    Generic,
26}
27
28impl Fixture {
29    pub const fn identify(org: &str, repo: &str) -> Self {
30        if org.eq_ignore_ascii_case("vectorized") && repo.eq_ignore_ascii_case("solady") {
31            Self::Solady
32        } else if org.eq_ignore_ascii_case("sorellalabs") && repo.eq_ignore_ascii_case("angstrom") {
33            Self::Angstrom
34        } else if org.eq_ignore_ascii_case("farcasterxyz") && repo.eq_ignore_ascii_case("contracts")
35        {
36            Self::Farcaster
37        } else if org.eq_ignore_ascii_case("foundry-rs") && repo.eq_ignore_ascii_case("foundry") {
38            Self::FoundryXor
39        } else {
40            Self::Generic
41        }
42    }
43
44    pub fn build_command(self) -> String {
45        format!(
46            "{PREFIX} {}",
47            match self {
48                Self::Angstrom => "forge build --root contracts",
49                Self::FoundryXor => "forge build --root benches/fixtures/symbolic/xor",
50                _ => "forge build",
51            }
52        )
53    }
54
55    pub fn test_command(self) -> String {
56        match self {
57            Self::Solady => {
58                format!("{PREFIX} forge test --symbolic --json --match-test '{SOLADY_MATCH}'")
59            }
60            Self::Angstrom => format!(
61                "{PREFIX} forge test --root contracts --symbolic --json --symbolic-timeout 5 --match-path test/libraries/X128MathLib.t.sol --match-test check_matchesSolady_fullMulX128"
62            ),
63            Self::Farcaster => format!(
64                "{PREFIX} forge test --symbolic --json --symbolic-timeout 5 --match-path '{FARCASTER_PATH}' --match-test 'check_'"
65            ),
66            Self::FoundryXor => format!(
67                "{PREFIX} forge test --root benches/fixtures/symbolic/xor --symbolic --json"
68            ),
69            Self::Generic => format!("{PREFIX} forge test --symbolic --json"),
70        }
71    }
72}
73
74/// Error-safe installation of the Farcaster benchmark overlay.
75pub struct Overlay {
76    path: Option<PathBuf>,
77}
78
79impl Overlay {
80    pub fn install(root: &Path, fixture: Fixture) -> Result<Self> {
81        if !matches!(fixture, Fixture::Farcaster) {
82            return Ok(Self { path: None });
83        }
84        let path = root.join(FARCASTER_PATH);
85        if path.exists() {
86            eyre::bail!("refusing to overwrite existing {}", path.display());
87        }
88        if let Err(write_err) = fs::write(&path, FARCASTER_TEST) {
89            if let Err(cleanup_err) = fs::remove_file(&path)
90                && cleanup_err.kind() != std::io::ErrorKind::NotFound
91            {
92                eyre::bail!(
93                    "failed to install Farcaster symbolic fixture: {write_err}; failed to remove partially written {}: {cleanup_err}",
94                    path.display()
95                );
96            }
97            return Err(write_err).wrap_err("failed to install Farcaster symbolic fixture");
98        }
99        Ok(Self { path: Some(path) })
100    }
101
102    pub fn finish(mut self) -> Result<()> {
103        self.cleanup()
104    }
105
106    fn cleanup(&mut self) -> Result<()> {
107        if let Some(path) = &self.path {
108            fs::remove_file(path).wrap_err_with(|| {
109                format!("failed to remove symbolic fixture {}", path.display())
110            })?;
111            self.path = None;
112        }
113        Ok(())
114    }
115}
116
117impl Drop for Overlay {
118    fn drop(&mut self) {
119        let _ = self.cleanup();
120    }
121}
122
123#[derive(Clone, Debug, Default, Serialize)]
124pub struct Metrics {
125    pub paths: u64,
126    pub solver_queries: u64,
127    pub smt_queries: u64,
128    pub sat_queries: u64,
129    pub model_queries: u64,
130    pub sat_cache_hits: u64,
131    pub model_cache_hits: u64,
132    pub heuristic_witnesses: u64,
133    pub solver_time_ms: u64,
134    pub smt_input_bytes: Option<u64>,
135    pub smt_max_query_bytes: Option<u64>,
136    pub smt_build_time_ms: Option<u64>,
137    pub smt_max_query_time_ms: Option<u64>,
138    #[serde(skip)]
139    smt_input_bytes_compat: u64,
140    #[serde(skip)]
141    smt_max_query_bytes_compat: u64,
142    #[serde(skip)]
143    smt_build_time_ms_compat: u64,
144    #[serde(skip)]
145    smt_max_query_time_ms_compat: u64,
146}
147
148#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
149#[serde(rename_all = "snake_case")]
150pub enum OutcomeStatus {
151    Passed,
152    Failed,
153    Incomplete,
154}
155
156#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
157pub struct TestOutcome {
158    pub suite: String,
159    pub signature: String,
160    pub status: OutcomeStatus,
161}
162
163impl TestOutcome {
164    fn identity(&self) -> String {
165        format!("{}::{}", self.suite, self.signature)
166    }
167}
168
169#[derive(Clone, Debug, Serialize)]
170pub struct ParsedRun {
171    pub outcomes: Vec<TestOutcome>,
172    pub passed: usize,
173    pub failed: usize,
174    pub incomplete: usize,
175    pub metrics: Metrics,
176}
177
178#[derive(Clone, Debug, Serialize)]
179pub struct Sample {
180    pub wall_time_seconds: f64,
181    pub exit_code: i32,
182    #[serde(flatten)]
183    pub run: ParsedRun,
184}
185
186#[derive(Clone, Debug, Serialize)]
187pub struct Sidecar {
188    pub schema: &'static str,
189    pub schema_version: u32,
190    pub fixture: FixtureMetadata,
191    pub samples: Vec<Sample>,
192}
193
194#[derive(Clone, Debug, Serialize)]
195pub struct FixtureMetadata {
196    pub name: Fixture,
197    pub repository: String,
198    pub revision: String,
199    pub build_command: String,
200    pub test_command: String,
201}
202
203impl Sidecar {
204    pub fn new(
205        fixture: Fixture,
206        repository: &str,
207        revision: &str,
208        build_command: &str,
209        test_command: &str,
210        samples: Vec<Sample>,
211    ) -> Self {
212        Self {
213            schema: "foundry:symbolic-benchmark@v1",
214            schema_version: 1,
215            fixture: FixtureMetadata {
216                name: fixture,
217                repository: repository.to_string(),
218                revision: revision.to_string(),
219                build_command: build_command.to_string(),
220                test_command: test_command.to_string(),
221            },
222            samples,
223        }
224    }
225}
226
227pub fn parse(stdout: &[u8]) -> Result<ParsedRun> {
228    let json: Value =
229        serde_json::from_slice(stdout).wrap_err("invalid forge test --json output")?;
230    let suites = json.as_object().ok_or_else(|| eyre::eyre!("expected JSON object"))?;
231    let stable = suites
232        .values()
233        .filter_map(|s| s.get("test_results").and_then(Value::as_object))
234        .flat_map(|r| r.values())
235        .any(|r| r.get("symbolic").is_some());
236    let mut run = ParsedRun {
237        outcomes: Vec::new(),
238        passed: 0,
239        failed: 0,
240        incomplete: 0,
241        metrics: Metrics::default(),
242    };
243    for (suite_name, suite) in suites {
244        let Some(results) = suite.get("test_results").and_then(Value::as_object) else { continue };
245        for (identity, result) in results {
246            let (stats, status) = if stable {
247                let Some(symbolic) = result.get("symbolic") else { continue };
248                if symbolic.get("schema_version").and_then(Value::as_u64) != Some(1) {
249                    eyre::bail!("unknown or missing symbolic schema_version");
250                }
251                let status = symbolic
252                    .get("status")
253                    .and_then(Value::as_str)
254                    .ok_or_else(|| eyre::eyre!("missing symbolic.status"))?;
255                let status = match status {
256                    "pass" => OutcomeStatus::Passed,
257                    "fail_counterexample" => OutcomeStatus::Failed,
258                    "incomplete" => OutcomeStatus::Incomplete,
259                    _ => {
260                        eyre::bail!("unknown symbolic.status {status}");
261                    }
262                };
263                let stats = symbolic
264                    .pointer("/solver/stats")
265                    .filter(|stats| stats.is_object())
266                    .ok_or_else(|| eyre::eyre!("missing or invalid symbolic.solver.stats"))?;
267                (stats, status)
268            } else {
269                let Some(stats) = result.pointer("/kind/Symbolic") else { continue };
270                let status = result.get("status").and_then(Value::as_str).unwrap_or_default();
271                let reason = result.get("reason").and_then(Value::as_str).unwrap_or_default();
272                let status = if status == "Success" {
273                    OutcomeStatus::Passed
274                } else if reason.contains("incomplete symbolic execution") {
275                    OutcomeStatus::Incomplete
276                } else {
277                    OutcomeStatus::Failed
278                };
279                (stats, status)
280            };
281            run.outcomes.push(TestOutcome {
282                suite: suite_name.clone(),
283                signature: identity.clone(),
284                status,
285            });
286            add_metrics(&mut run.metrics, stats, stable, run.outcomes.len() == 1)?;
287        }
288    }
289    if run.outcomes.is_empty() {
290        eyre::bail!("forge symbolic benchmark produced no symbolic test results");
291    }
292    run.outcomes.sort_by_key(TestOutcome::identity);
293    run.passed =
294        run.outcomes.iter().filter(|outcome| outcome.status == OutcomeStatus::Passed).count();
295    run.failed =
296        run.outcomes.iter().filter(|outcome| outcome.status == OutcomeStatus::Failed).count();
297    run.incomplete =
298        run.outcomes.iter().filter(|outcome| outcome.status == OutcomeStatus::Incomplete).count();
299    Ok(run)
300}
301
302fn add_metrics(out: &mut Metrics, stats: &Value, stable: bool, first: bool) -> Result<()> {
303    macro_rules! add {
304        ($field:ident) => {
305            out.$field += required(stats, stringify!($field), stable)?;
306        };
307    }
308    add!(paths);
309    add!(solver_queries);
310    add!(smt_queries);
311    add!(sat_queries);
312    add!(model_queries);
313    add!(sat_cache_hits);
314    add!(model_cache_hits);
315    add!(heuristic_witnesses);
316    add!(solver_time_ms);
317    optional_add(
318        &mut out.smt_input_bytes,
319        &mut out.smt_input_bytes_compat,
320        stats,
321        "smt_input_bytes",
322        first,
323    )?;
324    optional_add(
325        &mut out.smt_build_time_ms,
326        &mut out.smt_build_time_ms_compat,
327        stats,
328        "smt_build_time_ms",
329        first,
330    )?;
331    optional_max(
332        &mut out.smt_max_query_bytes,
333        &mut out.smt_max_query_bytes_compat,
334        stats,
335        "smt_max_query_bytes",
336        first,
337    )?;
338    optional_max(
339        &mut out.smt_max_query_time_ms,
340        &mut out.smt_max_query_time_ms_compat,
341        stats,
342        "smt_max_query_time_ms",
343        first,
344    )?;
345    Ok(())
346}
347
348fn required(v: &Value, key: &str, strict: bool) -> Result<u64> {
349    match v.get(key) {
350        Some(v) => v.as_u64().ok_or_else(|| eyre::eyre!("invalid metric {key}")),
351        None if strict => Err(eyre::eyre!("missing metric {key}")),
352        None => Ok(0),
353    }
354}
355
356fn optional_add(
357    out: &mut Option<u64>,
358    compatibility: &mut u64,
359    v: &Value,
360    key: &str,
361    first: bool,
362) -> Result<()> {
363    let value = v
364        .get(key)
365        .map(|n| n.as_u64().ok_or_else(|| eyre::eyre!("invalid metric {key}")))
366        .transpose()?;
367    *compatibility += value.unwrap_or_default();
368    *out = if first { value } else { (*out).zip(value).map(|(old, value)| old + value) };
369    Ok(())
370}
371
372fn optional_max(
373    out: &mut Option<u64>,
374    compatibility: &mut u64,
375    v: &Value,
376    key: &str,
377    first: bool,
378) -> Result<()> {
379    let value = v
380        .get(key)
381        .map(|n| n.as_u64().ok_or_else(|| eyre::eyre!("invalid metric {key}")))
382        .transpose()?;
383    *compatibility = (*compatibility).max(value.unwrap_or_default());
384    *out = if first { value } else { (*out).zip(value).map(|(old, value)| old.max(value)) };
385    Ok(())
386}
387
388pub const fn compatibility(run: &ParsedRun) -> SymbolicBenchmarkSummary {
389    let m = &run.metrics;
390    SymbolicBenchmarkSummary {
391        tests: run.outcomes.len(),
392        passed: run.passed,
393        failed: run.failed,
394        incomplete: run.incomplete,
395        paths: m.paths,
396        solver_queries: m.solver_queries,
397        smt_queries: m.smt_queries,
398        sat_queries: m.sat_queries,
399        model_queries: m.model_queries,
400        sat_cache_hits: m.sat_cache_hits,
401        model_cache_hits: m.model_cache_hits,
402        heuristic_witnesses: m.heuristic_witnesses,
403        solver_time_ms: m.solver_time_ms,
404        smt_input_bytes: m.smt_input_bytes_compat,
405        smt_max_query_bytes: m.smt_max_query_bytes_compat,
406        smt_build_time_ms: m.smt_build_time_ms_compat,
407        smt_max_query_time_ms: m.smt_max_query_time_ms_compat,
408    }
409}
410
411#[cfg(test)]
412mod tests {
413    use super::*;
414
415    fn stable(status: &str, schema: u64) -> Vec<u8> {
416        serde_json::to_vec(&serde_json::json!({"test/FarcasterNativeSymbolic.t.sol:FarcasterNativeSymbolicTest":{"test_results":{
417            "check_migrateOnlyMigrator(uint24,address,address,address,uint40)":{"symbolic":{"schema_version":schema,"status":status,"bounds":{},"solver":{"name":"z3","command":null,"portfolio":[],"stats":{"paths":1,"solver_queries":2,"smt_queries":3,"sat_queries":4,"model_queries":5,"sat_cache_hits":6,"model_cache_hits":7,"heuristic_witnesses":8,"solver_time_ms":9,"smt_input_bytes":10,"smt_max_query_bytes":11,"smt_build_time_ms":12,"smt_max_query_time_ms":13}}}},
418            "check_setMigratorOnlyOwner(uint24,address,address,address,address)":{"symbolic":{"schema_version":schema,"status":"incomplete","bounds":{},"solver":{"name":"z3","command":null,"portfolio":[],"stats":{"paths":10,"solver_queries":20,"smt_queries":30,"sat_queries":40,"model_queries":50,"sat_cache_hits":60,"model_cache_hits":70,"heuristic_witnesses":80,"solver_time_ms":90}}}}
419        }}})).unwrap()
420    }
421
422    #[test]
423    fn stable_parser_validates_statuses_and_missing_optional_metrics() {
424        let run = parse(&stable("pass", 1)).unwrap();
425        assert_eq!((run.passed, run.failed, run.incomplete), (1, 0, 1));
426        assert_eq!(run.metrics.paths, 11);
427        assert_eq!(run.metrics.solver_time_ms, 99);
428        assert_eq!(run.metrics.smt_max_query_bytes, None);
429        assert_eq!(run.metrics.smt_input_bytes, None);
430        assert_eq!(run.metrics.smt_build_time_ms, None);
431        assert_eq!(run.metrics.smt_max_query_time_ms, None);
432        assert_eq!(
433            run.outcomes[0].suite,
434            "test/FarcasterNativeSymbolic.t.sol:FarcasterNativeSymbolicTest"
435        );
436        let compatibility = compatibility(&run);
437        assert_eq!(compatibility.tests, 2);
438        assert_eq!(compatibility.smt_input_bytes, 10);
439    }
440
441    #[test]
442    fn rejects_unknown_stable_schema() {
443        assert!(parse(&stable("pass", 2)).is_err());
444    }
445
446    #[test]
447    fn aggregates_optional_metrics_only_when_every_test_reports_them() {
448        let first = serde_json::json!({
449            "paths": 1, "solver_queries": 1, "smt_queries": 1, "sat_queries": 1,
450            "model_queries": 1, "sat_cache_hits": 1, "model_cache_hits": 1,
451            "heuristic_witnesses": 1, "solver_time_ms": 1, "smt_input_bytes": 10,
452            "smt_max_query_bytes": 8, "smt_build_time_ms": 2, "smt_max_query_time_ms": 4
453        });
454        let second = serde_json::json!({
455            "paths": 1, "solver_queries": 1, "smt_queries": 1, "sat_queries": 1,
456            "model_queries": 1, "sat_cache_hits": 1, "model_cache_hits": 1,
457            "heuristic_witnesses": 1, "solver_time_ms": 1, "smt_input_bytes": 20,
458            "smt_max_query_bytes": 7, "smt_build_time_ms": 3, "smt_max_query_time_ms": 9
459        });
460        let mut metrics = Metrics::default();
461        add_metrics(&mut metrics, &first, true, true).unwrap();
462        add_metrics(&mut metrics, &second, true, false).unwrap();
463        assert_eq!(metrics.smt_input_bytes, Some(30));
464        assert_eq!(metrics.smt_build_time_ms, Some(5));
465        assert_eq!(metrics.smt_max_query_bytes, Some(8));
466        assert_eq!(metrics.smt_max_query_time_ms, Some(9));
467    }
468
469    #[test]
470    fn parses_whole_output_legacy_results() {
471        let stats = serde_json::json!({"paths": 1, "solver_queries": 2});
472        let stdout = serde_json::to_vec(&serde_json::json!({"test/FarcasterNativeSymbolic.t.sol:FarcasterNativeSymbolicTest":{"test_results":{
473            "check_migrateOnlyMigrator(uint24,address,address,address,uint40)":{
474                "kind":{"Symbolic":stats},"status":"Success"
475            },
476            "check_setMigratorOnlyOwner(uint24,address,address,address,address)":{
477                "kind":{"Symbolic":stats},"status":"Failure",
478                "reason":"incomplete symbolic execution (Timeout)"
479            }
480        }}}))
481        .unwrap();
482        let run = parse(&stdout).unwrap();
483        assert_eq!((run.passed, run.failed, run.incomplete), (1, 0, 1));
484        assert_eq!(run.metrics.paths, 2);
485    }
486
487    #[test]
488    fn unknown_repository_uses_generic_symbolic_command() {
489        let fixture = Fixture::identify("example", "contracts");
490        assert!(matches!(fixture, Fixture::Generic));
491        assert_eq!(fixture.test_command(), format!("{PREFIX} forge test --symbolic --json"));
492    }
493
494    #[test]
495    fn metrics_serialize_the_complete_v1_counter_set() {
496        let value = serde_json::to_value(Metrics::default()).unwrap();
497        let keys = value
498            .as_object()
499            .unwrap()
500            .keys()
501            .map(String::as_str)
502            .collect::<std::collections::BTreeSet<_>>();
503        assert_eq!(
504            keys,
505            std::collections::BTreeSet::from([
506                "heuristic_witnesses",
507                "model_cache_hits",
508                "model_queries",
509                "paths",
510                "sat_cache_hits",
511                "sat_queries",
512                "smt_build_time_ms",
513                "smt_input_bytes",
514                "smt_max_query_bytes",
515                "smt_max_query_time_ms",
516                "smt_queries",
517                "solver_queries",
518                "solver_time_ms",
519            ])
520        );
521    }
522
523    #[test]
524    fn sidecar_retains_samples_and_revision() {
525        let first = parse(&stable("pass", 1)).unwrap();
526        let second = parse(&stable("fail_counterexample", 1)).unwrap();
527        let samples = [first, second]
528            .into_iter()
529            .map(|run| Sample { wall_time_seconds: 1.0, exit_code: 0, run })
530            .collect();
531        let sidecar = Sidecar::new(
532            Fixture::Farcaster,
533            "farcasterxyz/contracts",
534            "test-revision",
535            "forge build",
536            "forge test --symbolic --json",
537            samples,
538        );
539        assert_eq!(sidecar.fixture.revision, "test-revision");
540        assert_eq!(sidecar.samples.len(), 2);
541    }
542
543    #[test]
544    fn overlay_is_explicitly_removed_and_existing_file_is_preserved() {
545        let dir = tempfile::tempdir().unwrap();
546        fs::create_dir(dir.path().join("test")).unwrap();
547        let path = dir.path().join(FARCASTER_PATH);
548        let overlay = Overlay::install(dir.path(), Fixture::Farcaster).unwrap();
549        assert_eq!(fs::read_to_string(&path).unwrap(), FARCASTER_TEST);
550        overlay.finish().unwrap();
551        assert!(!path.exists());
552        fs::write(&path, "unexpected").unwrap();
553        assert!(Overlay::install(dir.path(), Fixture::Farcaster).is_err());
554        assert_eq!(fs::read_to_string(path).unwrap(), "unexpected");
555    }
556}