1use crate::results::SymbolicBenchmarkSummary;
4use eyre::{Result, WrapErr};
5use serde::Serialize;
6use serde_json::Value;
7use std::{
8 fs,
9 path::{Path, PathBuf},
10};
11
12const FARCASTER_PATH: &str = "test/FarcasterNativeSymbolic.t.sol";
13const FARCASTER_TEST: &str = include_str!("../fixtures/symbolic/FarcasterNativeSymbolic.t.sol");
14const PREFIX: &str =
15 "FOUNDRY_DYNAMIC_TEST_LINKING=false FOUNDRY_LINT_LINT_ON_BUILD=false FOUNDRY_ISOLATE=false";
16const SOLADY_MATCH: &str = "check_(SaturatingAddEquivalence|SaturatingMulEquivalence|HasDuplicateHashmapCapacityTrickEquivalence|IsPermit2AndValueIsNotInfinityTrickEquivalence|IsNotUint256MaxTrickEquivalence|DelayRestriction|OperationStateDifferentialTrick|CarryBoundsTrick|SafeCastInt256ToIntTrickEquivalence|P256Normalized|AuxPackEquivalence|EcrecoverTrickEquivalence|EcrecoverLoopTrick)";
17
18#[derive(Clone, Copy, Debug, Serialize)]
19#[serde(rename_all = "snake_case")]
20pub enum Fixture {
21 Solady,
22 Angstrom,
23 Farcaster,
24 FoundryXor,
25 Generic,
26}
27
28impl Fixture {
29 pub const fn identify(org: &str, repo: &str) -> Self {
30 if org.eq_ignore_ascii_case("vectorized") && repo.eq_ignore_ascii_case("solady") {
31 Self::Solady
32 } else if org.eq_ignore_ascii_case("sorellalabs") && repo.eq_ignore_ascii_case("angstrom") {
33 Self::Angstrom
34 } else if org.eq_ignore_ascii_case("farcasterxyz") && repo.eq_ignore_ascii_case("contracts")
35 {
36 Self::Farcaster
37 } else if org.eq_ignore_ascii_case("foundry-rs") && repo.eq_ignore_ascii_case("foundry") {
38 Self::FoundryXor
39 } else {
40 Self::Generic
41 }
42 }
43
44 pub fn build_command(self) -> String {
45 format!(
46 "{PREFIX} {}",
47 match self {
48 Self::Angstrom => "forge build --root contracts",
49 Self::FoundryXor => "forge build --root benches/fixtures/symbolic/xor",
50 _ => "forge build",
51 }
52 )
53 }
54
55 pub fn test_command(self) -> String {
56 match self {
57 Self::Solady => {
58 format!("{PREFIX} forge test --symbolic --json --match-test '{SOLADY_MATCH}'")
59 }
60 Self::Angstrom => format!(
61 "{PREFIX} forge test --root contracts --symbolic --json --symbolic-timeout 5 --match-path test/libraries/X128MathLib.t.sol --match-test check_matchesSolady_fullMulX128"
62 ),
63 Self::Farcaster => format!(
64 "{PREFIX} forge test --symbolic --json --symbolic-timeout 5 --match-path '{FARCASTER_PATH}' --match-test 'check_'"
65 ),
66 Self::FoundryXor => format!(
67 "{PREFIX} forge test --root benches/fixtures/symbolic/xor --symbolic --json"
68 ),
69 Self::Generic => format!("{PREFIX} forge test --symbolic --json"),
70 }
71 }
72}
73
74pub struct Overlay {
76 path: Option<PathBuf>,
77}
78
79impl Overlay {
80 pub fn install(root: &Path, fixture: Fixture) -> Result<Self> {
81 if !matches!(fixture, Fixture::Farcaster) {
82 return Ok(Self { path: None });
83 }
84 let path = root.join(FARCASTER_PATH);
85 if path.exists() {
86 eyre::bail!("refusing to overwrite existing {}", path.display());
87 }
88 if let Err(write_err) = fs::write(&path, FARCASTER_TEST) {
89 if let Err(cleanup_err) = fs::remove_file(&path)
90 && cleanup_err.kind() != std::io::ErrorKind::NotFound
91 {
92 eyre::bail!(
93 "failed to install Farcaster symbolic fixture: {write_err}; failed to remove partially written {}: {cleanup_err}",
94 path.display()
95 );
96 }
97 return Err(write_err).wrap_err("failed to install Farcaster symbolic fixture");
98 }
99 Ok(Self { path: Some(path) })
100 }
101
102 pub fn finish(mut self) -> Result<()> {
103 self.cleanup()
104 }
105
106 fn cleanup(&mut self) -> Result<()> {
107 if let Some(path) = &self.path {
108 fs::remove_file(path).wrap_err_with(|| {
109 format!("failed to remove symbolic fixture {}", path.display())
110 })?;
111 self.path = None;
112 }
113 Ok(())
114 }
115}
116
117impl Drop for Overlay {
118 fn drop(&mut self) {
119 let _ = self.cleanup();
120 }
121}
122
123#[derive(Clone, Debug, Default, Serialize)]
124pub struct Metrics {
125 pub paths: u64,
126 pub solver_queries: u64,
127 pub smt_queries: u64,
128 pub sat_queries: u64,
129 pub model_queries: u64,
130 pub sat_cache_hits: u64,
131 pub model_cache_hits: u64,
132 pub heuristic_witnesses: u64,
133 pub solver_time_ms: u64,
134 pub smt_input_bytes: Option<u64>,
135 pub smt_max_query_bytes: Option<u64>,
136 pub smt_build_time_ms: Option<u64>,
137 pub smt_max_query_time_ms: Option<u64>,
138 #[serde(skip)]
139 smt_input_bytes_compat: u64,
140 #[serde(skip)]
141 smt_max_query_bytes_compat: u64,
142 #[serde(skip)]
143 smt_build_time_ms_compat: u64,
144 #[serde(skip)]
145 smt_max_query_time_ms_compat: u64,
146}
147
148#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
149#[serde(rename_all = "snake_case")]
150pub enum OutcomeStatus {
151 Passed,
152 Failed,
153 Incomplete,
154}
155
156#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
157pub struct TestOutcome {
158 pub suite: String,
159 pub signature: String,
160 pub status: OutcomeStatus,
161}
162
163impl TestOutcome {
164 fn identity(&self) -> String {
165 format!("{}::{}", self.suite, self.signature)
166 }
167}
168
169#[derive(Clone, Debug, Serialize)]
170pub struct ParsedRun {
171 pub outcomes: Vec<TestOutcome>,
172 pub passed: usize,
173 pub failed: usize,
174 pub incomplete: usize,
175 pub metrics: Metrics,
176}
177
178#[derive(Clone, Debug, Serialize)]
179pub struct Sample {
180 pub wall_time_seconds: f64,
181 pub exit_code: i32,
182 #[serde(flatten)]
183 pub run: ParsedRun,
184}
185
186#[derive(Clone, Debug, Serialize)]
187pub struct Sidecar {
188 pub schema: &'static str,
189 pub schema_version: u32,
190 pub fixture: FixtureMetadata,
191 pub samples: Vec<Sample>,
192}
193
194#[derive(Clone, Debug, Serialize)]
195pub struct FixtureMetadata {
196 pub name: Fixture,
197 pub repository: String,
198 pub revision: String,
199 pub build_command: String,
200 pub test_command: String,
201}
202
203impl Sidecar {
204 pub fn new(
205 fixture: Fixture,
206 repository: &str,
207 revision: &str,
208 build_command: &str,
209 test_command: &str,
210 samples: Vec<Sample>,
211 ) -> Self {
212 Self {
213 schema: "foundry:symbolic-benchmark@v1",
214 schema_version: 1,
215 fixture: FixtureMetadata {
216 name: fixture,
217 repository: repository.to_string(),
218 revision: revision.to_string(),
219 build_command: build_command.to_string(),
220 test_command: test_command.to_string(),
221 },
222 samples,
223 }
224 }
225}
226
227pub fn parse(stdout: &[u8]) -> Result<ParsedRun> {
228 let json: Value =
229 serde_json::from_slice(stdout).wrap_err("invalid forge test --json output")?;
230 let suites = json.as_object().ok_or_else(|| eyre::eyre!("expected JSON object"))?;
231 let stable = suites
232 .values()
233 .filter_map(|s| s.get("test_results").and_then(Value::as_object))
234 .flat_map(|r| r.values())
235 .any(|r| r.get("symbolic").is_some());
236 let mut run = ParsedRun {
237 outcomes: Vec::new(),
238 passed: 0,
239 failed: 0,
240 incomplete: 0,
241 metrics: Metrics::default(),
242 };
243 for (suite_name, suite) in suites {
244 let Some(results) = suite.get("test_results").and_then(Value::as_object) else { continue };
245 for (identity, result) in results {
246 let (stats, status) = if stable {
247 let Some(symbolic) = result.get("symbolic") else { continue };
248 if symbolic.get("schema_version").and_then(Value::as_u64) != Some(1) {
249 eyre::bail!("unknown or missing symbolic schema_version");
250 }
251 let status = symbolic
252 .get("status")
253 .and_then(Value::as_str)
254 .ok_or_else(|| eyre::eyre!("missing symbolic.status"))?;
255 let status = match status {
256 "pass" => OutcomeStatus::Passed,
257 "fail_counterexample" => OutcomeStatus::Failed,
258 "incomplete" => OutcomeStatus::Incomplete,
259 _ => {
260 eyre::bail!("unknown symbolic.status {status}");
261 }
262 };
263 let stats = symbolic
264 .pointer("/solver/stats")
265 .filter(|stats| stats.is_object())
266 .ok_or_else(|| eyre::eyre!("missing or invalid symbolic.solver.stats"))?;
267 (stats, status)
268 } else {
269 let Some(stats) = result.pointer("/kind/Symbolic") else { continue };
270 let status = result.get("status").and_then(Value::as_str).unwrap_or_default();
271 let reason = result.get("reason").and_then(Value::as_str).unwrap_or_default();
272 let status = if status == "Success" {
273 OutcomeStatus::Passed
274 } else if reason.contains("incomplete symbolic execution") {
275 OutcomeStatus::Incomplete
276 } else {
277 OutcomeStatus::Failed
278 };
279 (stats, status)
280 };
281 run.outcomes.push(TestOutcome {
282 suite: suite_name.clone(),
283 signature: identity.clone(),
284 status,
285 });
286 add_metrics(&mut run.metrics, stats, stable, run.outcomes.len() == 1)?;
287 }
288 }
289 if run.outcomes.is_empty() {
290 eyre::bail!("forge symbolic benchmark produced no symbolic test results");
291 }
292 run.outcomes.sort_by_key(TestOutcome::identity);
293 run.passed =
294 run.outcomes.iter().filter(|outcome| outcome.status == OutcomeStatus::Passed).count();
295 run.failed =
296 run.outcomes.iter().filter(|outcome| outcome.status == OutcomeStatus::Failed).count();
297 run.incomplete =
298 run.outcomes.iter().filter(|outcome| outcome.status == OutcomeStatus::Incomplete).count();
299 Ok(run)
300}
301
302fn add_metrics(out: &mut Metrics, stats: &Value, stable: bool, first: bool) -> Result<()> {
303 macro_rules! add {
304 ($field:ident) => {
305 out.$field += required(stats, stringify!($field), stable)?;
306 };
307 }
308 add!(paths);
309 add!(solver_queries);
310 add!(smt_queries);
311 add!(sat_queries);
312 add!(model_queries);
313 add!(sat_cache_hits);
314 add!(model_cache_hits);
315 add!(heuristic_witnesses);
316 add!(solver_time_ms);
317 optional_add(
318 &mut out.smt_input_bytes,
319 &mut out.smt_input_bytes_compat,
320 stats,
321 "smt_input_bytes",
322 first,
323 )?;
324 optional_add(
325 &mut out.smt_build_time_ms,
326 &mut out.smt_build_time_ms_compat,
327 stats,
328 "smt_build_time_ms",
329 first,
330 )?;
331 optional_max(
332 &mut out.smt_max_query_bytes,
333 &mut out.smt_max_query_bytes_compat,
334 stats,
335 "smt_max_query_bytes",
336 first,
337 )?;
338 optional_max(
339 &mut out.smt_max_query_time_ms,
340 &mut out.smt_max_query_time_ms_compat,
341 stats,
342 "smt_max_query_time_ms",
343 first,
344 )?;
345 Ok(())
346}
347
348fn required(v: &Value, key: &str, strict: bool) -> Result<u64> {
349 match v.get(key) {
350 Some(v) => v.as_u64().ok_or_else(|| eyre::eyre!("invalid metric {key}")),
351 None if strict => Err(eyre::eyre!("missing metric {key}")),
352 None => Ok(0),
353 }
354}
355
356fn optional_add(
357 out: &mut Option<u64>,
358 compatibility: &mut u64,
359 v: &Value,
360 key: &str,
361 first: bool,
362) -> Result<()> {
363 let value = v
364 .get(key)
365 .map(|n| n.as_u64().ok_or_else(|| eyre::eyre!("invalid metric {key}")))
366 .transpose()?;
367 *compatibility += value.unwrap_or_default();
368 *out = if first { value } else { (*out).zip(value).map(|(old, value)| old + value) };
369 Ok(())
370}
371
372fn optional_max(
373 out: &mut Option<u64>,
374 compatibility: &mut u64,
375 v: &Value,
376 key: &str,
377 first: bool,
378) -> Result<()> {
379 let value = v
380 .get(key)
381 .map(|n| n.as_u64().ok_or_else(|| eyre::eyre!("invalid metric {key}")))
382 .transpose()?;
383 *compatibility = (*compatibility).max(value.unwrap_or_default());
384 *out = if first { value } else { (*out).zip(value).map(|(old, value)| old.max(value)) };
385 Ok(())
386}
387
388pub const fn compatibility(run: &ParsedRun) -> SymbolicBenchmarkSummary {
389 let m = &run.metrics;
390 SymbolicBenchmarkSummary {
391 tests: run.outcomes.len(),
392 passed: run.passed,
393 failed: run.failed,
394 incomplete: run.incomplete,
395 paths: m.paths,
396 solver_queries: m.solver_queries,
397 smt_queries: m.smt_queries,
398 sat_queries: m.sat_queries,
399 model_queries: m.model_queries,
400 sat_cache_hits: m.sat_cache_hits,
401 model_cache_hits: m.model_cache_hits,
402 heuristic_witnesses: m.heuristic_witnesses,
403 solver_time_ms: m.solver_time_ms,
404 smt_input_bytes: m.smt_input_bytes_compat,
405 smt_max_query_bytes: m.smt_max_query_bytes_compat,
406 smt_build_time_ms: m.smt_build_time_ms_compat,
407 smt_max_query_time_ms: m.smt_max_query_time_ms_compat,
408 }
409}
410
411#[cfg(test)]
412mod tests {
413 use super::*;
414
415 fn stable(status: &str, schema: u64) -> Vec<u8> {
416 serde_json::to_vec(&serde_json::json!({"test/FarcasterNativeSymbolic.t.sol:FarcasterNativeSymbolicTest":{"test_results":{
417 "check_migrateOnlyMigrator(uint24,address,address,address,uint40)":{"symbolic":{"schema_version":schema,"status":status,"bounds":{},"solver":{"name":"z3","command":null,"portfolio":[],"stats":{"paths":1,"solver_queries":2,"smt_queries":3,"sat_queries":4,"model_queries":5,"sat_cache_hits":6,"model_cache_hits":7,"heuristic_witnesses":8,"solver_time_ms":9,"smt_input_bytes":10,"smt_max_query_bytes":11,"smt_build_time_ms":12,"smt_max_query_time_ms":13}}}},
418 "check_setMigratorOnlyOwner(uint24,address,address,address,address)":{"symbolic":{"schema_version":schema,"status":"incomplete","bounds":{},"solver":{"name":"z3","command":null,"portfolio":[],"stats":{"paths":10,"solver_queries":20,"smt_queries":30,"sat_queries":40,"model_queries":50,"sat_cache_hits":60,"model_cache_hits":70,"heuristic_witnesses":80,"solver_time_ms":90}}}}
419 }}})).unwrap()
420 }
421
422 #[test]
423 fn stable_parser_validates_statuses_and_missing_optional_metrics() {
424 let run = parse(&stable("pass", 1)).unwrap();
425 assert_eq!((run.passed, run.failed, run.incomplete), (1, 0, 1));
426 assert_eq!(run.metrics.paths, 11);
427 assert_eq!(run.metrics.solver_time_ms, 99);
428 assert_eq!(run.metrics.smt_max_query_bytes, None);
429 assert_eq!(run.metrics.smt_input_bytes, None);
430 assert_eq!(run.metrics.smt_build_time_ms, None);
431 assert_eq!(run.metrics.smt_max_query_time_ms, None);
432 assert_eq!(
433 run.outcomes[0].suite,
434 "test/FarcasterNativeSymbolic.t.sol:FarcasterNativeSymbolicTest"
435 );
436 let compatibility = compatibility(&run);
437 assert_eq!(compatibility.tests, 2);
438 assert_eq!(compatibility.smt_input_bytes, 10);
439 }
440
441 #[test]
442 fn rejects_unknown_stable_schema() {
443 assert!(parse(&stable("pass", 2)).is_err());
444 }
445
446 #[test]
447 fn aggregates_optional_metrics_only_when_every_test_reports_them() {
448 let first = serde_json::json!({
449 "paths": 1, "solver_queries": 1, "smt_queries": 1, "sat_queries": 1,
450 "model_queries": 1, "sat_cache_hits": 1, "model_cache_hits": 1,
451 "heuristic_witnesses": 1, "solver_time_ms": 1, "smt_input_bytes": 10,
452 "smt_max_query_bytes": 8, "smt_build_time_ms": 2, "smt_max_query_time_ms": 4
453 });
454 let second = serde_json::json!({
455 "paths": 1, "solver_queries": 1, "smt_queries": 1, "sat_queries": 1,
456 "model_queries": 1, "sat_cache_hits": 1, "model_cache_hits": 1,
457 "heuristic_witnesses": 1, "solver_time_ms": 1, "smt_input_bytes": 20,
458 "smt_max_query_bytes": 7, "smt_build_time_ms": 3, "smt_max_query_time_ms": 9
459 });
460 let mut metrics = Metrics::default();
461 add_metrics(&mut metrics, &first, true, true).unwrap();
462 add_metrics(&mut metrics, &second, true, false).unwrap();
463 assert_eq!(metrics.smt_input_bytes, Some(30));
464 assert_eq!(metrics.smt_build_time_ms, Some(5));
465 assert_eq!(metrics.smt_max_query_bytes, Some(8));
466 assert_eq!(metrics.smt_max_query_time_ms, Some(9));
467 }
468
469 #[test]
470 fn parses_whole_output_legacy_results() {
471 let stats = serde_json::json!({"paths": 1, "solver_queries": 2});
472 let stdout = serde_json::to_vec(&serde_json::json!({"test/FarcasterNativeSymbolic.t.sol:FarcasterNativeSymbolicTest":{"test_results":{
473 "check_migrateOnlyMigrator(uint24,address,address,address,uint40)":{
474 "kind":{"Symbolic":stats},"status":"Success"
475 },
476 "check_setMigratorOnlyOwner(uint24,address,address,address,address)":{
477 "kind":{"Symbolic":stats},"status":"Failure",
478 "reason":"incomplete symbolic execution (Timeout)"
479 }
480 }}}))
481 .unwrap();
482 let run = parse(&stdout).unwrap();
483 assert_eq!((run.passed, run.failed, run.incomplete), (1, 0, 1));
484 assert_eq!(run.metrics.paths, 2);
485 }
486
487 #[test]
488 fn unknown_repository_uses_generic_symbolic_command() {
489 let fixture = Fixture::identify("example", "contracts");
490 assert!(matches!(fixture, Fixture::Generic));
491 assert_eq!(fixture.test_command(), format!("{PREFIX} forge test --symbolic --json"));
492 }
493
494 #[test]
495 fn metrics_serialize_the_complete_v1_counter_set() {
496 let value = serde_json::to_value(Metrics::default()).unwrap();
497 let keys = value
498 .as_object()
499 .unwrap()
500 .keys()
501 .map(String::as_str)
502 .collect::<std::collections::BTreeSet<_>>();
503 assert_eq!(
504 keys,
505 std::collections::BTreeSet::from([
506 "heuristic_witnesses",
507 "model_cache_hits",
508 "model_queries",
509 "paths",
510 "sat_cache_hits",
511 "sat_queries",
512 "smt_build_time_ms",
513 "smt_input_bytes",
514 "smt_max_query_bytes",
515 "smt_max_query_time_ms",
516 "smt_queries",
517 "solver_queries",
518 "solver_time_ms",
519 ])
520 );
521 }
522
523 #[test]
524 fn sidecar_retains_samples_and_revision() {
525 let first = parse(&stable("pass", 1)).unwrap();
526 let second = parse(&stable("fail_counterexample", 1)).unwrap();
527 let samples = [first, second]
528 .into_iter()
529 .map(|run| Sample { wall_time_seconds: 1.0, exit_code: 0, run })
530 .collect();
531 let sidecar = Sidecar::new(
532 Fixture::Farcaster,
533 "farcasterxyz/contracts",
534 "test-revision",
535 "forge build",
536 "forge test --symbolic --json",
537 samples,
538 );
539 assert_eq!(sidecar.fixture.revision, "test-revision");
540 assert_eq!(sidecar.samples.len(), 2);
541 }
542
543 #[test]
544 fn overlay_is_explicitly_removed_and_existing_file_is_preserved() {
545 let dir = tempfile::tempdir().unwrap();
546 fs::create_dir(dir.path().join("test")).unwrap();
547 let path = dir.path().join(FARCASTER_PATH);
548 let overlay = Overlay::install(dir.path(), Fixture::Farcaster).unwrap();
549 assert_eq!(fs::read_to_string(&path).unwrap(), FARCASTER_TEST);
550 overlay.finish().unwrap();
551 assert!(!path.exists());
552 fs::write(&path, "unexpected").unwrap();
553 assert!(Overlay::install(dir.path(), Fixture::Farcaster).is_err());
554 assert_eq!(fs::read_to_string(path).unwrap(), "unexpected");
555 }
556}