1use crate::{
4 fuzz::{BaseCounterExample, FuzzedCases},
5 gas_report::GasReport,
6};
7use alloy_primitives::{
8 map::{AddressHashMap, HashMap},
9 Address, Log,
10};
11use eyre::Report;
12use foundry_common::{evm::Breakpoints, get_contract_name, get_file_name, shell};
13use foundry_evm::{
14 coverage::HitMaps,
15 decode::SkipReason,
16 executors::{invariant::InvariantMetrics, RawCallResult},
17 fuzz::{CounterExample, FuzzCase, FuzzFixtures, FuzzTestResult},
18 traces::{CallTraceArena, CallTraceDecoder, TraceKind, Traces},
19};
20use serde::{Deserialize, Serialize};
21use std::{
22 collections::{BTreeMap, HashMap as Map},
23 fmt::{self, Write},
24 time::Duration,
25};
26use yansi::Paint;
27
28#[derive(Clone, Debug)]
30pub struct TestOutcome {
31 pub results: BTreeMap<String, SuiteResult>,
35 pub allow_failure: bool,
37 pub last_run_decoder: Option<CallTraceDecoder>,
43 pub gas_report: Option<GasReport>,
45}
46
47impl TestOutcome {
48 pub fn new(results: BTreeMap<String, SuiteResult>, allow_failure: bool) -> Self {
50 Self { results, allow_failure, last_run_decoder: None, gas_report: None }
51 }
52
53 pub fn empty(allow_failure: bool) -> Self {
55 Self::new(BTreeMap::new(), allow_failure)
56 }
57
58 pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
60 self.tests().filter(|(_, t)| t.status.is_success())
61 }
62
63 pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
65 self.tests().filter(|(_, t)| t.status.is_skipped())
66 }
67
68 pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
70 self.tests().filter(|(_, t)| t.status.is_failure())
71 }
72
73 pub fn tests(&self) -> impl Iterator<Item = (&String, &TestResult)> {
75 self.results.values().flat_map(|suite| suite.tests())
76 }
77
78 pub fn into_tests_cloned(&self) -> impl Iterator<Item = SuiteTestResult> + '_ {
81 self.results
82 .iter()
83 .flat_map(|(file, suite)| {
84 suite
85 .test_results
86 .iter()
87 .map(move |(sig, result)| (file.clone(), sig.clone(), result.clone()))
88 })
89 .map(|(artifact_id, signature, result)| SuiteTestResult {
90 artifact_id,
91 signature,
92 result,
93 })
94 }
95
96 pub fn into_tests(self) -> impl Iterator<Item = SuiteTestResult> {
98 self.results
99 .into_iter()
100 .flat_map(|(file, suite)| {
101 suite.test_results.into_iter().map(move |t| (file.clone(), t))
102 })
103 .map(|(artifact_id, (signature, result))| SuiteTestResult {
104 artifact_id,
105 signature,
106 result,
107 })
108 }
109
110 pub fn passed(&self) -> usize {
112 self.successes().count()
113 }
114
115 pub fn skipped(&self) -> usize {
117 self.skips().count()
118 }
119
120 pub fn failed(&self) -> usize {
122 self.failures().count()
123 }
124
125 pub fn total_time(&self) -> Duration {
129 self.results.values().map(|suite| suite.duration).sum()
130 }
131
132 pub fn summary(&self, wall_clock_time: Duration) -> String {
134 let num_test_suites = self.results.len();
135 let suites = if num_test_suites == 1 { "suite" } else { "suites" };
136 let total_passed = self.passed();
137 let total_failed = self.failed();
138 let total_skipped = self.skipped();
139 let total_tests = total_passed + total_failed + total_skipped;
140 format!(
141 "\nRan {} test {} in {:.2?} ({:.2?} CPU time): {} tests passed, {} failed, {} skipped ({} total tests)",
142 num_test_suites,
143 suites,
144 wall_clock_time,
145 self.total_time(),
146 total_passed.green(),
147 total_failed.red(),
148 total_skipped.yellow(),
149 total_tests
150 )
151 }
152
153 pub fn ensure_ok(&self, silent: bool) -> eyre::Result<()> {
155 let outcome = self;
156 let failures = outcome.failures().count();
157 if outcome.allow_failure || failures == 0 {
158 return Ok(());
159 }
160
161 if shell::is_quiet() || silent {
162 std::process::exit(1);
164 }
165
166 sh_println!("\nFailing tests:")?;
167 for (suite_name, suite) in &outcome.results {
168 let failed = suite.failed();
169 if failed == 0 {
170 continue;
171 }
172
173 let term = if failed > 1 { "tests" } else { "test" };
174 sh_println!("Encountered {failed} failing {term} in {suite_name}")?;
175 for (name, result) in suite.failures() {
176 sh_println!("{}", result.short_result(name))?;
177 }
178 sh_println!()?;
179 }
180 let successes = outcome.passed();
181 sh_println!(
182 "Encountered a total of {} failing tests, {} tests succeeded",
183 failures.to_string().red(),
184 successes.to_string().green()
185 )?;
186
187 std::process::exit(1);
189 }
190
191 pub fn remove_first(&mut self) -> Option<(String, String, TestResult)> {
193 self.results.iter_mut().find_map(|(suite_name, suite)| {
194 if let Some(test_name) = suite.test_results.keys().next().cloned() {
195 let result = suite.test_results.remove(&test_name).unwrap();
196 Some((suite_name.clone(), test_name, result))
197 } else {
198 None
199 }
200 })
201 }
202}
203
204#[derive(Clone, Debug, Serialize)]
206pub struct SuiteResult {
207 #[serde(with = "humantime_serde")]
209 pub duration: Duration,
210 pub test_results: BTreeMap<String, TestResult>,
212 pub warnings: Vec<String>,
214}
215
216impl SuiteResult {
217 pub fn new(
218 duration: Duration,
219 test_results: BTreeMap<String, TestResult>,
220 mut warnings: Vec<String>,
221 ) -> Self {
222 let mut deprecated_cheatcodes = HashMap::new();
224 for test_result in test_results.values() {
225 deprecated_cheatcodes.extend(test_result.deprecated_cheatcodes.clone());
226 }
227 if !deprecated_cheatcodes.is_empty() {
228 let mut warning =
229 "the following cheatcode(s) are deprecated and will be removed in future versions:"
230 .to_string();
231 for (cheatcode, reason) in deprecated_cheatcodes {
232 write!(warning, "\n {cheatcode}").unwrap();
233 if let Some(reason) = reason {
234 write!(warning, ": {reason}").unwrap();
235 }
236 }
237 warnings.push(warning);
238 }
239
240 Self { duration, test_results, warnings }
241 }
242
243 pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
245 self.tests().filter(|(_, t)| t.status.is_success())
246 }
247
248 pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
250 self.tests().filter(|(_, t)| t.status.is_skipped())
251 }
252
253 pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
255 self.tests().filter(|(_, t)| t.status.is_failure())
256 }
257
258 pub fn passed(&self) -> usize {
260 self.successes().count()
261 }
262
263 pub fn skipped(&self) -> usize {
265 self.skips().count()
266 }
267
268 pub fn failed(&self) -> usize {
270 self.failures().count()
271 }
272
273 pub fn tests(&self) -> impl Iterator<Item = (&String, &TestResult)> {
275 self.test_results.iter()
276 }
277
278 pub fn is_empty(&self) -> bool {
280 self.test_results.is_empty()
281 }
282
283 pub fn len(&self) -> usize {
285 self.test_results.len()
286 }
287
288 pub fn total_time(&self) -> Duration {
292 self.test_results.values().map(|result| result.duration).sum()
293 }
294
295 pub fn summary(&self) -> String {
297 let failed = self.failed();
298 let result = if failed == 0 { "ok".green() } else { "FAILED".red() };
299 format!(
300 "Suite result: {}. {} passed; {} failed; {} skipped; finished in {:.2?} ({:.2?} CPU time)",
301 result,
302 self.passed().green(),
303 failed.red(),
304 self.skipped().yellow(),
305 self.duration,
306 self.total_time(),
307 )
308 }
309}
310
311#[derive(Clone, Debug)]
315pub struct SuiteTestResult {
316 pub artifact_id: String,
319 pub signature: String,
321 pub result: TestResult,
323}
324
325impl SuiteTestResult {
326 pub fn gas_used(&self) -> u64 {
328 self.result.kind.report().gas()
329 }
330
331 pub fn contract_name(&self) -> &str {
333 get_contract_name(&self.artifact_id)
334 }
335
336 pub fn file_name(&self) -> &str {
338 get_file_name(&self.artifact_id)
339 }
340}
341
342#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
344pub enum TestStatus {
345 Success,
346 #[default]
347 Failure,
348 Skipped,
349}
350
351impl TestStatus {
352 #[inline]
354 pub fn is_success(self) -> bool {
355 matches!(self, Self::Success)
356 }
357
358 #[inline]
360 pub fn is_failure(self) -> bool {
361 matches!(self, Self::Failure)
362 }
363
364 #[inline]
366 pub fn is_skipped(self) -> bool {
367 matches!(self, Self::Skipped)
368 }
369}
370
371#[derive(Clone, Debug, Default, Serialize, Deserialize)]
373pub struct TestResult {
374 pub status: TestStatus,
379
380 pub reason: Option<String>,
383
384 pub counterexample: Option<CounterExample>,
386
387 pub logs: Vec<Log>,
390
391 pub decoded_logs: Vec<String>,
394
395 pub kind: TestKind,
397
398 pub traces: Traces,
400
401 #[serde(skip)]
403 pub gas_report_traces: Vec<Vec<CallTraceArena>>,
404
405 #[serde(skip)]
407 pub coverage: Option<HitMaps>,
408
409 pub labeled_addresses: AddressHashMap<String>,
411
412 pub duration: Duration,
413
414 pub breakpoints: Breakpoints,
416
417 pub gas_snapshots: BTreeMap<String, BTreeMap<String, String>>,
419
420 #[serde(skip)]
422 pub deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>,
423}
424
425impl fmt::Display for TestResult {
426 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
427 match self.status {
428 TestStatus::Success => "[PASS]".green().fmt(f),
429 TestStatus::Skipped => {
430 let mut s = String::from("[SKIP");
431 if let Some(reason) = &self.reason {
432 write!(s, ": {reason}").unwrap();
433 }
434 s.push(']');
435 s.yellow().fmt(f)
436 }
437 TestStatus::Failure => {
438 let mut s = String::from("[FAIL");
439 if self.reason.is_some() || self.counterexample.is_some() {
440 if let Some(reason) = &self.reason {
441 write!(s, ": {reason}").unwrap();
442 }
443
444 if let Some(counterexample) = &self.counterexample {
445 match counterexample {
446 CounterExample::Single(ex) => {
447 write!(s, "; counterexample: {ex}]").unwrap();
448 }
449 CounterExample::Sequence(original, sequence) => {
450 s.push_str(
451 format!(
452 "]\n\t[Sequence] (original: {original}, shrunk: {})\n",
453 sequence.len()
454 )
455 .as_str(),
456 );
457 for ex in sequence {
458 writeln!(s, "{ex}").unwrap();
459 }
460 }
461 }
462 } else {
463 s.push(']');
464 }
465 } else {
466 s.push(']');
467 }
468 s.red().fmt(f)
469 }
470 }
471 }
472}
473
474impl TestResult {
475 pub fn new(setup: &TestSetup) -> Self {
477 Self {
478 labeled_addresses: setup.labels.clone(),
479 logs: setup.logs.clone(),
480 traces: setup.traces.clone(),
481 coverage: setup.coverage.clone(),
482 ..Default::default()
483 }
484 }
485
486 pub fn fail(reason: String) -> Self {
488 Self { status: TestStatus::Failure, reason: Some(reason), ..Default::default() }
489 }
490
491 pub fn setup_result(setup: TestSetup) -> Self {
493 Self {
494 status: if setup.skipped { TestStatus::Skipped } else { TestStatus::Failure },
495 reason: setup.reason,
496 logs: setup.logs,
497 traces: setup.traces,
498 coverage: setup.coverage,
499 labeled_addresses: setup.labels,
500 ..Default::default()
501 }
502 }
503
504 pub fn single_skip(&mut self, reason: SkipReason) {
506 self.status = TestStatus::Skipped;
507 self.reason = reason.0;
508 }
509
510 pub fn single_fail(&mut self, reason: Option<String>) {
512 self.status = TestStatus::Failure;
513 self.reason = reason;
514 }
515
516 pub fn single_result(
519 &mut self,
520 success: bool,
521 reason: Option<String>,
522 raw_call_result: RawCallResult,
523 ) {
524 self.kind =
525 TestKind::Unit { gas: raw_call_result.gas_used.wrapping_sub(raw_call_result.stipend) };
526
527 self.logs.extend(raw_call_result.logs);
529 self.labeled_addresses.extend(raw_call_result.labels);
530 self.traces.extend(raw_call_result.traces.map(|traces| (TraceKind::Execution, traces)));
531 self.merge_coverages(raw_call_result.coverage);
532
533 self.status = match success {
534 true => TestStatus::Success,
535 false => TestStatus::Failure,
536 };
537 self.reason = reason;
538 self.duration = Duration::default();
539 self.gas_report_traces = Vec::new();
540
541 if let Some(cheatcodes) = raw_call_result.cheatcodes {
542 self.breakpoints = cheatcodes.breakpoints;
543 self.gas_snapshots = cheatcodes.gas_snapshots;
544 self.deprecated_cheatcodes = cheatcodes.deprecated;
545 }
546 }
547
548 pub fn fuzz_result(&mut self, result: FuzzTestResult) {
551 self.kind = TestKind::Fuzz {
552 median_gas: result.median_gas(false),
553 mean_gas: result.mean_gas(false),
554 first_case: result.first_case,
555 runs: result.gas_by_case.len(),
556 };
557
558 self.logs.extend(result.logs);
560 self.labeled_addresses.extend(result.labeled_addresses);
561 self.traces.extend(result.traces.map(|traces| (TraceKind::Execution, traces)));
562 self.merge_coverages(result.coverage);
563
564 self.status = if result.skipped {
565 TestStatus::Skipped
566 } else if result.success {
567 TestStatus::Success
568 } else {
569 TestStatus::Failure
570 };
571 self.reason = result.reason;
572 self.counterexample = result.counterexample;
573 self.duration = Duration::default();
574 self.gas_report_traces = result.gas_report_traces.into_iter().map(|t| vec![t]).collect();
575 self.breakpoints = result.breakpoints.unwrap_or_default();
576 self.deprecated_cheatcodes = result.deprecated_cheatcodes;
577 }
578
579 pub fn invariant_skip(&mut self, reason: SkipReason) {
581 self.kind =
582 TestKind::Invariant { runs: 1, calls: 1, reverts: 1, metrics: HashMap::default() };
583 self.status = TestStatus::Skipped;
584 self.reason = reason.0;
585 }
586
587 pub fn invariant_replay_fail(
589 &mut self,
590 replayed_entirely: bool,
591 invariant_name: &String,
592 call_sequence: Vec<BaseCounterExample>,
593 ) {
594 self.kind =
595 TestKind::Invariant { runs: 1, calls: 1, reverts: 1, metrics: HashMap::default() };
596 self.status = TestStatus::Failure;
597 self.reason = if replayed_entirely {
598 Some(format!("{invariant_name} replay failure"))
599 } else {
600 Some(format!("{invariant_name} persisted failure revert"))
601 };
602 self.counterexample = Some(CounterExample::Sequence(call_sequence.len(), call_sequence));
603 }
604
605 pub fn invariant_setup_fail(&mut self, e: Report) {
607 self.kind =
608 TestKind::Invariant { runs: 0, calls: 0, reverts: 0, metrics: HashMap::default() };
609 self.status = TestStatus::Failure;
610 self.reason = Some(format!("failed to set up invariant testing environment: {e}"));
611 }
612
613 #[expect(clippy::too_many_arguments)]
615 pub fn invariant_result(
616 &mut self,
617 gas_report_traces: Vec<Vec<CallTraceArena>>,
618 success: bool,
619 reason: Option<String>,
620 counterexample: Option<CounterExample>,
621 cases: Vec<FuzzedCases>,
622 reverts: usize,
623 metrics: Map<String, InvariantMetrics>,
624 ) {
625 self.kind = TestKind::Invariant {
626 runs: cases.len(),
627 calls: cases.iter().map(|sequence| sequence.cases().len()).sum(),
628 reverts,
629 metrics,
630 };
631 self.status = match success {
632 true => TestStatus::Success,
633 false => TestStatus::Failure,
634 };
635 self.reason = reason;
636 self.counterexample = counterexample;
637 self.gas_report_traces = gas_report_traces;
638 }
639
640 pub fn is_fuzz(&self) -> bool {
642 matches!(self.kind, TestKind::Fuzz { .. })
643 }
644
645 pub fn short_result(&self, name: &str) -> String {
647 format!("{self} {name} {}", self.kind.report())
648 }
649
650 pub fn extend(&mut self, call_result: RawCallResult) {
652 self.logs.extend(call_result.logs);
653 self.labeled_addresses.extend(call_result.labels);
654 self.traces.extend(call_result.traces.map(|traces| (TraceKind::Execution, traces)));
655 self.merge_coverages(call_result.coverage);
656 }
657
658 pub fn merge_coverages(&mut self, other_coverage: Option<HitMaps>) {
660 HitMaps::merge_opt(&mut self.coverage, other_coverage);
661 }
662}
663
664#[derive(Clone, Debug, PartialEq, Eq)]
666pub enum TestKindReport {
667 Unit { gas: u64 },
668 Fuzz { runs: usize, mean_gas: u64, median_gas: u64 },
669 Invariant { runs: usize, calls: usize, reverts: usize, metrics: Map<String, InvariantMetrics> },
670}
671
672impl fmt::Display for TestKindReport {
673 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
674 match self {
675 Self::Unit { gas } => {
676 write!(f, "(gas: {gas})")
677 }
678 Self::Fuzz { runs, mean_gas, median_gas } => {
679 write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})")
680 }
681 Self::Invariant { runs, calls, reverts, metrics: _ } => {
682 write!(f, "(runs: {runs}, calls: {calls}, reverts: {reverts})")
683 }
684 }
685 }
686}
687
688impl TestKindReport {
689 pub fn gas(&self) -> u64 {
691 match *self {
692 Self::Unit { gas } => gas,
693 Self::Fuzz { median_gas, .. } => median_gas,
695 Self::Invariant { .. } => 0,
697 }
698 }
699}
700
701#[derive(Clone, Debug, Serialize, Deserialize)]
703pub enum TestKind {
704 Unit { gas: u64 },
706 Fuzz {
708 first_case: FuzzCase,
710 runs: usize,
711 mean_gas: u64,
712 median_gas: u64,
713 },
714 Invariant { runs: usize, calls: usize, reverts: usize, metrics: Map<String, InvariantMetrics> },
716}
717
718impl Default for TestKind {
719 fn default() -> Self {
720 Self::Unit { gas: 0 }
721 }
722}
723
724impl TestKind {
725 pub fn report(&self) -> TestKindReport {
727 match self {
728 Self::Unit { gas } => TestKindReport::Unit { gas: *gas },
729 Self::Fuzz { first_case: _, runs, mean_gas, median_gas } => {
730 TestKindReport::Fuzz { runs: *runs, mean_gas: *mean_gas, median_gas: *median_gas }
731 }
732 Self::Invariant { runs, calls, reverts, metrics: _ } => TestKindReport::Invariant {
733 runs: *runs,
734 calls: *calls,
735 reverts: *reverts,
736 metrics: HashMap::default(),
737 },
738 }
739 }
740}
741
742#[derive(Clone, Debug, Default)]
747pub struct TestSetup {
748 pub address: Address,
750 pub fuzz_fixtures: FuzzFixtures,
752
753 pub logs: Vec<Log>,
755 pub labels: AddressHashMap<String>,
757 pub traces: Traces,
759 pub coverage: Option<HitMaps>,
761 pub deployed_libs: Vec<Address>,
763
764 pub reason: Option<String>,
766 pub skipped: bool,
768 pub deployment_failure: bool,
770}
771
772impl TestSetup {
773 pub fn failed(reason: String) -> Self {
774 Self { reason: Some(reason), ..Default::default() }
775 }
776
777 pub fn skipped(reason: String) -> Self {
778 Self { reason: Some(reason), skipped: true, ..Default::default() }
779 }
780
781 pub fn extend(&mut self, raw: RawCallResult, trace_kind: TraceKind) {
782 self.logs.extend(raw.logs);
783 self.labels.extend(raw.labels);
784 self.traces.extend(raw.traces.map(|traces| (trace_kind, traces)));
785 HitMaps::merge_opt(&mut self.coverage, raw.coverage);
786 }
787}