1use crate::{
4 fuzz::{BaseCounterExample, FuzzedCases},
5 gas_report::GasReport,
6};
7use alloy_primitives::{
8 Address, Log,
9 map::{AddressHashMap, HashMap},
10};
11use eyre::Report;
12use foundry_common::{evm::Breakpoints, get_contract_name, get_file_name, shell};
13use foundry_evm::{
14 coverage::HitMaps,
15 decode::SkipReason,
16 executors::{RawCallResult, invariant::InvariantMetrics},
17 fuzz::{CounterExample, FuzzCase, FuzzFixtures, FuzzTestResult},
18 traces::{CallTraceArena, CallTraceDecoder, TraceKind, Traces},
19};
20use serde::{Deserialize, Serialize};
21use std::{
22 collections::{BTreeMap, HashMap as Map},
23 fmt::{self, Write},
24 time::Duration,
25};
26use yansi::Paint;
27
28#[derive(Clone, Debug)]
30pub struct TestOutcome {
31 pub results: BTreeMap<String, SuiteResult>,
35 pub allow_failure: bool,
37 pub last_run_decoder: Option<CallTraceDecoder>,
43 pub gas_report: Option<GasReport>,
45}
46
47impl TestOutcome {
48 pub fn new(results: BTreeMap<String, SuiteResult>, allow_failure: bool) -> Self {
50 Self { results, allow_failure, last_run_decoder: None, gas_report: None }
51 }
52
53 pub fn empty(allow_failure: bool) -> Self {
55 Self::new(BTreeMap::new(), allow_failure)
56 }
57
58 pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
60 self.tests().filter(|(_, t)| t.status.is_success())
61 }
62
63 pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
65 self.tests().filter(|(_, t)| t.status.is_skipped())
66 }
67
68 pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
70 self.tests().filter(|(_, t)| t.status.is_failure())
71 }
72
73 pub fn tests(&self) -> impl Iterator<Item = (&String, &TestResult)> {
75 self.results.values().flat_map(|suite| suite.tests())
76 }
77
78 pub fn into_tests_cloned(&self) -> impl Iterator<Item = SuiteTestResult> + '_ {
81 self.results
82 .iter()
83 .flat_map(|(file, suite)| {
84 suite
85 .test_results
86 .iter()
87 .map(move |(sig, result)| (file.clone(), sig.clone(), result.clone()))
88 })
89 .map(|(artifact_id, signature, result)| SuiteTestResult {
90 artifact_id,
91 signature,
92 result,
93 })
94 }
95
96 pub fn into_tests(self) -> impl Iterator<Item = SuiteTestResult> {
98 self.results
99 .into_iter()
100 .flat_map(|(file, suite)| {
101 suite.test_results.into_iter().map(move |t| (file.clone(), t))
102 })
103 .map(|(artifact_id, (signature, result))| SuiteTestResult {
104 artifact_id,
105 signature,
106 result,
107 })
108 }
109
110 pub fn passed(&self) -> usize {
112 self.successes().count()
113 }
114
115 pub fn skipped(&self) -> usize {
117 self.skips().count()
118 }
119
120 pub fn failed(&self) -> usize {
122 self.failures().count()
123 }
124
125 pub fn total_time(&self) -> Duration {
129 self.results.values().map(|suite| suite.duration).sum()
130 }
131
132 pub fn summary(&self, wall_clock_time: Duration) -> String {
134 let num_test_suites = self.results.len();
135 let suites = if num_test_suites == 1 { "suite" } else { "suites" };
136 let total_passed = self.passed();
137 let total_failed = self.failed();
138 let total_skipped = self.skipped();
139 let total_tests = total_passed + total_failed + total_skipped;
140 format!(
141 "\nRan {} test {} in {:.2?} ({:.2?} CPU time): {} tests passed, {} failed, {} skipped ({} total tests)",
142 num_test_suites,
143 suites,
144 wall_clock_time,
145 self.total_time(),
146 total_passed.green(),
147 total_failed.red(),
148 total_skipped.yellow(),
149 total_tests
150 )
151 }
152
153 pub fn ensure_ok(&self, silent: bool) -> eyre::Result<()> {
155 let outcome = self;
156 let failures = outcome.failures().count();
157 if outcome.allow_failure || failures == 0 {
158 return Ok(());
159 }
160
161 if shell::is_quiet() || silent {
162 std::process::exit(1);
164 }
165
166 sh_println!("\nFailing tests:")?;
167 for (suite_name, suite) in &outcome.results {
168 let failed = suite.failed();
169 if failed == 0 {
170 continue;
171 }
172
173 let term = if failed > 1 { "tests" } else { "test" };
174 sh_println!("Encountered {failed} failing {term} in {suite_name}")?;
175 for (name, result) in suite.failures() {
176 sh_println!("{}", result.short_result(name))?;
177 }
178 sh_println!()?;
179 }
180 let successes = outcome.passed();
181 sh_println!(
182 "Encountered a total of {} failing tests, {} tests succeeded",
183 failures.to_string().red(),
184 successes.to_string().green()
185 )?;
186
187 std::process::exit(1);
189 }
190
191 pub fn remove_first(&mut self) -> Option<(String, String, TestResult)> {
193 self.results.iter_mut().find_map(|(suite_name, suite)| {
194 if let Some(test_name) = suite.test_results.keys().next().cloned() {
195 let result = suite.test_results.remove(&test_name).unwrap();
196 Some((suite_name.clone(), test_name, result))
197 } else {
198 None
199 }
200 })
201 }
202}
203
204#[derive(Clone, Debug, Serialize)]
206pub struct SuiteResult {
207 #[serde(with = "foundry_common::serde_helpers::duration")]
209 pub duration: Duration,
210 pub test_results: BTreeMap<String, TestResult>,
212 pub warnings: Vec<String>,
214}
215
216impl SuiteResult {
217 pub fn new(
218 duration: Duration,
219 test_results: BTreeMap<String, TestResult>,
220 mut warnings: Vec<String>,
221 ) -> Self {
222 let mut deprecated_cheatcodes = HashMap::new();
224 for test_result in test_results.values() {
225 deprecated_cheatcodes.extend(test_result.deprecated_cheatcodes.clone());
226 }
227 if !deprecated_cheatcodes.is_empty() {
228 let mut warning =
229 "the following cheatcode(s) are deprecated and will be removed in future versions:"
230 .to_string();
231 for (cheatcode, reason) in deprecated_cheatcodes {
232 write!(warning, "\n {cheatcode}").unwrap();
233 if let Some(reason) = reason {
234 write!(warning, ": {reason}").unwrap();
235 }
236 }
237 warnings.push(warning);
238 }
239
240 Self { duration, test_results, warnings }
241 }
242
243 pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
245 self.tests().filter(|(_, t)| t.status.is_success())
246 }
247
248 pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
250 self.tests().filter(|(_, t)| t.status.is_skipped())
251 }
252
253 pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
255 self.tests().filter(|(_, t)| t.status.is_failure())
256 }
257
258 pub fn passed(&self) -> usize {
260 self.successes().count()
261 }
262
263 pub fn skipped(&self) -> usize {
265 self.skips().count()
266 }
267
268 pub fn failed(&self) -> usize {
270 self.failures().count()
271 }
272
273 pub fn tests(&self) -> impl Iterator<Item = (&String, &TestResult)> {
275 self.test_results.iter()
276 }
277
278 pub fn is_empty(&self) -> bool {
280 self.test_results.is_empty()
281 }
282
283 pub fn len(&self) -> usize {
285 self.test_results.len()
286 }
287
288 pub fn total_time(&self) -> Duration {
292 self.test_results.values().map(|result| result.duration).sum()
293 }
294
295 pub fn summary(&self) -> String {
297 let failed = self.failed();
298 let result = if failed == 0 { "ok".green() } else { "FAILED".red() };
299 format!(
300 "Suite result: {}. {} passed; {} failed; {} skipped; finished in {:.2?} ({:.2?} CPU time)",
301 result,
302 self.passed().green(),
303 failed.red(),
304 self.skipped().yellow(),
305 self.duration,
306 self.total_time(),
307 )
308 }
309}
310
311#[derive(Clone, Debug)]
315pub struct SuiteTestResult {
316 pub artifact_id: String,
319 pub signature: String,
321 pub result: TestResult,
323}
324
325impl SuiteTestResult {
326 pub fn gas_used(&self) -> u64 {
328 self.result.kind.report().gas()
329 }
330
331 pub fn contract_name(&self) -> &str {
333 get_contract_name(&self.artifact_id)
334 }
335
336 pub fn file_name(&self) -> &str {
338 get_file_name(&self.artifact_id)
339 }
340}
341
342#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
344pub enum TestStatus {
345 Success,
346 #[default]
347 Failure,
348 Skipped,
349}
350
351impl TestStatus {
352 #[inline]
354 pub fn is_success(self) -> bool {
355 matches!(self, Self::Success)
356 }
357
358 #[inline]
360 pub fn is_failure(self) -> bool {
361 matches!(self, Self::Failure)
362 }
363
364 #[inline]
366 pub fn is_skipped(self) -> bool {
367 matches!(self, Self::Skipped)
368 }
369}
370
371#[derive(Clone, Debug, Default, Serialize, Deserialize)]
373pub struct TestResult {
374 pub status: TestStatus,
379
380 pub reason: Option<String>,
383
384 pub counterexample: Option<CounterExample>,
386
387 pub logs: Vec<Log>,
390
391 pub decoded_logs: Vec<String>,
394
395 pub kind: TestKind,
397
398 pub traces: Traces,
400
401 #[serde(skip)]
403 pub gas_report_traces: Vec<Vec<CallTraceArena>>,
404
405 #[serde(skip)]
407 pub line_coverage: Option<HitMaps>,
408
409 pub labeled_addresses: AddressHashMap<String>,
411
412 #[serde(with = "foundry_common::serde_helpers::duration")]
413 pub duration: Duration,
414
415 pub breakpoints: Breakpoints,
417
418 pub gas_snapshots: BTreeMap<String, BTreeMap<String, String>>,
420
421 #[serde(skip)]
423 pub deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>,
424}
425
426impl fmt::Display for TestResult {
427 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
428 match self.status {
429 TestStatus::Success => "[PASS]".green().fmt(f),
430 TestStatus::Skipped => {
431 let mut s = String::from("[SKIP");
432 if let Some(reason) = &self.reason {
433 write!(s, ": {reason}").unwrap();
434 }
435 s.push(']');
436 s.yellow().fmt(f)
437 }
438 TestStatus::Failure => {
439 let mut s = String::from("[FAIL");
440 if self.reason.is_some() || self.counterexample.is_some() {
441 if let Some(reason) = &self.reason {
442 write!(s, ": {reason}").unwrap();
443 }
444
445 if let Some(counterexample) = &self.counterexample {
446 match counterexample {
447 CounterExample::Single(ex) => {
448 write!(s, "; counterexample: {ex}]").unwrap();
449 }
450 CounterExample::Sequence(original, sequence) => {
451 s.push_str(
452 format!(
453 "]\n\t[Sequence] (original: {original}, shrunk: {})\n",
454 sequence.len()
455 )
456 .as_str(),
457 );
458 for ex in sequence {
459 writeln!(s, "{ex}").unwrap();
460 }
461 }
462 }
463 } else {
464 s.push(']');
465 }
466 } else {
467 s.push(']');
468 }
469 s.red().fmt(f)
470 }
471 }
472 }
473}
474
475impl TestResult {
476 pub fn new(setup: &TestSetup) -> Self {
478 Self {
479 labeled_addresses: setup.labels.clone(),
480 logs: setup.logs.clone(),
481 traces: setup.traces.clone(),
482 line_coverage: setup.coverage.clone(),
483 ..Default::default()
484 }
485 }
486
487 pub fn fail(reason: String) -> Self {
489 Self { status: TestStatus::Failure, reason: Some(reason), ..Default::default() }
490 }
491
492 pub fn setup_result(setup: TestSetup) -> Self {
494 Self {
495 status: if setup.skipped { TestStatus::Skipped } else { TestStatus::Failure },
496 reason: setup.reason,
497 logs: setup.logs,
498 traces: setup.traces,
499 line_coverage: setup.coverage,
500 labeled_addresses: setup.labels,
501 ..Default::default()
502 }
503 }
504
505 pub fn single_skip(&mut self, reason: SkipReason) {
507 self.status = TestStatus::Skipped;
508 self.reason = reason.0;
509 }
510
511 pub fn single_fail(&mut self, reason: Option<String>) {
513 self.status = TestStatus::Failure;
514 self.reason = reason;
515 }
516
517 pub fn single_result(
520 &mut self,
521 success: bool,
522 reason: Option<String>,
523 raw_call_result: RawCallResult,
524 ) {
525 self.kind =
526 TestKind::Unit { gas: raw_call_result.gas_used.wrapping_sub(raw_call_result.stipend) };
527
528 self.logs.extend(raw_call_result.logs);
530 self.labeled_addresses.extend(raw_call_result.labels);
531 self.traces.extend(raw_call_result.traces.map(|traces| (TraceKind::Execution, traces)));
532 self.merge_coverages(raw_call_result.line_coverage);
533
534 self.status = match success {
535 true => TestStatus::Success,
536 false => TestStatus::Failure,
537 };
538 self.reason = reason;
539 self.duration = Duration::default();
540 self.gas_report_traces = Vec::new();
541
542 if let Some(cheatcodes) = raw_call_result.cheatcodes {
543 self.breakpoints = cheatcodes.breakpoints;
544 self.gas_snapshots = cheatcodes.gas_snapshots;
545 self.deprecated_cheatcodes = cheatcodes.deprecated;
546 }
547 }
548
549 pub fn fuzz_result(&mut self, result: FuzzTestResult) {
552 self.kind = TestKind::Fuzz {
553 median_gas: result.median_gas(false),
554 mean_gas: result.mean_gas(false),
555 first_case: result.first_case,
556 runs: result.gas_by_case.len(),
557 };
558
559 self.logs.extend(result.logs);
561 self.labeled_addresses.extend(result.labeled_addresses);
562 self.traces.extend(result.traces.map(|traces| (TraceKind::Execution, traces)));
563 self.merge_coverages(result.line_coverage);
564
565 self.status = if result.skipped {
566 TestStatus::Skipped
567 } else if result.success {
568 TestStatus::Success
569 } else {
570 TestStatus::Failure
571 };
572 self.reason = result.reason;
573 self.counterexample = result.counterexample;
574 self.duration = Duration::default();
575 self.gas_report_traces = result.gas_report_traces.into_iter().map(|t| vec![t]).collect();
576 self.breakpoints = result.breakpoints.unwrap_or_default();
577 self.deprecated_cheatcodes = result.deprecated_cheatcodes;
578 }
579
580 pub fn invariant_skip(&mut self, reason: SkipReason) {
582 self.kind = TestKind::Invariant {
583 runs: 1,
584 calls: 1,
585 reverts: 1,
586 metrics: HashMap::default(),
587 failed_corpus_replays: 0,
588 };
589 self.status = TestStatus::Skipped;
590 self.reason = reason.0;
591 }
592
593 pub fn invariant_replay_fail(
595 &mut self,
596 replayed_entirely: bool,
597 invariant_name: &String,
598 call_sequence: Vec<BaseCounterExample>,
599 ) {
600 self.kind = TestKind::Invariant {
601 runs: 1,
602 calls: 1,
603 reverts: 1,
604 metrics: HashMap::default(),
605 failed_corpus_replays: 0,
606 };
607 self.status = TestStatus::Failure;
608 self.reason = if replayed_entirely {
609 Some(format!("{invariant_name} replay failure"))
610 } else {
611 Some(format!("{invariant_name} persisted failure revert"))
612 };
613 self.counterexample = Some(CounterExample::Sequence(call_sequence.len(), call_sequence));
614 }
615
616 pub fn invariant_setup_fail(&mut self, e: Report) {
618 self.kind = TestKind::Invariant {
619 runs: 0,
620 calls: 0,
621 reverts: 0,
622 metrics: HashMap::default(),
623 failed_corpus_replays: 0,
624 };
625 self.status = TestStatus::Failure;
626 self.reason = Some(format!("failed to set up invariant testing environment: {e}"));
627 }
628
629 #[expect(clippy::too_many_arguments)]
631 pub fn invariant_result(
632 &mut self,
633 gas_report_traces: Vec<Vec<CallTraceArena>>,
634 success: bool,
635 reason: Option<String>,
636 counterexample: Option<CounterExample>,
637 cases: Vec<FuzzedCases>,
638 reverts: usize,
639 metrics: Map<String, InvariantMetrics>,
640 failed_corpus_replays: usize,
641 ) {
642 self.kind = TestKind::Invariant {
643 runs: cases.len(),
644 calls: cases.iter().map(|sequence| sequence.cases().len()).sum(),
645 reverts,
646 metrics,
647 failed_corpus_replays,
648 };
649 self.status = match success {
650 true => TestStatus::Success,
651 false => TestStatus::Failure,
652 };
653 self.reason = reason;
654 self.counterexample = counterexample;
655 self.gas_report_traces = gas_report_traces;
656 }
657
658 pub fn is_fuzz(&self) -> bool {
660 matches!(self.kind, TestKind::Fuzz { .. })
661 }
662
663 pub fn short_result(&self, name: &str) -> String {
665 format!("{self} {name} {}", self.kind.report())
666 }
667
668 pub fn extend(&mut self, call_result: RawCallResult) {
670 self.logs.extend(call_result.logs);
671 self.labeled_addresses.extend(call_result.labels);
672 self.traces.extend(call_result.traces.map(|traces| (TraceKind::Execution, traces)));
673 self.merge_coverages(call_result.line_coverage);
674 }
675
676 pub fn merge_coverages(&mut self, other_coverage: Option<HitMaps>) {
678 HitMaps::merge_opt(&mut self.line_coverage, other_coverage);
679 }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
684pub enum TestKindReport {
685 Unit {
686 gas: u64,
687 },
688 Fuzz {
689 runs: usize,
690 mean_gas: u64,
691 median_gas: u64,
692 },
693 Invariant {
694 runs: usize,
695 calls: usize,
696 reverts: usize,
697 metrics: Map<String, InvariantMetrics>,
698 failed_corpus_replays: usize,
699 },
700}
701
702impl fmt::Display for TestKindReport {
703 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
704 match self {
705 Self::Unit { gas } => {
706 write!(f, "(gas: {gas})")
707 }
708 Self::Fuzz { runs, mean_gas, median_gas } => {
709 write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})")
710 }
711 Self::Invariant { runs, calls, reverts, metrics: _, failed_corpus_replays } => {
712 if *failed_corpus_replays != 0 {
713 write!(
714 f,
715 "(runs: {runs}, calls: {calls}, reverts: {reverts}, failed corpus replays: {failed_corpus_replays})"
716 )
717 } else {
718 write!(f, "(runs: {runs}, calls: {calls}, reverts: {reverts})")
719 }
720 }
721 }
722 }
723}
724
725impl TestKindReport {
726 pub fn gas(&self) -> u64 {
728 match *self {
729 Self::Unit { gas } => gas,
730 Self::Fuzz { median_gas, .. } => median_gas,
732 Self::Invariant { .. } => 0,
734 }
735 }
736}
737
738#[derive(Clone, Debug, Serialize, Deserialize)]
740pub enum TestKind {
741 Unit { gas: u64 },
743 Fuzz {
745 first_case: FuzzCase,
747 runs: usize,
748 mean_gas: u64,
749 median_gas: u64,
750 },
751 Invariant {
753 runs: usize,
754 calls: usize,
755 reverts: usize,
756 metrics: Map<String, InvariantMetrics>,
757 failed_corpus_replays: usize,
758 },
759}
760
761impl Default for TestKind {
762 fn default() -> Self {
763 Self::Unit { gas: 0 }
764 }
765}
766
767impl TestKind {
768 pub fn report(&self) -> TestKindReport {
770 match self {
771 Self::Unit { gas } => TestKindReport::Unit { gas: *gas },
772 Self::Fuzz { first_case: _, runs, mean_gas, median_gas } => {
773 TestKindReport::Fuzz { runs: *runs, mean_gas: *mean_gas, median_gas: *median_gas }
774 }
775 Self::Invariant { runs, calls, reverts, metrics: _, failed_corpus_replays } => {
776 TestKindReport::Invariant {
777 runs: *runs,
778 calls: *calls,
779 reverts: *reverts,
780 metrics: HashMap::default(),
781 failed_corpus_replays: *failed_corpus_replays,
782 }
783 }
784 }
785 }
786}
787
788#[derive(Clone, Debug, Default)]
793pub struct TestSetup {
794 pub address: Address,
796 pub fuzz_fixtures: FuzzFixtures,
798
799 pub logs: Vec<Log>,
801 pub labels: AddressHashMap<String>,
803 pub traces: Traces,
805 pub coverage: Option<HitMaps>,
807 pub deployed_libs: Vec<Address>,
809
810 pub reason: Option<String>,
812 pub skipped: bool,
814 pub deployment_failure: bool,
816}
817
818impl TestSetup {
819 pub fn failed(reason: String) -> Self {
820 Self { reason: Some(reason), ..Default::default() }
821 }
822
823 pub fn skipped(reason: String) -> Self {
824 Self { reason: Some(reason), skipped: true, ..Default::default() }
825 }
826
827 pub fn extend(&mut self, raw: RawCallResult, trace_kind: TraceKind) {
828 self.logs.extend(raw.logs);
829 self.labels.extend(raw.labels);
830 self.traces.extend(raw.traces.map(|traces| (trace_kind, traces)));
831 HitMaps::merge_opt(&mut self.coverage, raw.line_coverage);
832 }
833}