use crate::{
fuzz::{BaseCounterExample, FuzzedCases},
gas_report::GasReport,
};
use alloy_primitives::{
map::{AddressHashMap, HashMap},
Address, Log,
};
use eyre::Report;
use foundry_common::{evm::Breakpoints, get_contract_name, get_file_name, shell};
use foundry_evm::{
coverage::HitMaps,
decode::SkipReason,
executors::{invariant::InvariantMetrics, RawCallResult},
fuzz::{CounterExample, FuzzCase, FuzzFixtures, FuzzTestResult},
traces::{CallTraceArena, CallTraceDecoder, TraceKind, Traces},
};
use serde::{Deserialize, Serialize};
use std::{
collections::{BTreeMap, HashMap as Map},
fmt::{self, Write},
time::Duration,
};
use yansi::Paint;
#[derive(Clone, Debug)]
pub struct TestOutcome {
pub results: BTreeMap<String, SuiteResult>,
pub allow_failure: bool,
pub last_run_decoder: Option<CallTraceDecoder>,
pub gas_report: Option<GasReport>,
}
impl TestOutcome {
pub fn new(results: BTreeMap<String, SuiteResult>, allow_failure: bool) -> Self {
Self { results, allow_failure, last_run_decoder: None, gas_report: None }
}
pub fn empty(allow_failure: bool) -> Self {
Self::new(BTreeMap::new(), allow_failure)
}
pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.tests().filter(|(_, t)| t.status.is_success())
}
pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.tests().filter(|(_, t)| t.status.is_skipped())
}
pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.tests().filter(|(_, t)| t.status.is_failure())
}
pub fn tests(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.results.values().flat_map(|suite| suite.tests())
}
pub fn into_tests_cloned(&self) -> impl Iterator<Item = SuiteTestResult> + '_ {
self.results
.iter()
.flat_map(|(file, suite)| {
suite
.test_results
.iter()
.map(move |(sig, result)| (file.clone(), sig.clone(), result.clone()))
})
.map(|(artifact_id, signature, result)| SuiteTestResult {
artifact_id,
signature,
result,
})
}
pub fn into_tests(self) -> impl Iterator<Item = SuiteTestResult> {
self.results
.into_iter()
.flat_map(|(file, suite)| {
suite.test_results.into_iter().map(move |t| (file.clone(), t))
})
.map(|(artifact_id, (signature, result))| SuiteTestResult {
artifact_id,
signature,
result,
})
}
pub fn passed(&self) -> usize {
self.successes().count()
}
pub fn skipped(&self) -> usize {
self.skips().count()
}
pub fn failed(&self) -> usize {
self.failures().count()
}
pub fn total_time(&self) -> Duration {
self.results.values().map(|suite| suite.duration).sum()
}
pub fn summary(&self, wall_clock_time: Duration) -> String {
let num_test_suites = self.results.len();
let suites = if num_test_suites == 1 { "suite" } else { "suites" };
let total_passed = self.passed();
let total_failed = self.failed();
let total_skipped = self.skipped();
let total_tests = total_passed + total_failed + total_skipped;
format!(
"\nRan {} test {} in {:.2?} ({:.2?} CPU time): {} tests passed, {} failed, {} skipped ({} total tests)",
num_test_suites,
suites,
wall_clock_time,
self.total_time(),
total_passed.green(),
total_failed.red(),
total_skipped.yellow(),
total_tests
)
}
pub fn ensure_ok(&self, silent: bool) -> eyre::Result<()> {
let outcome = self;
let failures = outcome.failures().count();
if outcome.allow_failure || failures == 0 {
return Ok(());
}
if shell::is_quiet() || silent {
std::process::exit(1);
}
sh_println!("\nFailing tests:")?;
for (suite_name, suite) in outcome.results.iter() {
let failed = suite.failed();
if failed == 0 {
continue;
}
let term = if failed > 1 { "tests" } else { "test" };
sh_println!("Encountered {failed} failing {term} in {suite_name}")?;
for (name, result) in suite.failures() {
sh_println!("{}", result.short_result(name))?;
}
sh_println!()?;
}
let successes = outcome.passed();
sh_println!(
"Encountered a total of {} failing tests, {} tests succeeded",
failures.to_string().red(),
successes.to_string().green()
)?;
std::process::exit(1);
}
pub fn remove_first(&mut self) -> Option<(String, String, TestResult)> {
self.results.iter_mut().find_map(|(suite_name, suite)| {
if let Some(test_name) = suite.test_results.keys().next().cloned() {
let result = suite.test_results.remove(&test_name).unwrap();
Some((suite_name.clone(), test_name, result))
} else {
None
}
})
}
}
#[derive(Clone, Debug, Serialize)]
pub struct SuiteResult {
#[serde(with = "humantime_serde")]
pub duration: Duration,
pub test_results: BTreeMap<String, TestResult>,
pub warnings: Vec<String>,
}
impl SuiteResult {
pub fn new(
duration: Duration,
test_results: BTreeMap<String, TestResult>,
mut warnings: Vec<String>,
) -> Self {
let mut deprecated_cheatcodes = HashMap::new();
for test_result in test_results.values() {
deprecated_cheatcodes.extend(test_result.deprecated_cheatcodes.clone());
}
if !deprecated_cheatcodes.is_empty() {
let mut warning =
"the following cheatcode(s) are deprecated and will be removed in future versions:"
.to_string();
for (cheatcode, reason) in deprecated_cheatcodes {
write!(warning, "\n {cheatcode}").unwrap();
if let Some(reason) = reason {
write!(warning, ": {reason}").unwrap();
}
}
warnings.push(warning);
}
Self { duration, test_results, warnings }
}
pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.tests().filter(|(_, t)| t.status.is_success())
}
pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.tests().filter(|(_, t)| t.status.is_skipped())
}
pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.tests().filter(|(_, t)| t.status.is_failure())
}
pub fn passed(&self) -> usize {
self.successes().count()
}
pub fn skipped(&self) -> usize {
self.skips().count()
}
pub fn failed(&self) -> usize {
self.failures().count()
}
pub fn tests(&self) -> impl Iterator<Item = (&String, &TestResult)> {
self.test_results.iter()
}
pub fn is_empty(&self) -> bool {
self.test_results.is_empty()
}
pub fn len(&self) -> usize {
self.test_results.len()
}
pub fn total_time(&self) -> Duration {
self.test_results.values().map(|result| result.duration).sum()
}
pub fn summary(&self) -> String {
let failed = self.failed();
let result = if failed == 0 { "ok".green() } else { "FAILED".red() };
format!(
"Suite result: {}. {} passed; {} failed; {} skipped; finished in {:.2?} ({:.2?} CPU time)",
result,
self.passed().green(),
failed.red(),
self.skipped().yellow(),
self.duration,
self.total_time(),
)
}
}
#[derive(Clone, Debug)]
pub struct SuiteTestResult {
pub artifact_id: String,
pub signature: String,
pub result: TestResult,
}
impl SuiteTestResult {
pub fn gas_used(&self) -> u64 {
self.result.kind.report().gas()
}
pub fn contract_name(&self) -> &str {
get_contract_name(&self.artifact_id)
}
pub fn file_name(&self) -> &str {
get_file_name(&self.artifact_id)
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub enum TestStatus {
Success,
#[default]
Failure,
Skipped,
}
impl TestStatus {
#[inline]
pub fn is_success(self) -> bool {
matches!(self, Self::Success)
}
#[inline]
pub fn is_failure(self) -> bool {
matches!(self, Self::Failure)
}
#[inline]
pub fn is_skipped(self) -> bool {
matches!(self, Self::Skipped)
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct TestResult {
pub status: TestStatus,
pub reason: Option<String>,
pub counterexample: Option<CounterExample>,
pub logs: Vec<Log>,
pub decoded_logs: Vec<String>,
pub kind: TestKind,
pub traces: Traces,
#[serde(skip)]
pub gas_report_traces: Vec<Vec<CallTraceArena>>,
#[serde(skip)]
pub coverage: Option<HitMaps>,
pub labeled_addresses: AddressHashMap<String>,
pub duration: Duration,
pub breakpoints: Breakpoints,
pub gas_snapshots: BTreeMap<String, BTreeMap<String, String>>,
#[serde(skip)]
pub deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>,
}
impl fmt::Display for TestResult {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.status {
TestStatus::Success => "[PASS]".green().fmt(f),
TestStatus::Skipped => {
let mut s = String::from("[SKIP");
if let Some(reason) = &self.reason {
write!(s, ": {reason}").unwrap();
}
s.push(']');
s.yellow().fmt(f)
}
TestStatus::Failure => {
let mut s = String::from("[FAIL");
if self.reason.is_some() || self.counterexample.is_some() {
if let Some(reason) = &self.reason {
write!(s, ": {reason}").unwrap();
}
if let Some(counterexample) = &self.counterexample {
match counterexample {
CounterExample::Single(ex) => {
write!(s, "; counterexample: {ex}]").unwrap();
}
CounterExample::Sequence(sequence) => {
s.push_str("]\n\t[Sequence]\n");
for ex in sequence {
writeln!(s, "\t\t{ex}").unwrap();
}
}
}
} else {
s.push(']');
}
} else {
s.push(']');
}
s.red().fmt(f)
}
}
}
}
impl TestResult {
pub fn new(setup: &TestSetup) -> Self {
Self {
labeled_addresses: setup.labels.clone(),
logs: setup.logs.clone(),
traces: setup.traces.clone(),
coverage: setup.coverage.clone(),
..Default::default()
}
}
pub fn fail(reason: String) -> Self {
Self { status: TestStatus::Failure, reason: Some(reason), ..Default::default() }
}
pub fn setup_result(setup: TestSetup) -> Self {
Self {
status: if setup.skipped { TestStatus::Skipped } else { TestStatus::Failure },
reason: setup.reason,
logs: setup.logs,
traces: setup.traces,
coverage: setup.coverage,
labeled_addresses: setup.labels,
..Default::default()
}
}
pub fn single_skip(&mut self, reason: SkipReason) {
self.status = TestStatus::Skipped;
self.reason = reason.0;
}
pub fn single_fail(&mut self, reason: Option<String>) {
self.status = TestStatus::Failure;
self.reason = reason;
}
pub fn single_result(
&mut self,
success: bool,
reason: Option<String>,
raw_call_result: RawCallResult,
) {
self.kind =
TestKind::Unit { gas: raw_call_result.gas_used.wrapping_sub(raw_call_result.stipend) };
self.logs.extend(raw_call_result.logs);
self.labeled_addresses.extend(raw_call_result.labels);
self.traces.extend(raw_call_result.traces.map(|traces| (TraceKind::Execution, traces)));
self.merge_coverages(raw_call_result.coverage);
self.status = match success {
true => TestStatus::Success,
false => TestStatus::Failure,
};
self.reason = reason;
self.duration = Duration::default();
self.gas_report_traces = Vec::new();
if let Some(cheatcodes) = raw_call_result.cheatcodes {
self.breakpoints = cheatcodes.breakpoints;
self.gas_snapshots = cheatcodes.gas_snapshots;
self.deprecated_cheatcodes = cheatcodes.deprecated;
}
}
pub fn fuzz_result(&mut self, result: FuzzTestResult) {
self.kind = TestKind::Fuzz {
median_gas: result.median_gas(false),
mean_gas: result.mean_gas(false),
first_case: result.first_case,
runs: result.gas_by_case.len(),
};
self.logs.extend(result.logs);
self.labeled_addresses.extend(result.labeled_addresses);
self.traces.extend(result.traces.map(|traces| (TraceKind::Execution, traces)));
self.merge_coverages(result.coverage);
self.status = if result.skipped {
TestStatus::Skipped
} else if result.success {
TestStatus::Success
} else {
TestStatus::Failure
};
self.reason = result.reason;
self.counterexample = result.counterexample;
self.duration = Duration::default();
self.gas_report_traces = result.gas_report_traces.into_iter().map(|t| vec![t]).collect();
self.breakpoints = result.breakpoints.unwrap_or_default();
self.deprecated_cheatcodes = result.deprecated_cheatcodes;
}
pub fn invariant_skip(&mut self, reason: SkipReason) {
self.kind =
TestKind::Invariant { runs: 1, calls: 1, reverts: 1, metrics: HashMap::default() };
self.status = TestStatus::Skipped;
self.reason = reason.0;
}
pub fn invariant_replay_fail(
&mut self,
replayed_entirely: bool,
invariant_name: &String,
call_sequence: Vec<BaseCounterExample>,
) {
self.kind =
TestKind::Invariant { runs: 1, calls: 1, reverts: 1, metrics: HashMap::default() };
self.status = TestStatus::Failure;
self.reason = if replayed_entirely {
Some(format!("{invariant_name} replay failure"))
} else {
Some(format!("{invariant_name} persisted failure revert"))
};
self.counterexample = Some(CounterExample::Sequence(call_sequence));
}
pub fn invariant_setup_fail(&mut self, e: Report) {
self.kind =
TestKind::Invariant { runs: 0, calls: 0, reverts: 0, metrics: HashMap::default() };
self.status = TestStatus::Failure;
self.reason = Some(format!("failed to set up invariant testing environment: {e}"));
}
#[allow(clippy::too_many_arguments)]
pub fn invariant_result(
&mut self,
gas_report_traces: Vec<Vec<CallTraceArena>>,
success: bool,
reason: Option<String>,
counterexample: Option<CounterExample>,
cases: Vec<FuzzedCases>,
reverts: usize,
metrics: Map<String, InvariantMetrics>,
) {
self.kind = TestKind::Invariant {
runs: cases.len(),
calls: cases.iter().map(|sequence| sequence.cases().len()).sum(),
reverts,
metrics,
};
self.status = match success {
true => TestStatus::Success,
false => TestStatus::Failure,
};
self.reason = reason;
self.counterexample = counterexample;
self.gas_report_traces = gas_report_traces;
}
pub fn is_fuzz(&self) -> bool {
matches!(self.kind, TestKind::Fuzz { .. })
}
pub fn short_result(&self, name: &str) -> String {
format!("{self} {name} {}", self.kind.report())
}
pub fn extend(&mut self, call_result: RawCallResult) {
self.logs.extend(call_result.logs);
self.labeled_addresses.extend(call_result.labels);
self.traces.extend(call_result.traces.map(|traces| (TraceKind::Execution, traces)));
self.merge_coverages(call_result.coverage);
}
pub fn merge_coverages(&mut self, other_coverage: Option<HitMaps>) {
HitMaps::merge_opt(&mut self.coverage, other_coverage);
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TestKindReport {
Unit { gas: u64 },
Fuzz { runs: usize, mean_gas: u64, median_gas: u64 },
Invariant { runs: usize, calls: usize, reverts: usize, metrics: Map<String, InvariantMetrics> },
}
impl fmt::Display for TestKindReport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Unit { gas } => {
write!(f, "(gas: {gas})")
}
Self::Fuzz { runs, mean_gas, median_gas } => {
write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})")
}
Self::Invariant { runs, calls, reverts, metrics: _ } => {
write!(f, "(runs: {runs}, calls: {calls}, reverts: {reverts})")
}
}
}
}
impl TestKindReport {
pub fn gas(&self) -> u64 {
match *self {
Self::Unit { gas } => gas,
Self::Fuzz { median_gas, .. } => median_gas,
Self::Invariant { .. } => 0,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum TestKind {
Unit { gas: u64 },
Fuzz {
first_case: FuzzCase,
runs: usize,
mean_gas: u64,
median_gas: u64,
},
Invariant { runs: usize, calls: usize, reverts: usize, metrics: Map<String, InvariantMetrics> },
}
impl Default for TestKind {
fn default() -> Self {
Self::Unit { gas: 0 }
}
}
impl TestKind {
pub fn report(&self) -> TestKindReport {
match self {
Self::Unit { gas } => TestKindReport::Unit { gas: *gas },
Self::Fuzz { first_case: _, runs, mean_gas, median_gas } => {
TestKindReport::Fuzz { runs: *runs, mean_gas: *mean_gas, median_gas: *median_gas }
}
Self::Invariant { runs, calls, reverts, metrics: _ } => TestKindReport::Invariant {
runs: *runs,
calls: *calls,
reverts: *reverts,
metrics: HashMap::default(),
},
}
}
}
#[derive(Clone, Debug, Default)]
pub struct TestSetup {
pub address: Address,
pub fuzz_fixtures: FuzzFixtures,
pub logs: Vec<Log>,
pub labels: AddressHashMap<String>,
pub traces: Traces,
pub coverage: Option<HitMaps>,
pub deployed_libs: Vec<Address>,
pub reason: Option<String>,
pub skipped: bool,
}
impl TestSetup {
pub fn failed(reason: String) -> Self {
Self { reason: Some(reason), ..Default::default() }
}
pub fn skipped(reason: String) -> Self {
Self { reason: Some(reason), skipped: true, ..Default::default() }
}
pub fn extend(&mut self, raw: RawCallResult, trace_kind: TraceKind) {
self.logs.extend(raw.logs);
self.labels.extend(raw.labels);
self.traces.extend(raw.traces.map(|traces| (trace_kind, traces)));
HitMaps::merge_opt(&mut self.coverage, raw.coverage);
}
}