1use crate::{
8 cmd::test::RerunFailure,
9 result::TestStatus,
10 sandbox::{RUN_TIMEOUT, Sandbox, TestFile, ensure_inside, run_session, stderr, suite_results},
11 workspace,
12};
13use alloy_primitives::{U256, keccak256};
14use clap::Parser;
15use eyre::{Context, Result, ensure, eyre};
16use foundry_cli::{opts::BuildOpts, utils::LoadConfig};
17use foundry_common::{fs::canonicalize_path, sh_println, sh_warn};
18use foundry_compilers::{
19 Graph,
20 compilers::{
21 Language,
22 multi::{MultiCompilerLanguage, MultiCompilerParser},
23 },
24 utils::source_files_iter,
25};
26use foundry_config::{Config, impl_figment_convert};
27use foundry_evm::fuzz::CounterExample;
28use regex::Regex;
29use serde::{Deserialize, Serialize};
30use solar::parse::Cursor;
31use std::{
32 cmp::Reverse,
33 collections::{BTreeMap, BTreeSet, HashSet},
34 ffi::OsString,
35 fs,
36 io::{self, Write},
37 ops::Add,
38 path::{Path, PathBuf},
39 process::{Command, Output, Stdio},
40 sync::{
41 LazyLock,
42 atomic::{AtomicUsize, Ordering},
43 },
44 thread,
45 time::SystemTime,
46};
47
48mod matcher;
49
50const DEFAULT_AGENTS: usize = 4;
55const DEFAULT_QUORUM: usize = 2;
56const DEFAULT_FOCUSES: [&str; 4] = [
61 "block and time transitions, accrual, vesting, auctions, deadlines, stale snapshots, and \
62 sequences whose result depends on elapsed time.",
63 "stale, reverting, zero, extreme, or adversarial oracle, strategy, pool, token, gauge, hook, \
64 and callback values.",
65 "cached or stored balances, donations, fees, rewards, bad debt, utilization, exchange rates, \
66 and state that can diverge from live balances.",
67 "agreement of max, preview, and view functions before, during, and after lifecycle \
68 transitions, especially empty, paused, removed, nested, and terminal states.",
69];
70const VERIFY_PROMPT: &str = include_str!("properties/verify_prompt.md");
73
74static BLOCKED: LazyLock<Regex> = LazyLock::new(|| {
78 Regex::new(r#"FFI is disabled|network access is disabled|environment variable "[^"]*" not found|failed to resolve env var|inline config cannot"#)
79 .unwrap()
80});
81static COMMENT: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?s)//[^\n]*|/\*.*?\*/").unwrap());
83static CONTRACT: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\bcontract\s+(\w+)").unwrap());
85#[derive(Clone, Debug, Parser)]
87pub struct PropertiesArgs {
88 #[arg(long, value_name = "PATH")]
92 target: Vec<PathBuf>,
93
94 #[arg(long, value_name = "PATH")]
98 context: Vec<PathBuf>,
99
100 #[arg(long, required = true, value_name = "PATH", verbatim_doc_comment)]
124 agent_command: PathBuf,
125
126 #[arg(long, allow_hyphen_values = true, value_name = "ARG")]
128 agent_arg: Vec<OsString>,
129
130 #[arg(long, value_name = "PATH", verbatim_doc_comment)]
153 verifier_command: Option<PathBuf>,
154
155 #[arg(long, allow_hyphen_values = true, requires = "verifier_command", value_name = "ARG")]
157 verifier_arg: Vec<OsString>,
158
159 #[arg(long, required = true, action = clap::ArgAction::Append, value_name = "SEED")]
161 seed: Vec<U256>,
162
163 #[arg(long, default_value_t = DEFAULT_AGENTS, value_name = "N")]
166 agents: usize,
167
168 #[arg(long, value_name = "TEXT")]
175 focus: Vec<String>,
176
177 #[arg(long, conflicts_with = "focus")]
179 no_focus: bool,
180
181 #[arg(long, value_name = "K")]
191 quorum: Option<usize>,
192
193 #[arg(long, value_name = "SECS")]
196 timeout: Option<u64>,
197
198 #[arg(long)]
203 allow_ffi: bool,
204
205 #[arg(long)]
210 allow_network: bool,
211
212 #[command(flatten)]
213 build: BuildOpts,
214}
215
216impl_figment_convert!(PropertiesArgs, build);
217
218#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)]
220struct Usage {
221 #[serde(default)]
222 input_tokens: u64,
223 #[serde(default)]
224 output_tokens: u64,
225 #[serde(default)]
226 cost_usd: f64,
227}
228
229impl Add for Usage {
230 type Output = Self;
231
232 fn add(self, other: Self) -> Self {
233 Self {
234 input_tokens: self.input_tokens + other.input_tokens,
235 output_tokens: self.output_tokens + other.output_tokens,
236 cost_usd: self.cost_usd + other.cost_usd,
237 }
238 }
239}
240
241#[derive(Clone, Debug, Serialize)]
243struct Failure {
244 test: String,
246 path: PathBuf,
247 reason: String,
248 #[serde(skip_serializing_if = "Option::is_none")]
250 claim: Option<String>,
251}
252
253#[derive(Debug, Serialize)]
255struct AgentFailure {
256 agent: usize,
257 #[serde(flatten)]
258 failure: Failure,
259 file: PathBuf,
261}
262
263impl std::fmt::Display for AgentFailure {
264 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265 let failure = &self.failure;
266 write!(f, " agent {}: {} ({})", self.agent, failure.test, failure.reason)?;
267 if let Some(claim) = &failure.claim {
268 write!(f, "\n claim: {claim}")?;
269 }
270 write!(f, "\n file: {}", self.file.display())
271 }
272}
273
274#[derive(Debug, Serialize)]
276struct Rejected {
277 agent: usize,
278 paths: Vec<PathBuf>,
279 reason: String,
280}
281
282#[derive(Debug, Serialize)]
284struct BugGroup {
285 agents: Vec<usize>,
287 failures: Vec<AgentFailure>,
289 #[serde(skip_serializing_if = "Option::is_none")]
291 verification: Option<Verification>,
292}
293
294#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
298#[serde(rename_all = "snake_case")]
299enum Verdict {
300 Confirmed,
302 #[default]
304 NeedsValidation,
305 Rejected,
307}
308
309#[derive(Debug, Deserialize, Serialize)]
311struct Evidence {
312 #[serde(default)]
313 file: String,
314 #[serde(default)]
315 line: u64,
316 #[serde(default)]
317 quote: String,
318}
319
320#[derive(Debug, Default, Deserialize, Serialize)]
322struct Judgment {
323 verdict: Verdict,
324 #[serde(default)]
325 reason: String,
326 #[serde(default)]
327 evidence: Vec<Evidence>,
328 #[serde(default, skip_serializing_if = "Option::is_none")]
330 unresolved: Option<String>,
331}
332
333#[derive(Debug, Serialize)]
335struct Verification {
336 #[serde(flatten)]
337 judgment: Judgment,
338 #[serde(skip_serializing_if = "Option::is_none")]
341 error: Option<String>,
342 #[serde(skip_serializing_if = "Option::is_none")]
343 usage: Option<Usage>,
344}
345
346impl Verification {
347 fn failed(error: String) -> Self {
349 Self { judgment: Judgment::default(), error: Some(error), usage: None }
350 }
351}
352
353impl std::fmt::Display for Verification {
354 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
355 let judgment = &self.judgment;
356 let mut lines = Vec::new();
357 if !judgment.reason.is_empty() {
358 lines.push(format!(" verifier: {}", judgment.reason));
359 }
360 if let Some(error) = &self.error {
361 lines.push(format!(" verifier error: {error}"));
362 }
363 if let Some(unresolved) = judgment.unresolved.as_deref().filter(|fact| !fact.is_empty()) {
364 lines.push(format!(" unresolved: {unresolved}"));
365 }
366 for evidence in &judgment.evidence {
367 let quote = evidence.quote.split_whitespace().collect::<Vec<_>>().join(" ");
368 lines.push(format!(" evidence: {}:{}: {quote}", evidence.file, evidence.line));
369 }
370 write!(f, "{}", lines.join("\n"))
371 }
372}
373
374#[derive(Default)]
376struct TestRun {
377 possible_bugs: Vec<Failure>,
379 passing: Vec<RerunFailure>,
381 failing_files: HashSet<PathBuf>,
383 errors: Vec<String>,
385}
386
387struct Session {
389 id: usize,
390 exit: AgentExit,
392 usage: Option<Usage>,
393 possible_bugs: Vec<AgentFailure>,
394 rejected: Vec<Rejected>,
395 kept: Vec<TestFile>,
397 files: Vec<TestFile>,
399 imports: BTreeMap<PathBuf, BTreeSet<PathBuf>>,
402}
403
404enum AgentExit {
406 Succeeded,
408 Failed(String),
410 TimedOut,
412}
413
414struct RunDirs(Vec<PathBuf>);
416
417impl Drop for RunDirs {
418 fn drop(&mut self) {
419 for dir in &self.0 {
420 let _ = fs::remove_dir(dir);
421 }
422 }
423}
424
425impl PropertiesArgs {
426 pub fn run(mut self) -> Result<()> {
427 ensure!(self.seed.len() >= 2, "at least two --seed values are required");
428 ensure!(
429 self.seed.iter().collect::<HashSet<_>>().len() == self.seed.len(),
430 "--seed values must be distinct"
431 );
432 ensure!(self.agents > 0, "--agents must be greater than zero");
433 let quorum = self.quorum.unwrap_or(self.agents.min(DEFAULT_QUORUM));
434 ensure!((1..=self.agents).contains(&quorum), "--quorum must be between 1 and --agents");
435 ensure!(self.focus.len() <= self.agents, "--focus can be given at most --agents times");
436 ensure!(
437 self.target.iter().all(|path| workspace::is_safe_relative_path(path)),
438 "--target paths must be project-relative paths"
439 );
440 let paths = &mut self.build.project_paths;
443 if paths.config_path.is_none() {
444 let root = paths.try_project_root()?;
445 paths.root = Some(root.canonicalize().wrap_err("failed to resolve project root")?);
446 }
447 let config = self.load_config()?;
448 let generated_tests =
449 workspace::relative_to_root(&config.root, &config.test).join("generated");
450 workspace::ensure_safe_relative_path(&generated_tests, "generated test", &config.test)?;
451 let generated_dir = config.root.join(&generated_tests);
454 ensure_inside(&config.root, &generated_dir, &generated_tests)?;
455 fs::create_dir_all(&generated_dir)
456 .wrap_err_with(|| format!("failed to create {}", generated_dir.display()))?;
457 let properties_cache = config.cache_path.join("properties");
458 fs::create_dir_all(&properties_cache)?;
459 let mut run = 1;
460 let (run_dirs, cache_root) = loop {
461 let mut dirs = RunDirs(Vec::new());
462 for id in 1..=self.agents {
463 let dir = generated_dir.join(format!("run-{run}-agent-{id}"));
464 match fs::create_dir(&dir) {
465 Ok(()) => dirs.0.push(dir),
466 Err(error) if error.kind() == io::ErrorKind::AlreadyExists => break,
467 Err(error) => {
468 return Err(error)
469 .wrap_err_with(|| format!("failed to create {}", dir.display()));
470 }
471 }
472 }
473 if dirs.0.len() == self.agents {
474 let cache = properties_cache.join(format!("run-{run}"));
477 match fs::create_dir(&cache) {
478 Ok(()) => break (dirs, cache),
479 Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
480 Err(error) => {
481 return Err(error)
482 .wrap_err_with(|| format!("failed to create {}", cache.display()));
483 }
484 }
485 }
486 run += 1;
487 };
488 let verifier_command = self
489 .verifier_command
490 .as_ref()
491 .map(|command| command.canonicalize().wrap_err("failed to resolve verifier command"))
492 .transpose()?;
493 let mut sandbox = Sandbox::new(
496 &config,
497 vec![config.root.join(&generated_tests)],
498 self.allow_ffi,
499 self.allow_network,
500 )?;
501 sandbox.end_on_interrupt(run_dirs.0.clone())?;
502 if let Some(errors) = sandbox.build()? {
503 sh_warn!("the project does not build, so every test run builds it: {errors}")?;
504 }
505 let campaign = Campaign {
506 args: &self,
507 config: &config,
508 sandbox,
509 verifier_command,
510 generated_tests,
511 run,
512 cache_root,
513 quorum,
514 };
515 campaign.run()
516 }
517}
518
519type Fingerprint = BTreeMap<PathBuf, (u64, Option<SystemTime>)>;
521
522struct Campaign<'a> {
524 args: &'a PropertiesArgs,
525 config: &'a Config,
526 sandbox: Sandbox<'a>,
527 verifier_command: Option<PathBuf>,
529 generated_tests: PathBuf,
531 run: usize,
533 cache_root: PathBuf,
534 quorum: usize,
535}
536
537struct AgentInputs {
539 agent_command: PathBuf,
540 targets: Vec<PathBuf>,
542 context: Vec<PathBuf>,
544}
545
546impl Campaign<'_> {
547 fn run(&self) -> Result<()> {
548 let agent_command =
549 self.args.agent_command.canonicalize().wrap_err("failed to resolve agent command")?;
550 let context = self
551 .args
552 .context
553 .iter()
554 .map(|path| {
555 canonicalize_path(path)
556 .wrap_err_with(|| format!("failed to resolve context {}", path.display()))
557 })
558 .collect::<Result<Vec<_>>>()?;
559 let targets = if self.args.target.is_empty() {
560 source_files_iter(&self.config.src, MultiCompilerLanguage::FILE_EXTENSIONS)
561 .map(|path| workspace::relative_to_root(&self.config.root, &path))
562 .collect()
563 } else {
564 self.args.target.clone()
565 };
566 let inputs = AgentInputs { agent_command, targets, context };
567 let fingerprint = project_fingerprint(self.config, &self.sandbox.dependencies());
570 let results = thread::scope(|scope| {
573 let inputs = &inputs;
574 let agents = (1..=self.args.agents)
575 .map(|id| scope.spawn(move || self.run_agent(id, inputs)))
576 .collect::<Vec<_>>();
577 agents
578 .into_iter()
579 .map(|agent| agent.join().expect("agent thread panicked"))
580 .collect::<Vec<_>>()
581 });
582 let mut sessions = Vec::new();
583 let mut errors = Vec::new();
584 for (id, result) in (1..).zip(results) {
585 match result {
586 Ok(session) => {
587 if let AgentExit::Failed(error) = &session.exit {
588 errors.push(format!("agent {id}: {error}"));
589 }
590 sessions.push(session);
591 }
592 Err(error) => errors.push(format!("agent {id}: {error:#}")),
593 }
594 }
595 self.report(&sessions, &inputs, &fingerprint)?;
596 ensure!(errors.is_empty(), "{}", errors.join("\n"));
597 Ok(())
598 }
599
600 fn report(
603 &self,
604 sessions: &[Session],
605 inputs: &AgentInputs,
606 fingerprint: &Fingerprint,
607 ) -> Result<()> {
608 let failures = sessions
609 .iter()
610 .flat_map(|session| &session.possible_bugs)
611 .map(|failure| AgentFailure {
612 agent: failure.agent,
613 failure: failure.failure.clone(),
614 file: failure.file.clone(),
615 })
616 .collect::<Vec<_>>();
617 let (blocked, failures) = failures
620 .into_iter()
621 .partition::<Vec<_>, _>(|failure| BLOCKED.is_match(&failure.failure.reason));
622 let (mut possible_bugs, below_quorum) = group_failures(failures)
623 .into_iter()
624 .partition::<Vec<_>, _>(|group| group.agents.len() >= self.quorum);
625 if let Some(command) = &self.verifier_command {
626 self.verify_groups(command, &mut possible_bugs, sessions, inputs);
627 }
628 let after = project_fingerprint(self.config, &self.sandbox.dependencies());
629 let tainted = after
630 .iter()
631 .filter(|(path, metadata)| fingerprint.get(*path) != Some(*metadata))
632 .map(|(path, _)| path)
633 .chain(fingerprint.keys().filter(|path| !after.contains_key(*path)))
634 .cloned()
635 .collect::<Vec<_>>();
636 let print_group = |label: &str, group: &BugGroup| -> Result<()> {
637 let lead = &group.failures[0].failure;
638 let summary = lead.claim.as_deref().unwrap_or(&lead.test);
639 sh_println!("{label} ({}/{} agents): {summary}", group.agents.len(), self.args.agents)?;
640 for failure in &group.failures {
641 sh_println!("{failure}")?;
642 }
643 if let Some(verification) = &group.verification {
644 sh_println!("{verification}")?;
645 }
646 Ok(())
647 };
648 for group in &possible_bugs {
649 let label = match group.verification.as_ref().map(|v| v.judgment.verdict) {
650 None => "possible bug",
651 Some(Verdict::Confirmed) => "confirmed",
652 Some(Verdict::NeedsValidation) => "needs validation",
653 Some(Verdict::Rejected) => "rejected",
654 };
655 print_group(label, group)?;
656 }
657 let report_path = self.cache_root.join("report.json");
658 let report_display = report_path.strip_prefix(&self.config.root).unwrap_or(&report_path);
659 if !below_quorum.is_empty() {
662 sh_println!(
663 "{} possible bug(s) below the quorum of {}:",
664 below_quorum.len(),
665 self.quorum
666 )?;
667 for group in &below_quorum {
668 let label = if group.agents.len() == 1 { "single-agent" } else { "below quorum" };
669 print_group(label, group)?;
670 }
671 }
672 let existing = collect_tests(&self.config.root, &self.generated_tests, 0, &mut vec![])
679 .unwrap_or_default();
680 let mut seen = existing
681 .iter()
682 .map(|file| normalize_test_content(&file.content))
683 .collect::<HashSet<_>>();
684 let mut held_back = Vec::new();
685 let mut kept = Vec::new();
686 let mut duplicates = Vec::new();
687 for session in sessions.iter().filter(|_| tainted.is_empty()) {
688 let conflict = session.kept.iter().find_map(|file| {
689 let path = self.config.root.join(&file.path);
690 if let Err(error) = ensure_inside(&self.config.root, &path, &file.path) {
692 return Some(format!("{error:#}"));
693 }
694 path.exists().then(|| format!("{} exists already", file.path.display()))
695 });
696 if let Some(reason) = conflict {
697 let paths = session.kept.iter().map(|file| file.path.clone()).collect();
698 held_back.push(Rejected { agent: session.id, paths, reason });
699 continue;
700 }
701 let files = &session.kept;
702 let imports = |file: &TestFile, other: &TestFile| {
703 session.imports.get(&file.path).is_some_and(|i| i.contains(&other.path))
704 };
705 let mut write = files
706 .iter()
707 .map(|file| seen.insert(normalize_test_content(&file.content)))
708 .collect::<Vec<_>>();
709 while let Some(index) = (0..files.len()).find(|&i| {
710 !write[i]
711 && files.iter().zip(&write).any(|(other, written)| {
712 *written && (imports(other, &files[i]) || imports(&files[i], other))
713 })
714 }) {
715 write[index] = true;
716 }
717 for (file, write) in files.iter().zip(write) {
718 if write {
719 kept.push(file);
720 } else {
721 duplicates.push(&file.path);
722 }
723 }
724 }
725 let rejected = sessions
726 .iter()
727 .flat_map(|session| &session.rejected)
728 .chain(&held_back)
729 .collect::<Vec<_>>();
730 if !rejected.is_empty() {
731 sh_println!(
732 "{} generated test file(s) did not compile or were not allowed: see {}",
733 rejected.iter().map(|rejected| rejected.paths.len()).sum::<usize>(),
734 report_display.display()
735 )?;
736 }
737 if !blocked.is_empty() {
738 sh_warn!(
739 "{} failing test(s) used something that Forge blocks (ffi, environment, network, or inline config), so they are not possible bugs: see {}",
740 blocked.len(),
741 report_display.display()
742 )?;
743 }
744 let timed_out = sessions
745 .iter()
746 .filter(|session| matches!(session.exit, AgentExit::TimedOut))
747 .map(|session| session.id)
748 .collect::<Vec<_>>();
749 if !timed_out.is_empty() {
750 sh_warn!(
751 "{} agent(s) reached the timeout, so Forge ended them and ran the tests that they wrote",
752 timed_out.len()
753 )?;
754 }
755 let usage = sessions
756 .iter()
757 .filter_map(|session| {
758 Some(serde_json::json!({ "agent": session.id, "usage": session.usage? }))
759 })
760 .collect::<Vec<_>>();
761 if let Some(total) = sessions.iter().filter_map(|session| session.usage).reduce(Usage::add)
762 {
763 sh_println!(
764 "agent usage: {} input tokens, {} output tokens, {:.2} USD",
765 total.input_tokens,
766 total.output_tokens,
767 total.cost_usd
768 )?;
769 }
770 if let Some(total) = possible_bugs
771 .iter()
772 .filter_map(|group| group.verification.as_ref()?.usage)
773 .reduce(Usage::add)
774 {
775 sh_println!(
776 "verifier usage: {} input tokens, {} output tokens, {:.2} USD",
777 total.input_tokens,
778 total.output_tokens,
779 total.cost_usd
780 )?;
781 }
782
783 fs::create_dir_all(&self.cache_root)?;
784 let report = serde_json::json!({
785 "agents": self.args.agents,
786 "quorum": self.quorum,
787 "possible_bugs": possible_bugs,
788 "below_quorum": below_quorum,
789 "blocked": blocked,
790 "rejected": rejected,
791 "timed_out": timed_out,
792 "tainted": tainted,
793 "kept": kept.iter().map(|file| &file.path).collect::<Vec<_>>(),
794 "duplicates": duplicates,
795 "usage": usage,
796 });
797 let report_bytes = serde_json::to_vec_pretty(&report)?;
798 fs::write(&report_path, &report_bytes)?;
799 let properties_root = self.cache_root.parent().expect("campaign cache has a parent");
802 let mut latest = tempfile::NamedTempFile::new_in(properties_root)?;
803 latest.write_all(&report_bytes)?;
804 latest.persist(properties_root.join("report.json")).map_err(|error| error.error)?;
805
806 ensure!(
809 tainted.is_empty(),
810 "{} project file(s) changed while the agents ran, so Forge kept no generated test: see {}",
811 tainted.len(),
812 report_display.display()
813 );
814 if kept.is_empty() {
815 sh_println!("no generated test file was kept")?;
816 return Ok(());
817 }
818 for file in &kept {
819 let path = self.config.root.join(&file.path);
820 fs::create_dir_all(path.parent().expect("test path has a parent"))?;
821 fs::OpenOptions::new()
822 .write(true)
823 .create_new(true)
824 .open(&path)
825 .and_then(|mut target| target.write_all(file.content.as_bytes()))
826 .wrap_err_with(|| format!("failed to write {}", file.path.display()))?;
827 }
828 sh_println!("added {} generated test file(s):", kept.len())?;
829 for file in &kept {
830 sh_println!(" {}", file.path.display())?;
831 }
832 if !duplicates.is_empty() {
833 sh_println!(
834 "skipped {} generated test file(s) that repeat a file of another agent or the project",
835 duplicates.len()
836 )?;
837 }
838 Ok(())
839 }
840
841 fn run_agent(&self, id: usize, inputs: &AgentInputs) -> Result<Session> {
843 let args = self.args;
844 let cache = self.cache_root.join(format!("agent-{id}"));
845 fs::create_dir_all(&cache)?;
846 let workspace = self.sandbox.workspace(&[])?;
847 let out = self.generated_tests.join(format!("run-{}-agent-{id}", self.run));
848 let out_dir = workspace.path().join(&out);
849 ensure_inside(workspace.path(), &out_dir, &out)?;
850 fs::create_dir_all(&out_dir)?;
851 let focuses = if args.no_focus || (args.focus.is_empty() && args.agents == 1) {
852 vec![]
853 } else if args.focus.is_empty() {
854 DEFAULT_FOCUSES.to_vec()
855 } else {
856 args.focus.iter().map(String::as_str).collect()
857 };
858 let focus = (!focuses.is_empty()).then(|| focuses[(id - 1) % focuses.len()]);
859 let distinct = args.agents > 1 && args.agents <= focuses.len();
861 let prompt = cache.join("prompt.md");
862 fs::write(
863 &prompt,
864 session_prompt(&inputs.targets, &inputs.context, &out_dir, focus, distinct),
865 )?;
866 let usage_path = cache.join("usage.json");
867 let log_path = cache.join("agent.log");
868 let log = fs::File::create(&log_path)?;
869 let seed: U256 =
871 keccak256([args.seed[0].to_be_bytes::<32>(), U256::from(id).to_be_bytes()].concat())
872 .into();
873 let lines = |paths: &[PathBuf]| {
874 paths.iter().map(|path| path.display().to_string()).collect::<Vec<_>>().join("\n")
875 };
876
877 let mut command = Command::new(&inputs.agent_command);
878 command
879 .args(&args.agent_arg)
880 .current_dir(workspace.path())
881 .env("FORGE_PROPERTIES_PROMPT", &prompt)
882 .env("FORGE_PROPERTIES_TARGETS", lines(&inputs.targets))
883 .env("FORGE_PROPERTIES_CONTEXT", lines(&inputs.context))
884 .env("FORGE_PROPERTIES_OUT", &out_dir)
885 .env("FORGE_PROPERTIES_SEED", format!("{seed:#x}"))
886 .env("FORGE_PROPERTIES_FOCUS", focus.unwrap_or_default())
887 .env("FORGE_PROPERTIES_USAGE", &usage_path)
888 .stdin(Stdio::null())
889 .stdout(log.try_clone()?)
890 .stderr(log);
891 let status = run_session(&mut command, args.timeout, "agent")?;
892 let log_display = log_path.strip_prefix(&self.config.root).unwrap_or(&log_path);
893 let exit = match status {
894 None => AgentExit::TimedOut,
895 Some(status) if status.success() => AgentExit::Succeeded,
896 Some(status) => AgentExit::Failed(format!(
897 "agent command failed ({status}): see {}",
898 log_display.display()
899 )),
900 };
901 let usage = read_usage(&usage_path, &format!("agent {id}"))?;
902 let mut session = Session {
903 id,
904 exit,
905 usage,
906 possible_bugs: vec![],
907 rejected: vec![],
908 kept: vec![],
909 files: vec![],
910 imports: BTreeMap::new(),
911 };
912 session.files = collect_tests(workspace.path(), &out, id, &mut session.rejected)?;
913 let files = &session.files;
914 let copy = |path: &Path| cache.join("tests").join(path.strip_prefix(&out).unwrap_or(path));
916 for file in files {
917 let path = copy(&file.path);
918 fs::create_dir_all(path.parent().expect("test path has a parent"))?;
919 fs::write(&path, &file.content)?;
920 }
921 if files.is_empty() {
922 return Ok(session);
923 }
924
925 let workspace = self.sandbox.workspace(files)?;
926 let imports = collected_imports(self.config, workspace.path(), files)?;
927 let run = match self.sandbox.build_errors(workspace.path())? {
928 None => self.run_tests(workspace.path(), files)?,
929 Some(_) => {
936 let no_imports = BTreeSet::new();
937 let (roots, others) = files.iter().partition::<Vec<_>, _>(|file| {
938 !imports.values().any(|imported| imported.contains(&file.path))
939 });
940 let mut grouped = HashSet::new();
941 let mut ran = HashSet::new();
944 let mut run = TestRun::default();
945 for root in roots.into_iter().chain(others) {
946 if grouped.contains(&root.path) {
947 continue;
948 }
949 let imported = imports.get(&root.path).unwrap_or(&no_imports);
950 let group = files
951 .iter()
952 .filter(|file| file.path == root.path || imported.contains(&file.path))
953 .cloned()
954 .collect::<Vec<_>>();
955 let workspace = self.sandbox.workspace(&group)?;
956 if let Some(errors) = self.sandbox.build_errors(workspace.path())? {
957 grouped.insert(root.path.clone());
958 let paths = vec![root.path.clone()];
959 session.rejected.push(Rejected { agent: id, paths, reason: errors });
960 continue;
961 }
962 grouped.extend(group.iter().map(|file| file.path.clone()));
963 let group_run = self.run_tests(workspace.path(), &group)?;
964 run.possible_bugs.extend(
965 group_run.possible_bugs.into_iter().filter(|bug| !ran.contains(&bug.path)),
966 );
967 run.passing.extend(
968 group_run
969 .passing
970 .into_iter()
971 .filter(|test| !ran.contains(&test_path(test))),
972 );
973 run.failing_files.extend(group_run.failing_files);
974 run.errors.extend(group_run.errors);
975 ran.extend(group.into_iter().map(|file| file.path));
976 }
977 run
978 }
979 };
980 if !run.errors.is_empty() {
981 let paths = files.iter().map(|file| file.path.clone()).collect();
982 session.rejected.push(Rejected { agent: id, paths, reason: run.errors.join("; ") });
983 }
984 let kept = kept_files(files, &imports, &run)?;
985 session.kept = files.iter().filter(|file| kept.contains(&file.path)).cloned().collect();
986 session.imports = imports;
987 let root = &self.config.root;
988 session.possible_bugs = run
989 .possible_bugs
990 .into_iter()
991 .map(|failure| {
992 let file = copy(&failure.path);
993 let file = file.strip_prefix(root).map(Path::to_path_buf).unwrap_or(file);
994 AgentFailure { agent: id, failure, file }
995 })
996 .collect();
997 Ok(session)
998 }
999
1000 fn run_tests(&self, workspace: &Path, files: &[TestFile]) -> Result<TestRun> {
1002 let mut run = TestRun::default();
1003 let mut outcomes = BTreeMap::<(String, String), (usize, Vec<String>)>::new();
1005 for seed in &self.args.seed {
1006 let mut command = self.test_command(workspace, files, seed);
1007 let Some(output) = self.sandbox.output(&mut command)? else {
1008 run.errors
1009 .push(format!("tests reached the timeout of {RUN_TIMEOUT:?} on seed {seed}"));
1010 continue;
1011 };
1012 if !output.status.success() && output.stdout.is_empty() {
1013 run.errors.push(format!("tests failed on seed {seed}: {}", stderr(&output)));
1014 continue;
1015 }
1016 for (test, status, reason) in test_results(&output)? {
1017 let (passes, failures) = outcomes.entry((test.contract, test.test)).or_default();
1018 match status {
1019 TestStatus::Success => *passes += 1,
1020 TestStatus::Failure => {
1021 let reason = reason
1022 .filter(|reason| !reason.is_empty())
1023 .unwrap_or_else(|| "the test failed without a reason".to_string());
1024 failures.push(format!("seed {seed}: {reason}"));
1025 }
1026 TestStatus::Skipped => {}
1027 }
1028 }
1029 }
1030 let seeds = self.args.seed.len();
1031 for ((contract, test), (passes, failures)) in outcomes {
1032 let test = RerunFailure { contract, test };
1033 if passes == seeds {
1034 run.passing.push(test);
1035 continue;
1036 }
1037 let path = test_path(&test);
1038 run.failing_files.insert(path.clone());
1039 if failures.len() == seeds {
1040 let contract = test.contract.rsplit_once(':').map_or("", |(_, name)| name);
1041 let function =
1042 test.test.split_once('(').map_or(test.test.as_str(), |(name, _)| name);
1043 let claim = files
1044 .iter()
1045 .find(|file| file.path == path)
1046 .and_then(|file| test_claim(&file.content, contract, function));
1047 run.possible_bugs.push(Failure {
1048 test: test_display_name(&test),
1049 path,
1050 reason: failures.join("; "),
1051 claim,
1052 });
1053 }
1054 }
1055 run.possible_bugs
1057 .sort_by(|a, b| (a.claim.is_none(), &a.test).cmp(&(b.claim.is_none(), &b.test)));
1058 Ok(run)
1059 }
1060
1061 fn test_command(&self, workspace: &Path, files: &[TestFile], seed: &U256) -> Command {
1065 let seed_dir = seed_dir(self.config, workspace, seed);
1066 let mut command = self.sandbox.test_command(workspace, files);
1067 command
1068 .env("FOUNDRY_FUZZ_SEED", format!("{seed:#x}"))
1069 .env("FOUNDRY_FUZZ_FAILURE_PERSIST_DIR", seed_dir.join("fuzz"))
1070 .env("FOUNDRY_INVARIANT_FAILURE_PERSIST_DIR", seed_dir.join("invariant"))
1071 .env("FOUNDRY_TEST_FAILURES_FILE", seed_dir.join("test-failures"));
1073 command
1074 }
1075
1076 fn verify_groups(
1079 &self,
1080 command: &Path,
1081 groups: &mut [BugGroup],
1082 sessions: &[Session],
1083 inputs: &AgentInputs,
1084 ) {
1085 let next = AtomicUsize::new(0);
1086 let verifications = {
1087 let groups = &*groups;
1088 thread::scope(|scope| {
1089 let workers = (0..self.args.agents.min(groups.len()))
1090 .map(|_| {
1091 scope.spawn(|| {
1092 let mut done = Vec::new();
1093 loop {
1094 let index = next.fetch_add(1, Ordering::Relaxed);
1095 let Some(group) = groups.get(index) else { break };
1096 let verification = self
1097 .verify(command, index + 1, group, sessions, inputs)
1098 .unwrap_or_else(|error| {
1099 Verification::failed(format!("{error:#}"))
1100 });
1101 done.push((index, verification));
1102 }
1103 done
1104 })
1105 })
1106 .collect::<Vec<_>>();
1107 workers
1108 .into_iter()
1109 .flat_map(|worker| worker.join().expect("verifier thread panicked"))
1110 .collect::<Vec<_>>()
1111 })
1112 };
1113 for (index, verification) in verifications {
1114 groups[index].verification = Some(verification);
1115 }
1116 groups.sort_by_key(|group| group.verification.as_ref().map(|v| v.judgment.verdict));
1118 }
1119
1120 fn verify(
1123 &self,
1124 command: &Path,
1125 id: usize,
1126 group: &BugGroup,
1127 sessions: &[Session],
1128 inputs: &AgentInputs,
1129 ) -> Result<Verification> {
1130 let cache = self.cache_root.join(format!("verify-{id}"));
1131 fs::create_dir_all(&cache)?;
1132 let workspace = self.sandbox.workspace(&group_files(sessions, group))?;
1133 let prompt_path = cache.join("prompt.md");
1134 let group_path = cache.join("group.json");
1135 let verdict_path = cache.join("verdict.json");
1136 let usage_path = cache.join("usage.json");
1137 let log_path = cache.join("verifier.log");
1138 fs::write(&prompt_path, verify_prompt(group, &inputs.targets, &group_path, &verdict_path))?;
1139 let failures = group.failures.iter().map(|failure| {
1140 serde_json::json!({
1141 "agent": failure.agent,
1142 "test": failure.failure.test,
1143 "path": failure.failure.path,
1144 "claim": failure.failure.claim,
1145 "failure": failure.failure.reason,
1146 })
1147 });
1148 let group_json = serde_json::json!({
1149 "targets": inputs.targets,
1150 "agents": group.agents,
1151 "failures": failures.collect::<Vec<_>>(),
1152 });
1153 fs::write(&group_path, serde_json::to_vec_pretty(&group_json)?)?;
1154 let log = fs::File::create(&log_path)?;
1155
1156 let mut command = Command::new(command);
1157 command
1158 .args(&self.args.verifier_arg)
1159 .current_dir(workspace.path())
1160 .env("FORGE_PROPERTIES_VERIFY_PROMPT", &prompt_path)
1161 .env("FORGE_PROPERTIES_GROUP", &group_path)
1162 .env("FORGE_PROPERTIES_VERDICT", &verdict_path)
1163 .env("FORGE_PROPERTIES_USAGE", &usage_path)
1164 .stdin(Stdio::null())
1165 .stdout(log.try_clone()?)
1166 .stderr(log);
1167 let status = run_session(&mut command, self.args.timeout, "verifier")?;
1168 let usage = read_usage(&usage_path, &format!("verifier {id}"))?;
1169
1170 let mut errors = Vec::new();
1171 match status {
1172 None => errors.push("the verifier reached the timeout".to_string()),
1173 Some(status) if !status.success() => {
1174 errors.push(format!("verifier command failed ({status})"))
1175 }
1176 Some(_) => {}
1177 }
1178 let judgment = match fs::read(&verdict_path) {
1179 Ok(verdict) => match serde_json::from_slice::<Judgment>(&verdict) {
1180 Ok(judgment)
1182 if judgment.verdict == Verdict::Rejected && judgment.evidence.is_empty() =>
1183 {
1184 errors.push("the verifier rejected the group without evidence".to_string());
1185 Judgment { verdict: Verdict::NeedsValidation, ..judgment }
1186 }
1187 Ok(judgment) => judgment,
1188 Err(error) => {
1189 errors.push(format!("the verifier wrote an invalid verdict: {error}"));
1190 Judgment::default()
1191 }
1192 },
1193 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1194 errors.push("the verifier wrote no verdict".to_string());
1195 Judgment::default()
1196 }
1197 Err(error) => return Err(error).wrap_err("failed to read verifier verdict"),
1198 };
1199 let log_display = log_path.strip_prefix(&self.config.root).unwrap_or(&log_path);
1200 let error = (!errors.is_empty())
1201 .then(|| format!("{}: see {}", errors.join("; "), log_display.display()));
1202 Ok(Verification { judgment, error, usage })
1203 }
1204}
1205
1206fn group_failures(failures: Vec<AgentFailure>) -> Vec<BugGroup> {
1209 let claims =
1210 failures.iter().map(|failure| failure.failure.claim.as_deref()).collect::<Vec<_>>();
1211 let group_of = matcher::root_cause_groups(&claims);
1212 let mut groups = BTreeMap::<usize, Vec<AgentFailure>>::new();
1213 for (failure, group) in failures.into_iter().zip(group_of) {
1214 groups.entry(group).or_default().push(failure);
1215 }
1216 let mut groups = groups
1217 .into_values()
1218 .map(|mut failures| {
1219 failures.sort_by(|a, b| {
1221 let key = |failure: &AgentFailure| {
1222 (failure.failure.claim.is_none(), failure.agent, failure.failure.test.clone())
1223 };
1224 key(a).cmp(&key(b))
1225 });
1226 let agents = failures.iter().map(|failure| failure.agent).collect::<BTreeSet<_>>();
1227 BugGroup { agents: agents.into_iter().collect(), failures, verification: None }
1228 })
1229 .collect::<Vec<_>>();
1230 groups.sort_by(|a, b| {
1231 let key = |group: &BugGroup| {
1232 let lead = &group.failures[0].failure;
1233 (lead.claim.is_none(), Reverse(group.agents.len()), lead.test.clone())
1234 };
1235 key(a).cmp(&key(b))
1236 });
1237 groups
1238}
1239
1240fn session_prompt(
1244 targets: &[PathBuf],
1245 context: &[PathBuf],
1246 out: &Path,
1247 focus: Option<&str>,
1248 distinct: bool,
1249) -> String {
1250 let list = |paths: &[PathBuf]| {
1251 paths.iter().map(|path| format!("- {}\n", path.display())).collect::<String>()
1252 };
1253 let mut prompt = format!(
1254 "Find bugs in these targets of the Foundry project in the current directory:\n{}",
1255 list(targets)
1256 );
1257 if !context.is_empty() {
1258 prompt.push_str(&format!(
1259 "\nThese context files state intended behavior:\n{}",
1260 list(context)
1261 ));
1262 }
1263 prompt.push_str(&format!(
1264 "\nProve each bug with a Foundry test in {} that fails because of the bug and encodes the \
1265 intended behavior. The code under test can be wrong: the intended behavior is what its \
1266 documentation, its interfaces, and the standards it claims state. Name each test after \
1267 the bug. Right above each test, write exactly one comment line in this form:\n\n\
1268 // @bug <Contract>.<function> <file>:<line> — <claim>\n\n\
1269 <Contract>.<function> is the target function that has the bug, <file>:<line> is the \
1270 target source line that breaks the intended behavior, and <claim> is one sentence. Use \
1271 `//`, not `///`: the compiler rejects `@bug` as a NatSpec tag. You may run `forge test` \
1272 freely.\n",
1273 out.display()
1274 ));
1275 if let Some(focus) = focus {
1276 let others = if distinct { " (the other agents have other focuses)" } else { "" };
1277 prompt.push_str(&format!(
1278 "\nYour focus{others}: {focus} Trace a small number of concrete high-signal paths in \
1279 this area first; then widen if budget is left.\n"
1280 ));
1281 }
1282 prompt
1283}
1284
1285fn collect_tests(
1289 workspace: &Path,
1290 out: &Path,
1291 agent: usize,
1292 rejected: &mut Vec<Rejected>,
1293) -> Result<Vec<TestFile>> {
1294 let out_dir = workspace.join(out);
1295 ensure_inside(workspace, &out_dir, out)?;
1296 match fs::symlink_metadata(&out_dir) {
1297 Ok(metadata) => ensure!(
1298 metadata.is_dir(),
1299 "the generated test directory {} is not a directory",
1300 out.display()
1301 ),
1302 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(vec![]),
1304 Err(error) => {
1305 return Err(error).wrap_err_with(|| format!("failed to read {}", out.display()));
1306 }
1307 }
1308 let mut files = Vec::new();
1309 let mut pending = vec![out.to_path_buf()];
1310 while let Some(dir) = pending.pop() {
1311 for entry in fs::read_dir(workspace.join(&dir))? {
1312 let entry = entry?;
1313 let path = dir.join(entry.file_name());
1314 let file_type = entry.file_type()?;
1315 if file_type.is_dir() {
1316 pending.push(path);
1317 continue;
1318 }
1319 if !file_type.is_file() || path.extension().is_none_or(|extension| extension != "sol") {
1320 continue;
1321 }
1322 let glob = ['*', '?', '[', ']', '{', '}', ',', '\\'];
1323 if path.iter().any(|component| component.to_string_lossy().contains(glob)) {
1324 let reason = "the path has a glob character".to_string();
1325 rejected.push(Rejected { agent, paths: vec![path], reason });
1326 continue;
1327 }
1328 match fs::read_to_string(entry.path()) {
1329 Ok(content) => files.push(TestFile { path, content }),
1330 Err(error) => {
1331 let reason = error.to_string();
1332 rejected.push(Rejected { agent, paths: vec![path], reason });
1333 }
1334 }
1335 }
1336 }
1337 files.sort_by(|a, b| a.path.cmp(&b.path));
1338 Ok(files)
1339}
1340
1341fn normalize_test_content(source: &str) -> Vec<&str> {
1344 Cursor::new(source)
1345 .with_position()
1346 .filter(|(_, token)| !token.kind.is_trivial())
1347 .map(|(pos, token)| &source[pos..pos + token.len as usize])
1348 .collect()
1349}
1350
1351fn test_claim(source: &str, contract: &str, function: &str) -> Option<String> {
1356 let code = COMMENT.replace_all(source, |comment: ®ex::Captures<'_>| {
1358 comment[0].bytes().map(|byte| if byte == b'\n' { '\n' } else { ' ' }).collect::<String>()
1359 });
1360 let mut contracts = CONTRACT.captures_iter(&code);
1361 let start = contracts.find(|captures| &captures[1] == contract)?.get(0)?.end();
1362 let end = contracts.next().map_or(code.len(), |captures| captures.get(0).unwrap().start());
1363 let declaration = Regex::new(&format!(r"\bfunction\s+{}\s*\(", regex::escape(function)))
1364 .ok()?
1365 .find(&code[start..end])?;
1366 let line_start = source[..start + declaration.start()].rfind('\n').map_or(0, |i| i + 1);
1367 let mut lines = source[..line_start]
1368 .lines()
1369 .rev()
1370 .map(str::trim)
1371 .take_while(|line| {
1372 line.starts_with("//") || line.starts_with('*') || line.starts_with("/*")
1373 })
1374 .filter(|line| !line.contains("forge-config:"))
1375 .map(|line| line.trim_start_matches(['/', '*']).trim_end_matches(['*', '/']).trim())
1376 .map(|line| {
1377 match line.strip_prefix('@').map(|tagged| tagged.split_once(char::is_whitespace)) {
1378 Some(Some((tag, text))) if !matches!(tag, "bug" | "custom:bug") => text,
1382 Some(None) => "",
1383 _ => line,
1384 }
1385 })
1386 .filter(|line| !line.is_empty())
1387 .collect::<Vec<_>>();
1388 lines.reverse();
1389 if let Some(tag) = lines.iter().position(|line| {
1391 line.split_whitespace().next().is_some_and(|tag| matches!(tag, "@bug" | "@custom:bug"))
1392 }) {
1393 lines.drain(..tag);
1394 }
1395 (!lines.is_empty()).then(|| lines.join(" "))
1396}
1397
1398fn collected_imports(
1400 config: &Config,
1401 workspace: &Path,
1402 files: &[TestFile],
1403) -> Result<BTreeMap<PathBuf, BTreeSet<PathBuf>>> {
1404 let rebased = workspace::rebase_config_paths(config, workspace);
1405 let graph =
1406 Graph::<MultiCompilerParser>::resolve(&rebased.project_paths::<MultiCompilerLanguage>())
1407 .wrap_err("failed to resolve generated test imports")?;
1408 let paths = files.iter().map(|file| file.path.clone()).collect::<HashSet<_>>();
1409 let mut imports = BTreeMap::new();
1410 for (index, node) in graph.nodes.iter().enumerate() {
1411 let path = workspace::relative_to_root(workspace, node.path());
1412 if paths.contains(&path) {
1413 let imported = graph
1414 .all_imported_nodes(index)
1415 .map(|index| workspace::relative_to_root(workspace, graph.node(index).path()))
1416 .filter(|path| paths.contains(path))
1417 .collect();
1418 imports.insert(path, imported);
1419 }
1420 }
1421 Ok(imports)
1422}
1423
1424fn kept_files(
1427 files: &[TestFile],
1428 imports: &BTreeMap<PathBuf, BTreeSet<PathBuf>>,
1429 run: &TestRun,
1430) -> Result<HashSet<PathBuf>> {
1431 let passing = files
1432 .iter()
1433 .map(|file| &file.path)
1434 .filter(|path| {
1435 !run.failing_files.contains(*path)
1436 && run.passing.iter().any(|test| test_path(test) == **path)
1437 })
1438 .collect::<Vec<_>>();
1439 if passing.is_empty() {
1440 return Ok(HashSet::new());
1441 }
1442 if passing.len() == files.len() {
1443 return Ok(passing.into_iter().cloned().collect());
1444 }
1445 let mut kept = HashSet::new();
1446 for path in passing {
1447 let imported = imports.get(path).ok_or_else(|| {
1448 eyre!("passing test file {} is absent from the import graph", path.display())
1449 })?;
1450 if imported.iter().all(|path| !run.failing_files.contains(path)) {
1451 kept.insert(path.clone());
1452 kept.extend(imported.iter().cloned());
1453 }
1454 }
1455 Ok(kept)
1456}
1457
1458fn project_fingerprint(config: &Config, dependencies: &[PathBuf]) -> Fingerprint {
1464 let mut files = BTreeMap::new();
1465 let mut read = HashSet::new();
1466 let mut pending = vec![(config.root.clone(), false)];
1468 while let Some((dir, shared)) = pending.pop() {
1469 if !fs::canonicalize(&dir).is_ok_and(|dir| read.insert(dir)) {
1470 continue;
1471 }
1472 let Ok(entries) = fs::read_dir(&dir) else { continue };
1473 for entry in entries.flatten() {
1474 let path = entry.path();
1475 if entry.file_name().to_string_lossy().starts_with('.')
1476 || path == config.out
1477 || path == config.cache_path
1478 {
1479 continue;
1480 }
1481 let shared = shared
1482 || dependencies.contains(&path)
1483 || (entry.file_type().is_ok_and(|file_type| file_type.is_symlink())
1484 && fs::canonicalize(&path)
1485 .is_ok_and(|target| !target.starts_with(&config.root)));
1486 let Ok(metadata) = (if shared { fs::metadata(&path) } else { entry.metadata() }) else {
1487 continue;
1488 };
1489 if metadata.is_dir() {
1490 pending.push((path, shared));
1491 } else if shared && metadata.is_file() {
1492 let path = path.strip_prefix(&config.root).unwrap_or(&path).to_path_buf();
1494 files.insert(path, (metadata.len(), metadata.modified().ok()));
1495 }
1496 }
1497 }
1498 files
1499}
1500
1501fn test_results(output: &Output) -> Result<Vec<(RerunFailure, TestStatus, Option<String>)>> {
1504 let mut results = Vec::new();
1505 for (contract, test, result) in suite_results(output)? {
1506 let key = |name: &str| RerunFailure { contract: contract.clone(), test: name.into() };
1507 let handler_failures = result
1509 .invariant_handler_failures
1510 .iter()
1511 .chain(result.invariant_failures.iter().filter(|f| f.predicate_name().is_none()))
1512 .map(|failure| {
1513 format!(
1514 "handler {}: {}",
1515 failure.name(),
1516 failure_reason(failure.reason(), failure.counterexample())
1517 )
1518 })
1519 .collect::<Vec<_>>();
1520 if result.invariant_predicate_results.is_empty() {
1521 let reason = result.reason.clone().or_else(|| {
1522 let reasons = result
1523 .invariant_failures
1524 .iter()
1525 .filter(|failure| failure.predicate_name().is_some())
1526 .map(|failure| failure_reason(failure.reason(), failure.counterexample()))
1527 .chain(handler_failures.iter().cloned())
1528 .collect::<Vec<_>>();
1529 (!reasons.is_empty()).then(|| reasons.join("; "))
1530 });
1531 let reason = match (reason, result.counterexample.as_ref()) {
1532 (Some(reason), counterexample) => Some(failure_reason(&reason, counterexample)),
1533 (None, Some(counterexample)) => Some(failure_reason("", Some(counterexample))),
1534 (None, None) => None,
1535 };
1536 let status =
1537 if handler_failures.is_empty() { result.status } else { TestStatus::Failure };
1538 results.push((key(&test), status, reason));
1539 continue;
1540 }
1541 for predicate in &result.invariant_predicate_results {
1542 let failure = result
1543 .invariant_failures
1544 .iter()
1545 .find(|failure| failure.predicate_name() == Some(predicate.name.as_str()));
1546 let reason = predicate.reason.as_deref().or(failure.map(|failure| failure.reason()));
1547 let reason = reason.map(|reason| {
1548 failure_reason(reason, failure.and_then(|failure| failure.counterexample()))
1549 });
1550 results.push((key(&predicate.name), predicate.status, reason));
1551 }
1552 if !handler_failures.is_empty() {
1554 let reason = Some(handler_failures.join("; "));
1555 results.push((key(&test), TestStatus::Failure, reason));
1556 }
1557 }
1558 Ok(results)
1559}
1560
1561fn failure_reason(reason: &str, counterexample: Option<&CounterExample>) -> String {
1563 let Some(counterexample) = counterexample else { return reason.to_string() };
1564 let calls = match counterexample {
1565 CounterExample::Single(call) => vec![call],
1566 CounterExample::Sequence(_, calls) => calls.iter().collect(),
1567 };
1568 let calls = calls.iter().map(|call| call.to_string().trim().to_string()).collect::<Vec<_>>();
1569 let counterexample = format!("counterexample: {}", calls.join(", "));
1570 if reason.is_empty() { counterexample } else { format!("{reason}; {counterexample}") }
1571}
1572
1573fn test_display_name(test: &RerunFailure) -> String {
1575 let contract = test.contract.rsplit_once(':').map_or(test.contract.as_str(), |(_, name)| name);
1576 let name = test.test.split_once('(').map_or(test.test.as_str(), |(name, _)| name);
1577 format!("{contract}::{name}")
1578}
1579
1580fn test_path(test: &RerunFailure) -> PathBuf {
1582 PathBuf::from(test.contract.rsplit_once(':').map_or(test.contract.as_str(), |(path, _)| path))
1583}
1584
1585fn seed_dir(config: &Config, workspace: &Path, seed: &U256) -> PathBuf {
1587 workspace
1588 .join(workspace::relative_to_root(&config.root, &config.cache_path))
1589 .join("properties/seeds")
1590 .join(format!("{seed:#x}"))
1591}
1592
1593fn verify_prompt(
1595 group: &BugGroup,
1596 targets: &[PathBuf],
1597 group_path: &Path,
1598 verdict: &Path,
1599) -> String {
1600 let targets = targets.iter().map(|path| format!("- {}\n", path.display())).collect::<String>();
1601 let members = group
1602 .failures
1603 .iter()
1604 .map(|failure| {
1605 let claim = failure.failure.claim.as_deref().unwrap_or("(none)");
1606 let reason = failure.failure.reason.trim().chars().take(600).collect::<String>();
1607 format!(
1608 "- file: {}\n test: {}\n claim: {claim}\n failure: {reason}\n",
1609 failure.failure.path.display(),
1610 failure.failure.test
1611 )
1612 })
1613 .collect::<Vec<_>>()
1614 .join("\n");
1615 VERIFY_PROMPT
1617 .replace("{targets}", &targets)
1618 .replace("{group}", &group_path.display().to_string())
1619 .replace("{verdict}", &verdict.display().to_string())
1620 .replace("{members}", &members)
1621}
1622
1623fn group_files(sessions: &[Session], group: &BugGroup) -> Vec<TestFile> {
1626 let mut files = Vec::new();
1627 for session in sessions.iter().filter(|session| group.agents.contains(&session.id)) {
1628 let failing = group
1629 .failures
1630 .iter()
1631 .filter(|failure| failure.agent == session.id)
1632 .map(|failure| &failure.failure.path)
1633 .collect::<BTreeSet<_>>();
1634 let paths = failing
1635 .iter()
1636 .copied()
1637 .chain(failing.iter().filter_map(|path| session.imports.get(*path)).flatten())
1638 .collect::<BTreeSet<_>>();
1639 files.extend(session.files.iter().filter(|file| paths.contains(&file.path)).cloned());
1640 }
1641 files
1642}
1643
1644fn read_usage(path: &Path, session: &str) -> Result<Option<Usage>> {
1647 match fs::read(path) {
1648 Ok(usage) => Ok(serde_json::from_slice::<Usage>(&usage)
1649 .inspect_err(|error| {
1650 let _ = sh_warn!("{session} wrote invalid usage JSON: {error}");
1651 })
1652 .ok()),
1653 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
1654 Err(error) => Err(error).wrap_err_with(|| format!("failed to read {session} usage")),
1655 }
1656}
1657
1658#[cfg(test)]
1659mod tests {
1660 use super::*;
1661
1662 fn failure(agent: usize, test: &str, claim: &str) -> AgentFailure {
1663 AgentFailure {
1664 agent,
1665 failure: Failure {
1666 test: test.to_string(),
1667 path: PathBuf::from("test/generated/Fee.t.sol"),
1668 reason: "revert".to_string(),
1669 claim: (!claim.is_empty()).then(|| claim.to_string()),
1670 },
1671 file: PathBuf::new(),
1672 }
1673 }
1674
1675 #[test]
1678 fn groups_link_agreeing_failures_by_agent() {
1679 let groups = group_failures(vec![
1680 failure(3, "C::zero", "A zero amount is charged; src/Fee.sol:11 charges()."),
1681 failure(1, "A::cap", "The fee is capped at 100 ether; src/Fee.sol:7 fee()."),
1682 failure(2, "B::cap", "Large amounts pay the maximum; src/Fee.sol:7 fee()."),
1683 failure(2, "B::limit", "Large amounts pay the maximum fee (src/Fee.sol:8)."),
1684 failure(1, "D::none", ""),
1685 ]);
1686 let groups = groups
1687 .iter()
1688 .map(|group| {
1689 let tests = group.failures.iter().map(|failure| failure.failure.test.as_str());
1690 (group.agents.clone(), tests.collect::<Vec<_>>())
1691 })
1692 .collect::<Vec<_>>();
1693 assert_eq!(
1694 groups,
1695 [
1696 (vec![1, 2], vec!["A::cap", "B::cap"]),
1697 (vec![2], vec!["B::limit"]),
1699 (vec![3], vec!["C::zero"]),
1700 (vec![1], vec!["D::none"]),
1701 ]
1702 );
1703 }
1704
1705 #[test]
1709 fn test_claim_reads_the_comment_above_the_test() {
1710 let source = r#"
1711pragma solidity ^0.8.20;
1712contract FeeTest {
1713 // The fee is capped at 100 ether (src/Fee.sol:7).
1714 function testCap() public {}
1715
1716 /// @notice A zero amount is charged,
1717 /// src/Fee.sol:11.
1718 /// forge-config: default.fuzz.runs = 10
1719 function testZero(uint256 amount) public {}
1720
1721 function testBare() public {}
1722
1723 // @bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.
1724 function testTagged() public {}
1725
1726 /// @custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.
1727 function testCustomTag() public {}
1728
1729 /// @notice Large amounts.
1730 /// @custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether,
1731 /// not 100 ether.
1732 function testLateTag() public {}
1733
1734 // @bug Fee.pay src/Fee.sol:20 — the contract pays twice.
1735 function testPaysTwice() public {}
1736}
1737contract OtherTest {
1738 /* Rounds down: src/Fee.sol:6 */
1739 function testCap() public {}
1740}
1741"#;
1742 assert_eq!(
1743 test_claim(source, "FeeTest", "testCap").as_deref(),
1744 Some("The fee is capped at 100 ether (src/Fee.sol:7).")
1745 );
1746 assert_eq!(
1747 test_claim(source, "FeeTest", "testZero").as_deref(),
1748 Some("A zero amount is charged, src/Fee.sol:11.")
1749 );
1750 assert_eq!(test_claim(source, "FeeTest", "testBare"), None);
1751 assert_eq!(
1752 test_claim(source, "FeeTest", "testTagged").as_deref(),
1753 Some("@bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.")
1754 );
1755 assert_eq!(
1756 test_claim(source, "FeeTest", "testCustomTag").as_deref(),
1757 Some("@custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.")
1758 );
1759 assert_eq!(
1760 test_claim(source, "FeeTest", "testLateTag").as_deref(),
1761 Some("@custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether, not 100 ether.")
1762 );
1763 assert_eq!(
1764 test_claim(source, "FeeTest", "testPaysTwice").as_deref(),
1765 Some("@bug Fee.pay src/Fee.sol:20 — the contract pays twice.")
1766 );
1767 assert_eq!(
1768 test_claim(source, "OtherTest", "testCap").as_deref(),
1769 Some("Rounds down: src/Fee.sol:6")
1770 );
1771 assert_eq!(test_claim(source, "OtherTest", "testZero"), None);
1772 assert_eq!(test_claim(source, "MissingTest", "testCap"), None);
1773 }
1774
1775 #[test]
1779 fn collect_tests_reads_solidity_files_under_the_output_directory() {
1780 let workspace = tempfile::tempdir().unwrap();
1781 let out = Path::new("test/generated/agent-1");
1782 let write = |path: &str, content: &str| {
1783 let path = workspace.path().join(out).join(path);
1784 fs::create_dir_all(path.parent().unwrap()).unwrap();
1785 fs::write(path, content).unwrap();
1786 };
1787 write("Fee.t.sol", "contract FeeTest {}");
1788 write("helpers/Helper.sol", "library Helper {}");
1789 write("notes.md", "notes");
1790 write("A{B,C}.t.sol", "contract GlobTest {}");
1791 #[cfg(unix)]
1793 write("dir*/Star.t.sol", "contract StarTest {}");
1794 #[cfg(unix)]
1795 std::os::unix::fs::symlink(
1796 workspace.path().join(out).join("Fee.t.sol"),
1797 workspace.path().join(out).join("Link.t.sol"),
1798 )
1799 .unwrap();
1800
1801 let mut rejected = Vec::new();
1802 let files = collect_tests(workspace.path(), out, 1, &mut rejected).unwrap();
1803 assert_eq!(
1805 files
1806 .iter()
1807 .map(|file| (file.path.as_path(), file.content.as_str()))
1808 .collect::<Vec<_>>(),
1809 [
1810 (Path::new("test/generated/agent-1/Fee.t.sol"), "contract FeeTest {}"),
1811 (Path::new("test/generated/agent-1/helpers/Helper.sol"), "library Helper {}"),
1812 ]
1813 );
1814 rejected.sort_by(|a, b| a.paths.cmp(&b.paths));
1815 assert_eq!(
1816 rejected
1817 .iter()
1818 .map(|rejected| (rejected.paths[0].as_path(), rejected.reason.as_str()))
1819 .collect::<Vec<_>>(),
1820 ["test/generated/agent-1/A{B,C}.t.sol"]
1821 .into_iter()
1822 .chain(cfg!(unix).then_some("test/generated/agent-1/dir*/Star.t.sol"))
1823 .map(|path| (Path::new(path), "the path has a glob character"))
1824 .collect::<Vec<_>>()
1825 );
1826
1827 #[cfg(unix)]
1829 {
1830 let linked = Path::new("test/generated/agent-2");
1831 std::os::unix::fs::symlink(workspace.path().join(out), workspace.path().join(linked))
1832 .unwrap();
1833 let error = collect_tests(workspace.path(), linked, 2, &mut rejected).unwrap_err();
1834 assert!(error.to_string().contains("is not a directory"), "{error}");
1835 }
1836 }
1837
1838 #[test]
1839 fn normalize_test_content_keeps_string_literals_and_strips_comments() {
1840 let with_space = r#"
1841 // Fixture case.
1842 contract EchoSpecTest {
1843 function testEcho() public pure {
1844 require(keccak256(bytes(Echo.echo("a b"))) == keccak256(bytes("a b")), "echo changed");
1845 }
1846 }
1847 "#;
1848 let without_space = r#"
1849 // Other fixture.
1850 contract EchoSpecTest {
1851 function testEcho() public pure {
1852 require(keccak256(bytes(Echo.echo("ab"))) == keccak256(bytes("ab")), "echo changed");
1853 }
1854 }
1855 "#;
1856 let url_one = r#"require(Echo.echo("https://one.example") == "https://one.example");"#;
1857 let url_two = r#"require(Echo.echo("https://two.example") == "https://two.example");"#;
1858 let comment_only_a =
1859 "contract T { function testA() public pure { /* note */ require(true); } }";
1860 let comment_only_b =
1861 "contract T { function testA() public pure { require(true); // note\n } }";
1862
1863 assert_ne!(normalize_test_content(with_space), normalize_test_content(without_space));
1864 assert_ne!(normalize_test_content(url_one), normalize_test_content(url_two));
1865 assert_eq!(normalize_test_content(comment_only_a), normalize_test_content(comment_only_b));
1866 for literal in
1867 [r#""https://one.example""#, r#"'a\' /* b */'"#, r#"unicode"été //""#, r#"hex"abcd""#]
1868 {
1869 assert_eq!(normalize_test_content(literal), [literal]);
1870 }
1871 assert_ne!(normalize_test_content("a + ++b"), normalize_test_content("a++ + b"));
1872 }
1873}