Skip to main content

forge/cmd/
properties.rs

1//! The `forge properties` command: several agents look for bugs in the targets independently and
2//! in parallel, each in one session in its own copy of the project, and prove each bug with a
3//! failing test. Forge runs the tests of every agent again in a fresh copy on every seed, reports a
4//! bug only when enough agents agree on its root cause, and keeps one copy of each passing test
5//! file. An optional verifier then tries to disprove each reported bug.
6
7use crate::{
8    cmd::test::RerunFailure,
9    result::TestStatus,
10    sandbox::{RUN_TIMEOUT, Sandbox, TestFile, ensure_inside, run_session, stderr, suite_results},
11    workspace,
12};
13use alloy_primitives::{U256, keccak256};
14use clap::Parser;
15use eyre::{Context, Result, ensure, eyre};
16use foundry_cli::{opts::BuildOpts, utils::LoadConfig};
17use foundry_common::{fs::canonicalize_path, sh_println, sh_warn};
18use foundry_compilers::{
19    Graph,
20    compilers::{
21        Language,
22        multi::{MultiCompilerLanguage, MultiCompilerParser},
23    },
24    utils::source_files_iter,
25};
26use foundry_config::{Config, impl_figment_convert};
27use foundry_evm::fuzz::CounterExample;
28use regex::Regex;
29use serde::{Deserialize, Serialize};
30use solar::parse::Cursor;
31use std::{
32    cmp::Reverse,
33    collections::{BTreeMap, BTreeSet, HashSet},
34    ffi::OsString,
35    fs,
36    io::{self, Write},
37    ops::Add,
38    path::{Path, PathBuf},
39    process::{Command, Output, Stdio},
40    sync::{
41        LazyLock,
42        atomic::{AtomicUsize, Ordering},
43    },
44    thread,
45    time::SystemTime,
46};
47
48mod matcher;
49
50/// On 12 held-out cases, four small-model agents with a quorum of two found bugs in as many cases
51/// as five (8.6 against 9.0 of 12, a tie in a bootstrap over cases) at four fifths of the cost.
52/// With a quorum of two, they kept no report that reviewers labelled wrong, because wrong claims
53/// of independent agents rarely repeat.
54const DEFAULT_AGENTS: usize = 4;
55const DEFAULT_QUORUM: usize = 2;
56/// The focus of each agent without `--focus`, by agent number, wrapping: the four strategy lanes
57/// of ultrafuzz's smoke benchmark (MIT). On an audited vault, five runs of four agents with these
58/// focuses found 3.6 distinct real bugs per run against 2.2 without, at a lower cost and with
59/// the same hit rate on the audit's findings. A single agent gets no focus.
60const DEFAULT_FOCUSES: [&str; 4] = [
61    "block and time transitions, accrual, vesting, auctions, deadlines, stale snapshots, and \
62     sequences whose result depends on elapsed time.",
63    "stale, reverting, zero, extreme, or adversarial oracle, strategy, pool, token, gauge, hook, \
64     and callback values.",
65    "cached or stored balances, donations, fees, rewards, bad debt, utilization, exchange rates, \
66     and state that can diverge from live balances.",
67    "agreement of max, preview, and view functions before, during, and after lifecycle \
68     transitions, especially empty, paused, removed, nested, and terminal states.",
69];
70/// The task of a verifier session. The placeholders are the targets, the group file, the verdict
71/// file, and the tests of the group.
72const VERIFY_PROMPT: &str = include_str!("properties/verify_prompt.md");
73
74/// The errors of what Forge blocks in its workspaces: `vm.ffi` without `--allow-ffi`, network
75/// cheatcodes without `--allow-network`, env cheatcodes, which see only a few variables, and
76/// inline config that `--restrict-inline-config` refuses.
77static BLOCKED: LazyLock<Regex> = LazyLock::new(|| {
78    Regex::new(r#"FFI is disabled|network access is disabled|environment variable "[^"]*" not found|failed to resolve env var|inline config cannot"#)
79        .unwrap()
80});
81/// Solidity line and block comments.
82static COMMENT: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?s)//[^\n]*|/\*.*?\*/").unwrap());
83/// A contract declaration and its name.
84static CONTRACT: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\bcontract\s+(\w+)").unwrap());
85/// Run agents that prove bugs with failing tests and report the bugs that they agree on.
86#[derive(Clone, Debug, Parser)]
87pub struct PropertiesArgs {
88    /// Source file to look for bugs in. Can be given more than once.
89    ///
90    /// Defaults to every source file in the project's source directory.
91    #[arg(long, value_name = "PATH")]
92    target: Vec<PathBuf>,
93
94    /// Reference material for the agents, such as docs, a specification, or a threat model.
95    ///
96    /// Forge gives every agent the absolute path of each file. Can be given more than once.
97    #[arg(long, value_name = "PATH")]
98    context: Vec<PathBuf>,
99
100    /// Executable that runs one agent session, such as a wrapper around any coding agent.
101    ///
102    /// Forge runs `PATH [ARG...]` once per agent, in parallel, each in its own copy of the
103    /// project, with these environment variables:
104    ///
105    /// FORGE_PROPERTIES_PROMPT   the task: targets, context files, and output directory
106    /// FORGE_PROPERTIES_TARGETS  target files, relative to the copy, one per line
107    /// FORGE_PROPERTIES_CONTEXT  absolute paths of the --context files, one per line
108    /// FORGE_PROPERTIES_OUT      absolute, empty directory for the agent's tests
109    /// FORGE_PROPERTIES_SEED     the agent's own seed, in hex
110    /// FORGE_PROPERTIES_FOCUS    the agent's focus, as in the task, or empty
111    /// FORGE_PROPERTIES_USAGE    file where the agent can write its model usage as JSON:
112    ///                           {"input_tokens": N, "output_tokens": N, "cost_usd": X}
113    ///
114    /// The agent writes Solidity tests under FORGE_PROPERTIES_OUT and can run `forge test` in the
115    /// copy. When it exits or reaches --timeout, Forge ends its process group and runs every
116    /// `*.sol` file under FORGE_PROPERTIES_OUT in a fresh copy of the project on every seed. A
117    /// test that fails on every seed is a possible bug. Its claim is the comment right above the
118    /// test function, `// @bug <Contract>.<function> <file>:<line> — <claim>`, else the test name.
119    /// Forge keeps the files whose tests all pass. A nonzero exit status is an error, but
120    /// Forge still runs the agent's tests. In Forge's runs, inline config can only lower runs,
121    /// depth, and rejects and change a few fuzzer and invariant settings, and a run that takes
122    /// more than ten minutes ends.
123    #[arg(long, required = true, value_name = "PATH", verbatim_doc_comment)]
124    agent_command: PathBuf,
125
126    /// Argument passed to the agent command. Can be given more than once.
127    #[arg(long, allow_hyphen_values = true, value_name = "ARG")]
128    agent_arg: Vec<OsString>,
129
130    /// Executable that tries to disprove each possible bug that reaches the quorum, such as a
131    /// wrapper around any coding agent. Without it, Forge runs no verifier.
132    ///
133    /// Forge runs `PATH [ARG...]` once per group, at most --agents at a time, each in a fresh copy
134    /// of the project that also holds the tests of the group, with these environment variables:
135    ///
136    /// FORGE_PROPERTIES_VERIFY_PROMPT  the task: try to disprove the claims of the group
137    /// FORGE_PROPERTIES_GROUP          JSON file with the agents, the claims, the tests, and the
138    ///                                 failure output of the group
139    /// FORGE_PROPERTIES_VERDICT        file where the verifier writes its verdict as JSON:
140    ///                                 {"verdict": "confirmed" | "needs_validation" | "rejected",
141    ///                                  "reason": "...",
142    ///                                  "evidence": [{"file": "...", "line": N, "quote": "..."}],
143    ///                                  "unresolved": "..."}
144    /// FORGE_PROPERTIES_USAGE          file where the verifier can write its model usage as JSON,
145    ///                                 as for an agent
146    ///
147    /// When the verifier exits or reaches --timeout, Forge ends its process group and reads the
148    /// verdict. Forge lists the confirmed groups first, then the groups that need validation with
149    /// the open fact, then the rejected groups with the disproof. A missing or invalid verdict,
150    /// and a rejection without evidence, count as needs_validation. Groups below the quorum are
151    /// not verified.
152    #[arg(long, value_name = "PATH", verbatim_doc_comment)]
153    verifier_command: Option<PathBuf>,
154
155    /// Argument passed to the verifier command. Can be given more than once.
156    #[arg(long, allow_hyphen_values = true, requires = "verifier_command", value_name = "ARG")]
157    verifier_arg: Vec<OsString>,
158
159    /// Deterministic seeds used to run the tests of every agent. Supply at least two.
160    #[arg(long, required = true, action = clap::ArgAction::Append, value_name = "SEED")]
161    seed: Vec<U256>,
162
163    /// Number of agents. Each agent runs one session independently and in parallel in its own
164    /// copy of the project, with its own seed.
165    #[arg(long, default_value_t = DEFAULT_AGENTS, value_name = "N")]
166    agents: usize,
167
168    /// An area for one agent to search first. Agent N gets the Nth focus, wrapping. Can be given
169    /// at most --agents times.
170    ///
171    /// Without it, two or more agents get built-in focuses: time and sequence effects, external
172    /// dependency boundaries, stored state against live balances, and limit and view functions
173    /// across lifecycle states.
174    #[arg(long, value_name = "TEXT")]
175    focus: Vec<String>,
176
177    /// Give every agent the same task, without a focus.
178    #[arg(long, conflicts_with = "focus")]
179    no_focus: bool,
180
181    /// Number of agents that must report the same root cause before Forge reports it as a
182    /// possible bug.
183    ///
184    /// Two failing tests have the same root cause when their `@bug` tags name the same function
185    /// in the same Solidity file at near lines. Without both tags, their claims must name the same
186    /// Solidity file and share a function name at near lines or at least 30% of their words. A
187    /// test joins a group only when it has the same root cause as every test in it. Forge lists
188    /// the groups below the quorum after the possible bugs. Defaults to 2, or 1 with a
189    /// single agent.
190    #[arg(long, value_name = "K")]
191    quorum: Option<usize>,
192
193    /// Seconds after which Forge ends an agent session and runs the tests that it wrote, or ends a
194    /// verifier and reads its verdict.
195    #[arg(long, value_name = "SECS")]
196    timeout: Option<u64>,
197
198    /// Let generated tests run host commands with `vm.ffi` if the project enables `ffi`.
199    ///
200    /// Generated tests run under Forge, outside the sandbox of the agent that wrote them, so
201    /// Forge turns `ffi` off in its copies of the project by default.
202    #[arg(long)]
203    allow_ffi: bool,
204
205    /// Let generated tests open network connections, for example with `vm.createFork`.
206    ///
207    /// By default, cheatcodes that open a connection fail in Forge's copies of the project, and
208    /// the copies have no RPC or Etherscan settings.
209    #[arg(long)]
210    allow_network: bool,
211
212    #[command(flatten)]
213    build: BuildOpts,
214}
215
216impl_figment_convert!(PropertiesArgs, build);
217
218/// Model usage that an agent reports.
219#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)]
220struct Usage {
221    #[serde(default)]
222    input_tokens: u64,
223    #[serde(default)]
224    output_tokens: u64,
225    #[serde(default)]
226    cost_usd: f64,
227}
228
229impl Add for Usage {
230    type Output = Self;
231
232    fn add(self, other: Self) -> Self {
233        Self {
234            input_tokens: self.input_tokens + other.input_tokens,
235            output_tokens: self.output_tokens + other.output_tokens,
236            cost_usd: self.cost_usd + other.cost_usd,
237        }
238    }
239}
240
241/// A test that fails on every seed.
242#[derive(Clone, Debug, Serialize)]
243struct Failure {
244    /// `Contract::test`.
245    test: String,
246    path: PathBuf,
247    reason: String,
248    /// The comment above the test.
249    #[serde(skip_serializing_if = "Option::is_none")]
250    claim: Option<String>,
251}
252
253/// A test of one agent that fails on every seed.
254#[derive(Debug, Serialize)]
255struct AgentFailure {
256    agent: usize,
257    #[serde(flatten)]
258    failure: Failure,
259    /// The copy of the test file for review, relative to the project root.
260    file: PathBuf,
261}
262
263impl std::fmt::Display for AgentFailure {
264    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265        let failure = &self.failure;
266        write!(f, "  agent {}: {} ({})", self.agent, failure.test, failure.reason)?;
267        if let Some(claim) = &failure.claim {
268            write!(f, "\n    claim: {claim}")?;
269        }
270        write!(f, "\n    file: {}", self.file.display())
271    }
272}
273
274/// Files of an agent that Forge did not run.
275#[derive(Debug, Serialize)]
276struct Rejected {
277    agent: usize,
278    paths: Vec<PathBuf>,
279    reason: String,
280}
281
282/// The failing tests of all agents that have one root cause.
283#[derive(Debug, Serialize)]
284struct BugGroup {
285    /// The agents that report the root cause.
286    agents: Vec<usize>,
287    /// The failing tests, by agent. The first test has a claim, if any test has.
288    failures: Vec<AgentFailure>,
289    /// What the verifier concluded, if there is one.
290    #[serde(skip_serializing_if = "Option::is_none")]
291    verification: Option<Verification>,
292}
293
294/// The verdict of a verifier.
295///
296/// The order is the order of the report.
297#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
298#[serde(rename_all = "snake_case")]
299enum Verdict {
300    /// Every attempt to disprove the claim failed.
301    Confirmed,
302    /// One open fact decides the claim.
303    #[default]
304    NeedsValidation,
305    /// A disproof holds for every claim of the group.
306    Rejected,
307}
308
309/// A quote from a file that supports a verdict.
310#[derive(Debug, Deserialize, Serialize)]
311struct Evidence {
312    #[serde(default)]
313    file: String,
314    #[serde(default)]
315    line: u64,
316    #[serde(default)]
317    quote: String,
318}
319
320/// The verdict file of a verifier.
321#[derive(Debug, Default, Deserialize, Serialize)]
322struct Judgment {
323    verdict: Verdict,
324    #[serde(default)]
325    reason: String,
326    #[serde(default)]
327    evidence: Vec<Evidence>,
328    /// The open fact of a verdict that needs validation.
329    #[serde(default, skip_serializing_if = "Option::is_none")]
330    unresolved: Option<String>,
331}
332
333/// What one verifier session left for a bug group.
334#[derive(Debug, Serialize)]
335struct Verification {
336    #[serde(flatten)]
337    judgment: Judgment,
338    /// What went wrong with the verifier, such as a missing verdict. A group whose verifier left
339    /// no valid verdict needs validation.
340    #[serde(skip_serializing_if = "Option::is_none")]
341    error: Option<String>,
342    #[serde(skip_serializing_if = "Option::is_none")]
343    usage: Option<Usage>,
344}
345
346impl Verification {
347    /// Returns the verification of a group whose verifier left no valid verdict.
348    fn failed(error: String) -> Self {
349        Self { judgment: Judgment::default(), error: Some(error), usage: None }
350    }
351}
352
353impl std::fmt::Display for Verification {
354    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
355        let judgment = &self.judgment;
356        let mut lines = Vec::new();
357        if !judgment.reason.is_empty() {
358            lines.push(format!("  verifier: {}", judgment.reason));
359        }
360        if let Some(error) = &self.error {
361            lines.push(format!("  verifier error: {error}"));
362        }
363        if let Some(unresolved) = judgment.unresolved.as_deref().filter(|fact| !fact.is_empty()) {
364            lines.push(format!("    unresolved: {unresolved}"));
365        }
366        for evidence in &judgment.evidence {
367            let quote = evidence.quote.split_whitespace().collect::<Vec<_>>().join(" ");
368            lines.push(format!("    evidence: {}:{}: {quote}", evidence.file, evidence.line));
369        }
370        write!(f, "{}", lines.join("\n"))
371    }
372}
373
374/// The tests of some files, run on every seed.
375#[derive(Default)]
376struct TestRun {
377    /// Tests that fail on every seed.
378    possible_bugs: Vec<Failure>,
379    /// Tests that pass on every seed.
380    passing: Vec<RerunFailure>,
381    /// Files with a test that does not pass on every seed.
382    failing_files: HashSet<PathBuf>,
383    /// Seeds on which Forge could not run the tests.
384    errors: Vec<String>,
385}
386
387/// What one agent session left.
388struct Session {
389    id: usize,
390    /// How the agent command ended.
391    exit: AgentExit,
392    usage: Option<Usage>,
393    possible_bugs: Vec<AgentFailure>,
394    rejected: Vec<Rejected>,
395    /// The files whose tests pass on every seed, with the files that they import.
396    kept: Vec<TestFile>,
397    /// Every file that the agent wrote and Forge ran.
398    files: Vec<TestFile>,
399    /// The files of `files` that each of them imports, directly or not, as the compiler resolves
400    /// the imports.
401    imports: BTreeMap<PathBuf, BTreeSet<PathBuf>>,
402}
403
404/// How an agent command ended.
405enum AgentExit {
406    /// The command exited successfully.
407    Succeeded,
408    /// The command failed, for this reason.
409    Failed(String),
410    /// The command reached the timeout, so Forge ended it.
411    TimedOut,
412}
413
414/// The output directories that a run reserved. The ones it left empty are removed on drop.
415struct RunDirs(Vec<PathBuf>);
416
417impl Drop for RunDirs {
418    fn drop(&mut self) {
419        for dir in &self.0 {
420            let _ = fs::remove_dir(dir);
421        }
422    }
423}
424
425impl PropertiesArgs {
426    pub fn run(mut self) -> Result<()> {
427        ensure!(self.seed.len() >= 2, "at least two --seed values are required");
428        ensure!(
429            self.seed.iter().collect::<HashSet<_>>().len() == self.seed.len(),
430            "--seed values must be distinct"
431        );
432        ensure!(self.agents > 0, "--agents must be greater than zero");
433        let quorum = self.quorum.unwrap_or(self.agents.min(DEFAULT_QUORUM));
434        ensure!((1..=self.agents).contains(&quorum), "--quorum must be between 1 and --agents");
435        ensure!(self.focus.len() <= self.agents, "--focus can be given at most --agents times");
436        ensure!(
437            self.target.iter().all(|path| workspace::is_safe_relative_path(path)),
438            "--target paths must be project-relative paths"
439        );
440        // Workspaces map project paths by the resolved root, so the root must have no links.
441        // `--config-path` resolves already.
442        let paths = &mut self.build.project_paths;
443        if paths.config_path.is_none() {
444            let root = paths.try_project_root()?;
445            paths.root = Some(root.canonicalize().wrap_err("failed to resolve project root")?);
446        }
447        let config = self.load_config()?;
448        let generated_tests =
449            workspace::relative_to_root(&config.root, &config.test).join("generated");
450        workspace::ensure_safe_relative_path(&generated_tests, "generated test", &config.test)?;
451        // Each run writes to directories of its own, so a run never meets the files of another.
452        // Creating them reserves them against concurrent runs.
453        let generated_dir = config.root.join(&generated_tests);
454        ensure_inside(&config.root, &generated_dir, &generated_tests)?;
455        fs::create_dir_all(&generated_dir)
456            .wrap_err_with(|| format!("failed to create {}", generated_dir.display()))?;
457        let properties_cache = config.cache_path.join("properties");
458        fs::create_dir_all(&properties_cache)?;
459        let mut run = 1;
460        let (run_dirs, cache_root) = loop {
461            let mut dirs = RunDirs(Vec::new());
462            for id in 1..=self.agents {
463                let dir = generated_dir.join(format!("run-{run}-agent-{id}"));
464                match fs::create_dir(&dir) {
465                    Ok(()) => dirs.0.push(dir),
466                    Err(error) if error.kind() == io::ErrorKind::AlreadyExists => break,
467                    Err(error) => {
468                        return Err(error)
469                            .wrap_err_with(|| format!("failed to create {}", dir.display()));
470                    }
471                }
472            }
473            if dirs.0.len() == self.agents {
474                // Empty generated directories are removed on drop, but campaign caches remain.
475                // Reserve the cache too, so a later run never reads an earlier run's outputs.
476                let cache = properties_cache.join(format!("run-{run}"));
477                match fs::create_dir(&cache) {
478                    Ok(()) => break (dirs, cache),
479                    Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
480                    Err(error) => {
481                        return Err(error)
482                            .wrap_err_with(|| format!("failed to create {}", cache.display()));
483                    }
484                }
485            }
486            run += 1;
487        };
488        let verifier_command = self
489            .verifier_command
490            .as_ref()
491            .map(|command| command.canonicalize().wrap_err("failed to resolve verifier command"))
492            .transpose()?;
493        // Workspaces leave out the project's generated tests, so every agent starts without
494        // earlier generated tests.
495        let mut sandbox = Sandbox::new(
496            &config,
497            vec![config.root.join(&generated_tests)],
498            self.allow_ffi,
499            self.allow_network,
500        )?;
501        sandbox.end_on_interrupt(run_dirs.0.clone())?;
502        if let Some(errors) = sandbox.build()? {
503            sh_warn!("the project does not build, so every test run builds it: {errors}")?;
504        }
505        let campaign = Campaign {
506            args: &self,
507            config: &config,
508            sandbox,
509            verifier_command,
510            generated_tests,
511            run,
512            cache_root,
513            quorum,
514        };
515        campaign.run()
516    }
517}
518
519/// The size and modification time of each project file, by path relative to the root.
520type Fingerprint = BTreeMap<PathBuf, (u64, Option<SystemTime>)>;
521
522/// The settings of one `forge properties` run.
523struct Campaign<'a> {
524    args: &'a PropertiesArgs,
525    config: &'a Config,
526    sandbox: Sandbox<'a>,
527    /// The verifier command, if the run has a verifier.
528    verifier_command: Option<PathBuf>,
529    /// The directory of generated tests, relative to the project root.
530    generated_tests: PathBuf,
531    /// The number of this run. Agent `N` writes to `run-<run>-agent-<N>` in `generated_tests`.
532    run: usize,
533    cache_root: PathBuf,
534    quorum: usize,
535}
536
537/// The inputs that every agent shares.
538struct AgentInputs {
539    agent_command: PathBuf,
540    /// The target files, relative to the project root.
541    targets: Vec<PathBuf>,
542    /// The absolute paths of the context files.
543    context: Vec<PathBuf>,
544}
545
546impl Campaign<'_> {
547    fn run(&self) -> Result<()> {
548        let agent_command =
549            self.args.agent_command.canonicalize().wrap_err("failed to resolve agent command")?;
550        let context = self
551            .args
552            .context
553            .iter()
554            .map(|path| {
555                canonicalize_path(path)
556                    .wrap_err_with(|| format!("failed to resolve context {}", path.display()))
557            })
558            .collect::<Result<Vec<_>>>()?;
559        let targets = if self.args.target.is_empty() {
560            source_files_iter(&self.config.src, MultiCompilerLanguage::FILE_EXTENSIONS)
561                .map(|path| workspace::relative_to_root(&self.config.root, &path))
562                .collect()
563        } else {
564            self.args.target.clone()
565        };
566        let inputs = AgentInputs { agent_command, targets, context };
567        // Agents work in copies of the project, but the copies link the entries of dependency
568        // directories. A shared file that changes while they run taints the run.
569        let fingerprint = project_fingerprint(self.config, &self.sandbox.dependencies());
570        // An agent error ends only that agent, and the run reports it after the other agents, so
571        // their results are not lost.
572        let results = thread::scope(|scope| {
573            let inputs = &inputs;
574            let agents = (1..=self.args.agents)
575                .map(|id| scope.spawn(move || self.run_agent(id, inputs)))
576                .collect::<Vec<_>>();
577            agents
578                .into_iter()
579                .map(|agent| agent.join().expect("agent thread panicked"))
580                .collect::<Vec<_>>()
581        });
582        let mut sessions = Vec::new();
583        let mut errors = Vec::new();
584        for (id, result) in (1..).zip(results) {
585            match result {
586                Ok(session) => {
587                    if let AgentExit::Failed(error) = &session.exit {
588                        errors.push(format!("agent {id}: {error}"));
589                    }
590                    sessions.push(session);
591                }
592                Err(error) => errors.push(format!("agent {id}: {error:#}")),
593            }
594        }
595        self.report(&sessions, &inputs, &fingerprint)?;
596        ensure!(errors.is_empty(), "{}", errors.join("\n"));
597        Ok(())
598    }
599
600    /// Prints the possible bugs that reach the quorum and writes one copy of each kept file to the
601    /// project. A verifier runs first, and the project files must match `fingerprint` after it.
602    fn report(
603        &self,
604        sessions: &[Session],
605        inputs: &AgentInputs,
606        fingerprint: &Fingerprint,
607    ) -> Result<()> {
608        let failures = sessions
609            .iter()
610            .flat_map(|session| &session.possible_bugs)
611            .map(|failure| AgentFailure {
612                agent: failure.agent,
613                failure: failure.failure.clone(),
614                file: failure.file.clone(),
615            })
616            .collect::<Vec<_>>();
617        // A test that fails only because Forge blocked a cheatcode or its inline config is not
618        // evidence of a bug.
619        let (blocked, failures) = failures
620            .into_iter()
621            .partition::<Vec<_>, _>(|failure| BLOCKED.is_match(&failure.failure.reason));
622        let (mut possible_bugs, below_quorum) = group_failures(failures)
623            .into_iter()
624            .partition::<Vec<_>, _>(|group| group.agents.len() >= self.quorum);
625        if let Some(command) = &self.verifier_command {
626            self.verify_groups(command, &mut possible_bugs, sessions, inputs);
627        }
628        let after = project_fingerprint(self.config, &self.sandbox.dependencies());
629        let tainted = after
630            .iter()
631            .filter(|(path, metadata)| fingerprint.get(*path) != Some(*metadata))
632            .map(|(path, _)| path)
633            .chain(fingerprint.keys().filter(|path| !after.contains_key(*path)))
634            .cloned()
635            .collect::<Vec<_>>();
636        let print_group = |label: &str, group: &BugGroup| -> Result<()> {
637            let lead = &group.failures[0].failure;
638            let summary = lead.claim.as_deref().unwrap_or(&lead.test);
639            sh_println!("{label} ({}/{} agents): {summary}", group.agents.len(), self.args.agents)?;
640            for failure in &group.failures {
641                sh_println!("{failure}")?;
642            }
643            if let Some(verification) = &group.verification {
644                sh_println!("{verification}")?;
645            }
646            Ok(())
647        };
648        for group in &possible_bugs {
649            let label = match group.verification.as_ref().map(|v| v.judgment.verdict) {
650                None => "possible bug",
651                Some(Verdict::Confirmed) => "confirmed",
652                Some(Verdict::NeedsValidation) => "needs validation",
653                Some(Verdict::Rejected) => "rejected",
654            };
655            print_group(label, group)?;
656        }
657        let report_path = self.cache_root.join("report.json");
658        let report_display = report_path.strip_prefix(&self.config.root).unwrap_or(&report_path);
659        // The groups below the quorum hold most of the bugs that agents find, so they follow the
660        // possible bugs as a separate list.
661        if !below_quorum.is_empty() {
662            sh_println!(
663                "{} possible bug(s) below the quorum of {}:",
664                below_quorum.len(),
665                self.quorum
666            )?;
667            for group in &below_quorum {
668                let label = if group.agents.len() == 1 { "single-agent" } else { "below quorum" };
669                print_group(label, group)?;
670            }
671        }
672        // An agent's kept files go to the project together or not at all: a path that leads
673        // outside the project, or that a concurrent run took, holds back the agent's files. A
674        // file that repeats a generated test of the project or of another agent is not written,
675        // unless a written file of its agent imports it or it imports one, so every written test
676        // compiles against the files that it ran with. Nothing is kept when files that the
677        // workspaces share with the project changed while the agents ran.
678        let existing = collect_tests(&self.config.root, &self.generated_tests, 0, &mut vec![])
679            .unwrap_or_default();
680        let mut seen = existing
681            .iter()
682            .map(|file| normalize_test_content(&file.content))
683            .collect::<HashSet<_>>();
684        let mut held_back = Vec::new();
685        let mut kept = Vec::new();
686        let mut duplicates = Vec::new();
687        for session in sessions.iter().filter(|_| tainted.is_empty()) {
688            let conflict = session.kept.iter().find_map(|file| {
689                let path = self.config.root.join(&file.path);
690                // A linked directory on the path must not lead outside the project.
691                if let Err(error) = ensure_inside(&self.config.root, &path, &file.path) {
692                    return Some(format!("{error:#}"));
693                }
694                path.exists().then(|| format!("{} exists already", file.path.display()))
695            });
696            if let Some(reason) = conflict {
697                let paths = session.kept.iter().map(|file| file.path.clone()).collect();
698                held_back.push(Rejected { agent: session.id, paths, reason });
699                continue;
700            }
701            let files = &session.kept;
702            let imports = |file: &TestFile, other: &TestFile| {
703                session.imports.get(&file.path).is_some_and(|i| i.contains(&other.path))
704            };
705            let mut write = files
706                .iter()
707                .map(|file| seen.insert(normalize_test_content(&file.content)))
708                .collect::<Vec<_>>();
709            while let Some(index) = (0..files.len()).find(|&i| {
710                !write[i]
711                    && files.iter().zip(&write).any(|(other, written)| {
712                        *written && (imports(other, &files[i]) || imports(&files[i], other))
713                    })
714            }) {
715                write[index] = true;
716            }
717            for (file, write) in files.iter().zip(write) {
718                if write {
719                    kept.push(file);
720                } else {
721                    duplicates.push(&file.path);
722                }
723            }
724        }
725        let rejected = sessions
726            .iter()
727            .flat_map(|session| &session.rejected)
728            .chain(&held_back)
729            .collect::<Vec<_>>();
730        if !rejected.is_empty() {
731            sh_println!(
732                "{} generated test file(s) did not compile or were not allowed: see {}",
733                rejected.iter().map(|rejected| rejected.paths.len()).sum::<usize>(),
734                report_display.display()
735            )?;
736        }
737        if !blocked.is_empty() {
738            sh_warn!(
739                "{} failing test(s) used something that Forge blocks (ffi, environment, network, or inline config), so they are not possible bugs: see {}",
740                blocked.len(),
741                report_display.display()
742            )?;
743        }
744        let timed_out = sessions
745            .iter()
746            .filter(|session| matches!(session.exit, AgentExit::TimedOut))
747            .map(|session| session.id)
748            .collect::<Vec<_>>();
749        if !timed_out.is_empty() {
750            sh_warn!(
751                "{} agent(s) reached the timeout, so Forge ended them and ran the tests that they wrote",
752                timed_out.len()
753            )?;
754        }
755        let usage = sessions
756            .iter()
757            .filter_map(|session| {
758                Some(serde_json::json!({ "agent": session.id, "usage": session.usage? }))
759            })
760            .collect::<Vec<_>>();
761        if let Some(total) = sessions.iter().filter_map(|session| session.usage).reduce(Usage::add)
762        {
763            sh_println!(
764                "agent usage: {} input tokens, {} output tokens, {:.2} USD",
765                total.input_tokens,
766                total.output_tokens,
767                total.cost_usd
768            )?;
769        }
770        if let Some(total) = possible_bugs
771            .iter()
772            .filter_map(|group| group.verification.as_ref()?.usage)
773            .reduce(Usage::add)
774        {
775            sh_println!(
776                "verifier usage: {} input tokens, {} output tokens, {:.2} USD",
777                total.input_tokens,
778                total.output_tokens,
779                total.cost_usd
780            )?;
781        }
782
783        fs::create_dir_all(&self.cache_root)?;
784        let report = serde_json::json!({
785            "agents": self.args.agents,
786            "quorum": self.quorum,
787            "possible_bugs": possible_bugs,
788            "below_quorum": below_quorum,
789            "blocked": blocked,
790            "rejected": rejected,
791            "timed_out": timed_out,
792            "tainted": tainted,
793            "kept": kept.iter().map(|file| &file.path).collect::<Vec<_>>(),
794            "duplicates": duplicates,
795            "usage": usage,
796        });
797        let report_bytes = serde_json::to_vec_pretty(&report)?;
798        fs::write(&report_path, &report_bytes)?;
799        // Publish the convenience copy atomically: concurrent campaigns must not interleave
800        // writes into one report. Each campaign also keeps its own copy under `run-<n>/`.
801        let properties_root = self.cache_root.parent().expect("campaign cache has a parent");
802        let mut latest = tempfile::NamedTempFile::new_in(properties_root)?;
803        latest.write_all(&report_bytes)?;
804        latest.persist(properties_root.join("report.json")).map_err(|error| error.error)?;
805
806        // Agents can reach the project's dependencies through their copies, so a change can come
807        // from an agent, and no result of the run can be trusted.
808        ensure!(
809            tainted.is_empty(),
810            "{} project file(s) changed while the agents ran, so Forge kept no generated test: see {}",
811            tainted.len(),
812            report_display.display()
813        );
814        if kept.is_empty() {
815            sh_println!("no generated test file was kept")?;
816            return Ok(());
817        }
818        for file in &kept {
819            let path = self.config.root.join(&file.path);
820            fs::create_dir_all(path.parent().expect("test path has a parent"))?;
821            fs::OpenOptions::new()
822                .write(true)
823                .create_new(true)
824                .open(&path)
825                .and_then(|mut target| target.write_all(file.content.as_bytes()))
826                .wrap_err_with(|| format!("failed to write {}", file.path.display()))?;
827        }
828        sh_println!("added {} generated test file(s):", kept.len())?;
829        for file in &kept {
830            sh_println!("  {}", file.path.display())?;
831        }
832        if !duplicates.is_empty() {
833            sh_println!(
834                "skipped {} generated test file(s) that repeat a file of another agent or the project",
835                duplicates.len()
836            )?;
837        }
838        Ok(())
839    }
840
841    /// Runs one agent session in its own copy of the project, then runs the tests that it wrote.
842    fn run_agent(&self, id: usize, inputs: &AgentInputs) -> Result<Session> {
843        let args = self.args;
844        let cache = self.cache_root.join(format!("agent-{id}"));
845        fs::create_dir_all(&cache)?;
846        let workspace = self.sandbox.workspace(&[])?;
847        let out = self.generated_tests.join(format!("run-{}-agent-{id}", self.run));
848        let out_dir = workspace.path().join(&out);
849        ensure_inside(workspace.path(), &out_dir, &out)?;
850        fs::create_dir_all(&out_dir)?;
851        let focuses = if args.no_focus || (args.focus.is_empty() && args.agents == 1) {
852            vec![]
853        } else if args.focus.is_empty() {
854            DEFAULT_FOCUSES.to_vec()
855        } else {
856            args.focus.iter().map(String::as_str).collect()
857        };
858        let focus = (!focuses.is_empty()).then(|| focuses[(id - 1) % focuses.len()]);
859        // The other agents have other focuses only when each agent has its own.
860        let distinct = args.agents > 1 && args.agents <= focuses.len();
861        let prompt = cache.join("prompt.md");
862        fs::write(
863            &prompt,
864            session_prompt(&inputs.targets, &inputs.context, &out_dir, focus, distinct),
865        )?;
866        let usage_path = cache.join("usage.json");
867        let log_path = cache.join("agent.log");
868        let log = fs::File::create(&log_path)?;
869        // Agents differ by seed.
870        let seed: U256 =
871            keccak256([args.seed[0].to_be_bytes::<32>(), U256::from(id).to_be_bytes()].concat())
872                .into();
873        let lines = |paths: &[PathBuf]| {
874            paths.iter().map(|path| path.display().to_string()).collect::<Vec<_>>().join("\n")
875        };
876
877        let mut command = Command::new(&inputs.agent_command);
878        command
879            .args(&args.agent_arg)
880            .current_dir(workspace.path())
881            .env("FORGE_PROPERTIES_PROMPT", &prompt)
882            .env("FORGE_PROPERTIES_TARGETS", lines(&inputs.targets))
883            .env("FORGE_PROPERTIES_CONTEXT", lines(&inputs.context))
884            .env("FORGE_PROPERTIES_OUT", &out_dir)
885            .env("FORGE_PROPERTIES_SEED", format!("{seed:#x}"))
886            .env("FORGE_PROPERTIES_FOCUS", focus.unwrap_or_default())
887            .env("FORGE_PROPERTIES_USAGE", &usage_path)
888            .stdin(Stdio::null())
889            .stdout(log.try_clone()?)
890            .stderr(log);
891        let status = run_session(&mut command, args.timeout, "agent")?;
892        let log_display = log_path.strip_prefix(&self.config.root).unwrap_or(&log_path);
893        let exit = match status {
894            None => AgentExit::TimedOut,
895            Some(status) if status.success() => AgentExit::Succeeded,
896            Some(status) => AgentExit::Failed(format!(
897                "agent command failed ({status}): see {}",
898                log_display.display()
899            )),
900        };
901        let usage = read_usage(&usage_path, &format!("agent {id}"))?;
902        let mut session = Session {
903            id,
904            exit,
905            usage,
906            possible_bugs: vec![],
907            rejected: vec![],
908            kept: vec![],
909            files: vec![],
910            imports: BTreeMap::new(),
911        };
912        session.files = collect_tests(workspace.path(), &out, id, &mut session.rejected)?;
913        let files = &session.files;
914        // Forge writes only passing files to the project, so the copies keep the failing tests.
915        let copy = |path: &Path| cache.join("tests").join(path.strip_prefix(&out).unwrap_or(path));
916        for file in files {
917            let path = copy(&file.path);
918            fs::create_dir_all(path.parent().expect("test path has a parent"))?;
919            fs::write(&path, &file.content)?;
920        }
921        if files.is_empty() {
922            return Ok(session);
923        }
924
925        let workspace = self.sandbox.workspace(files)?;
926        let imports = collected_imports(self.config, workspace.path(), files)?;
927        let run = match self.sandbox.build_errors(workspace.path())? {
928            None => self.run_tests(workspace.path(), files)?,
929            // A file that does not compile, such as one that the agent was writing at the
930            // timeout, must not hide the other tests. Each file that no other file imports then
931            // runs on its own, with the files that it imports. Files that import each other in a
932            // cycle have no such file, so the first of them that no group holds yet starts one.
933            // A group that does not compile rejects and holds only its first file, so the files
934            // that it imports still run without it.
935            Some(_) => {
936                let no_imports = BTreeSet::new();
937                let (roots, others) = files.iter().partition::<Vec<_>, _>(|file| {
938                    !imports.values().any(|imported| imported.contains(&file.path))
939                });
940                let mut grouped = HashSet::new();
941                // A file that several groups import runs in each of them, but its results count
942                // once.
943                let mut ran = HashSet::new();
944                let mut run = TestRun::default();
945                for root in roots.into_iter().chain(others) {
946                    if grouped.contains(&root.path) {
947                        continue;
948                    }
949                    let imported = imports.get(&root.path).unwrap_or(&no_imports);
950                    let group = files
951                        .iter()
952                        .filter(|file| file.path == root.path || imported.contains(&file.path))
953                        .cloned()
954                        .collect::<Vec<_>>();
955                    let workspace = self.sandbox.workspace(&group)?;
956                    if let Some(errors) = self.sandbox.build_errors(workspace.path())? {
957                        grouped.insert(root.path.clone());
958                        let paths = vec![root.path.clone()];
959                        session.rejected.push(Rejected { agent: id, paths, reason: errors });
960                        continue;
961                    }
962                    grouped.extend(group.iter().map(|file| file.path.clone()));
963                    let group_run = self.run_tests(workspace.path(), &group)?;
964                    run.possible_bugs.extend(
965                        group_run.possible_bugs.into_iter().filter(|bug| !ran.contains(&bug.path)),
966                    );
967                    run.passing.extend(
968                        group_run
969                            .passing
970                            .into_iter()
971                            .filter(|test| !ran.contains(&test_path(test))),
972                    );
973                    run.failing_files.extend(group_run.failing_files);
974                    run.errors.extend(group_run.errors);
975                    ran.extend(group.into_iter().map(|file| file.path));
976                }
977                run
978            }
979        };
980        if !run.errors.is_empty() {
981            let paths = files.iter().map(|file| file.path.clone()).collect();
982            session.rejected.push(Rejected { agent: id, paths, reason: run.errors.join("; ") });
983        }
984        let kept = kept_files(files, &imports, &run)?;
985        session.kept = files.iter().filter(|file| kept.contains(&file.path)).cloned().collect();
986        session.imports = imports;
987        let root = &self.config.root;
988        session.possible_bugs = run
989            .possible_bugs
990            .into_iter()
991            .map(|failure| {
992                let file = copy(&failure.path);
993                let file = file.strip_prefix(root).map(Path::to_path_buf).unwrap_or(file);
994                AgentFailure { agent: id, failure, file }
995            })
996            .collect();
997        Ok(session)
998    }
999
1000    /// Runs every test in the files on every seed.
1001    fn run_tests(&self, workspace: &Path, files: &[TestFile]) -> Result<TestRun> {
1002        let mut run = TestRun::default();
1003        // The passing seeds and the failures of each test.
1004        let mut outcomes = BTreeMap::<(String, String), (usize, Vec<String>)>::new();
1005        for seed in &self.args.seed {
1006            let mut command = self.test_command(workspace, files, seed);
1007            let Some(output) = self.sandbox.output(&mut command)? else {
1008                run.errors
1009                    .push(format!("tests reached the timeout of {RUN_TIMEOUT:?} on seed {seed}"));
1010                continue;
1011            };
1012            if !output.status.success() && output.stdout.is_empty() {
1013                run.errors.push(format!("tests failed on seed {seed}: {}", stderr(&output)));
1014                continue;
1015            }
1016            for (test, status, reason) in test_results(&output)? {
1017                let (passes, failures) = outcomes.entry((test.contract, test.test)).or_default();
1018                match status {
1019                    TestStatus::Success => *passes += 1,
1020                    TestStatus::Failure => {
1021                        let reason = reason
1022                            .filter(|reason| !reason.is_empty())
1023                            .unwrap_or_else(|| "the test failed without a reason".to_string());
1024                        failures.push(format!("seed {seed}: {reason}"));
1025                    }
1026                    TestStatus::Skipped => {}
1027                }
1028            }
1029        }
1030        let seeds = self.args.seed.len();
1031        for ((contract, test), (passes, failures)) in outcomes {
1032            let test = RerunFailure { contract, test };
1033            if passes == seeds {
1034                run.passing.push(test);
1035                continue;
1036            }
1037            let path = test_path(&test);
1038            run.failing_files.insert(path.clone());
1039            if failures.len() == seeds {
1040                let contract = test.contract.rsplit_once(':').map_or("", |(_, name)| name);
1041                let function =
1042                    test.test.split_once('(').map_or(test.test.as_str(), |(name, _)| name);
1043                let claim = files
1044                    .iter()
1045                    .find(|file| file.path == path)
1046                    .and_then(|file| test_claim(&file.content, contract, function));
1047                run.possible_bugs.push(Failure {
1048                    test: test_display_name(&test),
1049                    path,
1050                    reason: failures.join("; "),
1051                    claim,
1052                });
1053            }
1054        }
1055        // Possible bugs with a claim first.
1056        run.possible_bugs
1057            .sort_by(|a, b| (a.claim.is_none(), &a.test).cmp(&(b.claim.is_none(), &b.test)));
1058        Ok(run)
1059    }
1060
1061    /// Returns the test command for the files on one seed. Each seed persists fuzz and invariant
1062    /// failures in its own directory, so a counterexample found with one seed is not replayed with
1063    /// another.
1064    fn test_command(&self, workspace: &Path, files: &[TestFile], seed: &U256) -> Command {
1065        let seed_dir = seed_dir(self.config, workspace, seed);
1066        let mut command = self.sandbox.test_command(workspace, files);
1067        command
1068            .env("FOUNDRY_FUZZ_SEED", format!("{seed:#x}"))
1069            .env("FOUNDRY_FUZZ_FAILURE_PERSIST_DIR", seed_dir.join("fuzz"))
1070            .env("FOUNDRY_INVARIANT_FAILURE_PERSIST_DIR", seed_dir.join("invariant"))
1071            // Internal runs must not replace the project's own `--rerun` state.
1072            .env("FOUNDRY_TEST_FAILURES_FILE", seed_dir.join("test-failures"));
1073        command
1074    }
1075
1076    /// Runs one verifier per group, at most `--agents` at a time, and orders the groups by
1077    /// verdict. A verifier error leaves the group in need of validation.
1078    fn verify_groups(
1079        &self,
1080        command: &Path,
1081        groups: &mut [BugGroup],
1082        sessions: &[Session],
1083        inputs: &AgentInputs,
1084    ) {
1085        let next = AtomicUsize::new(0);
1086        let verifications = {
1087            let groups = &*groups;
1088            thread::scope(|scope| {
1089                let workers = (0..self.args.agents.min(groups.len()))
1090                    .map(|_| {
1091                        scope.spawn(|| {
1092                            let mut done = Vec::new();
1093                            loop {
1094                                let index = next.fetch_add(1, Ordering::Relaxed);
1095                                let Some(group) = groups.get(index) else { break };
1096                                let verification = self
1097                                    .verify(command, index + 1, group, sessions, inputs)
1098                                    .unwrap_or_else(|error| {
1099                                        Verification::failed(format!("{error:#}"))
1100                                    });
1101                                done.push((index, verification));
1102                            }
1103                            done
1104                        })
1105                    })
1106                    .collect::<Vec<_>>();
1107                workers
1108                    .into_iter()
1109                    .flat_map(|worker| worker.join().expect("verifier thread panicked"))
1110                    .collect::<Vec<_>>()
1111            })
1112        };
1113        for (index, verification) in verifications {
1114            groups[index].verification = Some(verification);
1115        }
1116        // The sort is stable, so groups with one verdict keep their order.
1117        groups.sort_by_key(|group| group.verification.as_ref().map(|v| v.judgment.verdict));
1118    }
1119
1120    /// Runs the verifier on one group in a fresh workspace that holds the tests of the group, then
1121    /// reads its verdict.
1122    fn verify(
1123        &self,
1124        command: &Path,
1125        id: usize,
1126        group: &BugGroup,
1127        sessions: &[Session],
1128        inputs: &AgentInputs,
1129    ) -> Result<Verification> {
1130        let cache = self.cache_root.join(format!("verify-{id}"));
1131        fs::create_dir_all(&cache)?;
1132        let workspace = self.sandbox.workspace(&group_files(sessions, group))?;
1133        let prompt_path = cache.join("prompt.md");
1134        let group_path = cache.join("group.json");
1135        let verdict_path = cache.join("verdict.json");
1136        let usage_path = cache.join("usage.json");
1137        let log_path = cache.join("verifier.log");
1138        fs::write(&prompt_path, verify_prompt(group, &inputs.targets, &group_path, &verdict_path))?;
1139        let failures = group.failures.iter().map(|failure| {
1140            serde_json::json!({
1141                "agent": failure.agent,
1142                "test": failure.failure.test,
1143                "path": failure.failure.path,
1144                "claim": failure.failure.claim,
1145                "failure": failure.failure.reason,
1146            })
1147        });
1148        let group_json = serde_json::json!({
1149            "targets": inputs.targets,
1150            "agents": group.agents,
1151            "failures": failures.collect::<Vec<_>>(),
1152        });
1153        fs::write(&group_path, serde_json::to_vec_pretty(&group_json)?)?;
1154        let log = fs::File::create(&log_path)?;
1155
1156        let mut command = Command::new(command);
1157        command
1158            .args(&self.args.verifier_arg)
1159            .current_dir(workspace.path())
1160            .env("FORGE_PROPERTIES_VERIFY_PROMPT", &prompt_path)
1161            .env("FORGE_PROPERTIES_GROUP", &group_path)
1162            .env("FORGE_PROPERTIES_VERDICT", &verdict_path)
1163            .env("FORGE_PROPERTIES_USAGE", &usage_path)
1164            .stdin(Stdio::null())
1165            .stdout(log.try_clone()?)
1166            .stderr(log);
1167        let status = run_session(&mut command, self.args.timeout, "verifier")?;
1168        let usage = read_usage(&usage_path, &format!("verifier {id}"))?;
1169
1170        let mut errors = Vec::new();
1171        match status {
1172            None => errors.push("the verifier reached the timeout".to_string()),
1173            Some(status) if !status.success() => {
1174                errors.push(format!("verifier command failed ({status})"))
1175            }
1176            Some(_) => {}
1177        }
1178        let judgment = match fs::read(&verdict_path) {
1179            Ok(verdict) => match serde_json::from_slice::<Judgment>(&verdict) {
1180                // A disproof needs a quote that the group can check.
1181                Ok(judgment)
1182                    if judgment.verdict == Verdict::Rejected && judgment.evidence.is_empty() =>
1183                {
1184                    errors.push("the verifier rejected the group without evidence".to_string());
1185                    Judgment { verdict: Verdict::NeedsValidation, ..judgment }
1186                }
1187                Ok(judgment) => judgment,
1188                Err(error) => {
1189                    errors.push(format!("the verifier wrote an invalid verdict: {error}"));
1190                    Judgment::default()
1191                }
1192            },
1193            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1194                errors.push("the verifier wrote no verdict".to_string());
1195                Judgment::default()
1196            }
1197            Err(error) => return Err(error).wrap_err("failed to read verifier verdict"),
1198        };
1199        let log_display = log_path.strip_prefix(&self.config.root).unwrap_or(&log_path);
1200        let error = (!errors.is_empty())
1201            .then(|| format!("{}: see {}", errors.join("; "), log_display.display()));
1202        Ok(Verification { judgment, error, usage })
1203    }
1204}
1205
1206/// Groups the failing tests of all agents by root cause with the root-cause matcher. Groups with a
1207/// claim come first, then groups with more agents.
1208fn group_failures(failures: Vec<AgentFailure>) -> Vec<BugGroup> {
1209    let claims =
1210        failures.iter().map(|failure| failure.failure.claim.as_deref()).collect::<Vec<_>>();
1211    let group_of = matcher::root_cause_groups(&claims);
1212    let mut groups = BTreeMap::<usize, Vec<AgentFailure>>::new();
1213    for (failure, group) in failures.into_iter().zip(group_of) {
1214        groups.entry(group).or_default().push(failure);
1215    }
1216    let mut groups = groups
1217        .into_values()
1218        .map(|mut failures| {
1219            // A failure with a claim leads, then the failures by agent.
1220            failures.sort_by(|a, b| {
1221                let key = |failure: &AgentFailure| {
1222                    (failure.failure.claim.is_none(), failure.agent, failure.failure.test.clone())
1223                };
1224                key(a).cmp(&key(b))
1225            });
1226            let agents = failures.iter().map(|failure| failure.agent).collect::<BTreeSet<_>>();
1227            BugGroup { agents: agents.into_iter().collect(), failures, verification: None }
1228        })
1229        .collect::<Vec<_>>();
1230    groups.sort_by(|a, b| {
1231        let key = |group: &BugGroup| {
1232            let lead = &group.failures[0].failure;
1233            (lead.claim.is_none(), Reverse(group.agents.len()), lead.test.clone())
1234        };
1235        key(a).cmp(&key(b))
1236    });
1237    groups
1238}
1239
1240/// Returns the task of an agent session. On 26 held-out cases, plain independent small-model
1241/// audits with a task of this kind found 0.05 to 0.10 more planted bugs at every spend level than
1242/// agents that returned candidate JSON in short rounds.
1243fn session_prompt(
1244    targets: &[PathBuf],
1245    context: &[PathBuf],
1246    out: &Path,
1247    focus: Option<&str>,
1248    distinct: bool,
1249) -> String {
1250    let list = |paths: &[PathBuf]| {
1251        paths.iter().map(|path| format!("- {}\n", path.display())).collect::<String>()
1252    };
1253    let mut prompt = format!(
1254        "Find bugs in these targets of the Foundry project in the current directory:\n{}",
1255        list(targets)
1256    );
1257    if !context.is_empty() {
1258        prompt.push_str(&format!(
1259            "\nThese context files state intended behavior:\n{}",
1260            list(context)
1261        ));
1262    }
1263    prompt.push_str(&format!(
1264        "\nProve each bug with a Foundry test in {} that fails because of the bug and encodes the \
1265         intended behavior. The code under test can be wrong: the intended behavior is what its \
1266         documentation, its interfaces, and the standards it claims state. Name each test after \
1267         the bug. Right above each test, write exactly one comment line in this form:\n\n\
1268         // @bug <Contract>.<function> <file>:<line> — <claim>\n\n\
1269         <Contract>.<function> is the target function that has the bug, <file>:<line> is the \
1270         target source line that breaks the intended behavior, and <claim> is one sentence. Use \
1271         `//`, not `///`: the compiler rejects `@bug` as a NatSpec tag. You may run `forge test` \
1272         freely.\n",
1273        out.display()
1274    ));
1275    if let Some(focus) = focus {
1276        let others = if distinct { " (the other agents have other focuses)" } else { "" };
1277        prompt.push_str(&format!(
1278            "\nYour focus{others}: {focus} Trace a small number of concrete high-signal paths in \
1279             this area first; then widen if budget is left.\n"
1280        ));
1281    }
1282    prompt
1283}
1284
1285/// Returns the Solidity files under the output directory `out` of a workspace, by path relative
1286/// to the workspace, in path order. Links are not followed, and `out` itself must not be one. A
1287/// file whose path has a glob character is rejected: Forge selects the files to run by a glob.
1288fn collect_tests(
1289    workspace: &Path,
1290    out: &Path,
1291    agent: usize,
1292    rejected: &mut Vec<Rejected>,
1293) -> Result<Vec<TestFile>> {
1294    let out_dir = workspace.join(out);
1295    ensure_inside(workspace, &out_dir, out)?;
1296    match fs::symlink_metadata(&out_dir) {
1297        Ok(metadata) => ensure!(
1298            metadata.is_dir(),
1299            "the generated test directory {} is not a directory",
1300            out.display()
1301        ),
1302        // The agent removed it.
1303        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(vec![]),
1304        Err(error) => {
1305            return Err(error).wrap_err_with(|| format!("failed to read {}", out.display()));
1306        }
1307    }
1308    let mut files = Vec::new();
1309    let mut pending = vec![out.to_path_buf()];
1310    while let Some(dir) = pending.pop() {
1311        for entry in fs::read_dir(workspace.join(&dir))? {
1312            let entry = entry?;
1313            let path = dir.join(entry.file_name());
1314            let file_type = entry.file_type()?;
1315            if file_type.is_dir() {
1316                pending.push(path);
1317                continue;
1318            }
1319            if !file_type.is_file() || path.extension().is_none_or(|extension| extension != "sol") {
1320                continue;
1321            }
1322            let glob = ['*', '?', '[', ']', '{', '}', ',', '\\'];
1323            if path.iter().any(|component| component.to_string_lossy().contains(glob)) {
1324                let reason = "the path has a glob character".to_string();
1325                rejected.push(Rejected { agent, paths: vec![path], reason });
1326                continue;
1327            }
1328            match fs::read_to_string(entry.path()) {
1329                Ok(content) => files.push(TestFile { path, content }),
1330                Err(error) => {
1331                    let reason = error.to_string();
1332                    rejected.push(Rejected { agent, paths: vec![path], reason });
1333                }
1334            }
1335        }
1336    }
1337    files.sort_by(|a, b| a.path.cmp(&b.path));
1338    Ok(files)
1339}
1340
1341/// Compares Solidity tokens without comments or whitespace, preserving literal bytes and token
1342/// boundaries so different expressions cannot collapse into the same comparison key.
1343fn normalize_test_content(source: &str) -> Vec<&str> {
1344    Cursor::new(source)
1345        .with_position()
1346        .filter(|(_, token)| !token.kind.is_trivial())
1347        .map(|(pos, token)| &source[pos..pos + token.len as usize])
1348        .collect()
1349}
1350
1351/// Returns the comment right above a test function, without comment markers, as one line. The
1352/// function is the first one with this name after the contract declaration and before the next
1353/// one. Declarations are searched outside comments, so a claim such as `the contract pays twice`
1354/// is not a declaration.
1355fn test_claim(source: &str, contract: &str, function: &str) -> Option<String> {
1356    // Blanking each comment byte keeps the offsets of the code.
1357    let code = COMMENT.replace_all(source, |comment: &regex::Captures<'_>| {
1358        comment[0].bytes().map(|byte| if byte == b'\n' { '\n' } else { ' ' }).collect::<String>()
1359    });
1360    let mut contracts = CONTRACT.captures_iter(&code);
1361    let start = contracts.find(|captures| &captures[1] == contract)?.get(0)?.end();
1362    let end = contracts.next().map_or(code.len(), |captures| captures.get(0).unwrap().start());
1363    let declaration = Regex::new(&format!(r"\bfunction\s+{}\s*\(", regex::escape(function)))
1364        .ok()?
1365        .find(&code[start..end])?;
1366    let line_start = source[..start + declaration.start()].rfind('\n').map_or(0, |i| i + 1);
1367    let mut lines = source[..line_start]
1368        .lines()
1369        .rev()
1370        .map(str::trim)
1371        .take_while(|line| {
1372            line.starts_with("//") || line.starts_with('*') || line.starts_with("/*")
1373        })
1374        .filter(|line| !line.contains("forge-config:"))
1375        .map(|line| line.trim_start_matches(['/', '*']).trim_end_matches(['*', '/']).trim())
1376        .map(|line| {
1377            match line.strip_prefix('@').map(|tagged| tagged.split_once(char::is_whitespace)) {
1378                // The `@bug` tag names the target, so it stays. A NatSpec tag such as `@notice` is
1379                // not part of the claim. The compiler rejects `/// @bug`, so
1380                // `@custom:bug` stays too.
1381                Some(Some((tag, text))) if !matches!(tag, "bug" | "custom:bug") => text,
1382                Some(None) => "",
1383                _ => line,
1384            }
1385        })
1386        .filter(|line| !line.is_empty())
1387        .collect::<Vec<_>>();
1388    lines.reverse();
1389    // The claim starts at the tag, wherever the comment has it.
1390    if let Some(tag) = lines.iter().position(|line| {
1391        line.split_whitespace().next().is_some_and(|tag| matches!(tag, "@bug" | "@custom:bug"))
1392    }) {
1393        lines.drain(..tag);
1394    }
1395    (!lines.is_empty()).then(|| lines.join(" "))
1396}
1397
1398/// Returns the collected files that each collected file imports, directly or not.
1399fn collected_imports(
1400    config: &Config,
1401    workspace: &Path,
1402    files: &[TestFile],
1403) -> Result<BTreeMap<PathBuf, BTreeSet<PathBuf>>> {
1404    let rebased = workspace::rebase_config_paths(config, workspace);
1405    let graph =
1406        Graph::<MultiCompilerParser>::resolve(&rebased.project_paths::<MultiCompilerLanguage>())
1407            .wrap_err("failed to resolve generated test imports")?;
1408    let paths = files.iter().map(|file| file.path.clone()).collect::<HashSet<_>>();
1409    let mut imports = BTreeMap::new();
1410    for (index, node) in graph.nodes.iter().enumerate() {
1411        let path = workspace::relative_to_root(workspace, node.path());
1412        if paths.contains(&path) {
1413            let imported = graph
1414                .all_imported_nodes(index)
1415                .map(|index| workspace::relative_to_root(workspace, graph.node(index).path()))
1416                .filter(|path| paths.contains(path))
1417                .collect();
1418            imports.insert(path, imported);
1419        }
1420    }
1421    Ok(imports)
1422}
1423
1424/// Returns the files whose tests all pass on every seed, with the files that they import. A
1425/// passing file that imports a file with a failing test is not kept.
1426fn kept_files(
1427    files: &[TestFile],
1428    imports: &BTreeMap<PathBuf, BTreeSet<PathBuf>>,
1429    run: &TestRun,
1430) -> Result<HashSet<PathBuf>> {
1431    let passing = files
1432        .iter()
1433        .map(|file| &file.path)
1434        .filter(|path| {
1435            !run.failing_files.contains(*path)
1436                && run.passing.iter().any(|test| test_path(test) == **path)
1437        })
1438        .collect::<Vec<_>>();
1439    if passing.is_empty() {
1440        return Ok(HashSet::new());
1441    }
1442    if passing.len() == files.len() {
1443        return Ok(passing.into_iter().cloned().collect());
1444    }
1445    let mut kept = HashSet::new();
1446    for path in passing {
1447        let imported = imports.get(path).ok_or_else(|| {
1448            eyre!("passing test file {} is absent from the import graph", path.display())
1449        })?;
1450        if imported.iter().all(|path| !run.failing_files.contains(path)) {
1451            kept.insert(path.clone());
1452            kept.extend(imported.iter().cloned());
1453        }
1454    }
1455    Ok(kept)
1456}
1457
1458/// Returns the size and modification time of each file that workspaces share with the project, by
1459/// path relative to the root: the files in the `dependencies` directories, whose entries workspaces
1460/// link, and behind links that lead outside the project. Workspaces copy every other file, so a
1461/// change to one cannot come from an agent. Hidden entries and the build and cache directories are
1462/// skipped. Each directory is read once.
1463fn project_fingerprint(config: &Config, dependencies: &[PathBuf]) -> Fingerprint {
1464    let mut files = BTreeMap::new();
1465    let mut read = HashSet::new();
1466    // Each directory, and whether workspaces share its files.
1467    let mut pending = vec![(config.root.clone(), false)];
1468    while let Some((dir, shared)) = pending.pop() {
1469        if !fs::canonicalize(&dir).is_ok_and(|dir| read.insert(dir)) {
1470            continue;
1471        }
1472        let Ok(entries) = fs::read_dir(&dir) else { continue };
1473        for entry in entries.flatten() {
1474            let path = entry.path();
1475            if entry.file_name().to_string_lossy().starts_with('.')
1476                || path == config.out
1477                || path == config.cache_path
1478            {
1479                continue;
1480            }
1481            let shared = shared
1482                || dependencies.contains(&path)
1483                || (entry.file_type().is_ok_and(|file_type| file_type.is_symlink())
1484                    && fs::canonicalize(&path)
1485                        .is_ok_and(|target| !target.starts_with(&config.root)));
1486            let Ok(metadata) = (if shared { fs::metadata(&path) } else { entry.metadata() }) else {
1487                continue;
1488            };
1489            if metadata.is_dir() {
1490                pending.push((path, shared));
1491            } else if shared && metadata.is_file() {
1492                // The path through a link, not the link's target.
1493                let path = path.strip_prefix(&config.root).unwrap_or(&path).to_path_buf();
1494                files.insert(path, (metadata.len(), metadata.modified().ok()));
1495            }
1496        }
1497    }
1498    files
1499}
1500
1501/// Returns every test result in Forge's JSON output, with one result per invariant predicate of a
1502/// merged invariant campaign.
1503fn test_results(output: &Output) -> Result<Vec<(RerunFailure, TestStatus, Option<String>)>> {
1504    let mut results = Vec::new();
1505    for (contract, test, result) in suite_results(output)? {
1506        let key = |name: &str| RerunFailure { contract: contract.clone(), test: name.into() };
1507        // Handler assertion failures belong to the whole campaign, not to one predicate.
1508        let handler_failures = result
1509            .invariant_handler_failures
1510            .iter()
1511            .chain(result.invariant_failures.iter().filter(|f| f.predicate_name().is_none()))
1512            .map(|failure| {
1513                format!(
1514                    "handler {}: {}",
1515                    failure.name(),
1516                    failure_reason(failure.reason(), failure.counterexample())
1517                )
1518            })
1519            .collect::<Vec<_>>();
1520        if result.invariant_predicate_results.is_empty() {
1521            let reason = result.reason.clone().or_else(|| {
1522                let reasons = result
1523                    .invariant_failures
1524                    .iter()
1525                    .filter(|failure| failure.predicate_name().is_some())
1526                    .map(|failure| failure_reason(failure.reason(), failure.counterexample()))
1527                    .chain(handler_failures.iter().cloned())
1528                    .collect::<Vec<_>>();
1529                (!reasons.is_empty()).then(|| reasons.join("; "))
1530            });
1531            let reason = match (reason, result.counterexample.as_ref()) {
1532                (Some(reason), counterexample) => Some(failure_reason(&reason, counterexample)),
1533                (None, Some(counterexample)) => Some(failure_reason("", Some(counterexample))),
1534                (None, None) => None,
1535            };
1536            let status =
1537                if handler_failures.is_empty() { result.status } else { TestStatus::Failure };
1538            results.push((key(&test), status, reason));
1539            continue;
1540        }
1541        for predicate in &result.invariant_predicate_results {
1542            let failure = result
1543                .invariant_failures
1544                .iter()
1545                .find(|failure| failure.predicate_name() == Some(predicate.name.as_str()));
1546            let reason = predicate.reason.as_deref().or(failure.map(|failure| failure.reason()));
1547            let reason = reason.map(|reason| {
1548                failure_reason(reason, failure.and_then(|failure| failure.counterexample()))
1549            });
1550            results.push((key(&predicate.name), predicate.status, reason));
1551        }
1552        // One campaign result per seed, so the count of failing seeds stays correct.
1553        if !handler_failures.is_empty() {
1554            let reason = Some(handler_failures.join("; "));
1555            results.push((key(&test), TestStatus::Failure, reason));
1556        }
1557    }
1558    Ok(results)
1559}
1560
1561/// Appends the counterexample calls to a failure reason.
1562fn failure_reason(reason: &str, counterexample: Option<&CounterExample>) -> String {
1563    let Some(counterexample) = counterexample else { return reason.to_string() };
1564    let calls = match counterexample {
1565        CounterExample::Single(call) => vec![call],
1566        CounterExample::Sequence(_, calls) => calls.iter().collect(),
1567    };
1568    let calls = calls.iter().map(|call| call.to_string().trim().to_string()).collect::<Vec<_>>();
1569    let counterexample = format!("counterexample: {}", calls.join(", "));
1570    if reason.is_empty() { counterexample } else { format!("{reason}; {counterexample}") }
1571}
1572
1573/// Returns `Contract::test` for a test identity.
1574fn test_display_name(test: &RerunFailure) -> String {
1575    let contract = test.contract.rsplit_once(':').map_or(test.contract.as_str(), |(_, name)| name);
1576    let name = test.test.split_once('(').map_or(test.test.as_str(), |(name, _)| name);
1577    format!("{contract}::{name}")
1578}
1579
1580/// Returns the file of a test identity.
1581fn test_path(test: &RerunFailure) -> PathBuf {
1582    PathBuf::from(test.contract.rsplit_once(':').map_or(test.contract.as_str(), |(path, _)| path))
1583}
1584
1585/// Returns the directory for one seed's builds and persisted state in a workspace.
1586fn seed_dir(config: &Config, workspace: &Path, seed: &U256) -> PathBuf {
1587    workspace
1588        .join(workspace::relative_to_root(&config.root, &config.cache_path))
1589        .join("properties/seeds")
1590        .join(format!("{seed:#x}"))
1591}
1592
1593/// Returns the task of a verifier session.
1594fn verify_prompt(
1595    group: &BugGroup,
1596    targets: &[PathBuf],
1597    group_path: &Path,
1598    verdict: &Path,
1599) -> String {
1600    let targets = targets.iter().map(|path| format!("- {}\n", path.display())).collect::<String>();
1601    let members = group
1602        .failures
1603        .iter()
1604        .map(|failure| {
1605            let claim = failure.failure.claim.as_deref().unwrap_or("(none)");
1606            let reason = failure.failure.reason.trim().chars().take(600).collect::<String>();
1607            format!(
1608                "- file: {}\n  test: {}\n  claim: {claim}\n  failure: {reason}\n",
1609                failure.failure.path.display(),
1610                failure.failure.test
1611            )
1612        })
1613        .collect::<Vec<_>>()
1614        .join("\n");
1615    // The tests come last, so a placeholder in a failure reason stays as it is.
1616    VERIFY_PROMPT
1617        .replace("{targets}", &targets)
1618        .replace("{group}", &group_path.display().to_string())
1619        .replace("{verdict}", &verdict.display().to_string())
1620        .replace("{members}", &members)
1621}
1622
1623/// Returns the files of a group: the files of its failing tests, with the files of the same agent
1624/// that they import, directly or not.
1625fn group_files(sessions: &[Session], group: &BugGroup) -> Vec<TestFile> {
1626    let mut files = Vec::new();
1627    for session in sessions.iter().filter(|session| group.agents.contains(&session.id)) {
1628        let failing = group
1629            .failures
1630            .iter()
1631            .filter(|failure| failure.agent == session.id)
1632            .map(|failure| &failure.failure.path)
1633            .collect::<BTreeSet<_>>();
1634        let paths = failing
1635            .iter()
1636            .copied()
1637            .chain(failing.iter().filter_map(|path| session.imports.get(*path)).flatten())
1638            .collect::<BTreeSet<_>>();
1639        files.extend(session.files.iter().filter(|file| paths.contains(&file.path)).cloned());
1640    }
1641    files
1642}
1643
1644/// Reads the usage file of an agent or verifier. A missing file is no usage, and an invalid one
1645/// is a warning.
1646fn read_usage(path: &Path, session: &str) -> Result<Option<Usage>> {
1647    match fs::read(path) {
1648        Ok(usage) => Ok(serde_json::from_slice::<Usage>(&usage)
1649            .inspect_err(|error| {
1650                let _ = sh_warn!("{session} wrote invalid usage JSON: {error}");
1651            })
1652            .ok()),
1653        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
1654        Err(error) => Err(error).wrap_err_with(|| format!("failed to read {session} usage")),
1655    }
1656}
1657
1658#[cfg(test)]
1659mod tests {
1660    use super::*;
1661
1662    fn failure(agent: usize, test: &str, claim: &str) -> AgentFailure {
1663        AgentFailure {
1664            agent,
1665            failure: Failure {
1666                test: test.to_string(),
1667                path: PathBuf::from("test/generated/Fee.t.sol"),
1668                reason: "revert".to_string(),
1669                claim: (!claim.is_empty()).then(|| claim.to_string()),
1670            },
1671            file: PathBuf::new(),
1672        }
1673    }
1674
1675    // Agreeing failures form one group with the set of their agents. Groups with a claim and more
1676    // agents come first; a failure without a claim agrees with none.
1677    #[test]
1678    fn groups_link_agreeing_failures_by_agent() {
1679        let groups = group_failures(vec![
1680            failure(3, "C::zero", "A zero amount is charged; src/Fee.sol:11 charges()."),
1681            failure(1, "A::cap", "The fee is capped at 100 ether; src/Fee.sol:7 fee()."),
1682            failure(2, "B::cap", "Large amounts pay the maximum; src/Fee.sol:7 fee()."),
1683            failure(2, "B::limit", "Large amounts pay the maximum fee (src/Fee.sol:8)."),
1684            failure(1, "D::none", ""),
1685        ]);
1686        let groups = groups
1687            .iter()
1688            .map(|group| {
1689                let tests = group.failures.iter().map(|failure| failure.failure.test.as_str());
1690                (group.agents.clone(), tests.collect::<Vec<_>>())
1691            })
1692            .collect::<Vec<_>>();
1693        assert_eq!(
1694            groups,
1695            [
1696                (vec![1, 2], vec!["A::cap", "B::cap"]),
1697                // It agrees with `B::cap` but not with `A::cap`, so it does not join their group.
1698                (vec![2], vec!["B::limit"]),
1699                (vec![3], vec!["C::zero"]),
1700                (vec![1], vec!["D::none"]),
1701            ]
1702        );
1703    }
1704
1705    // The claim is the comment right above the test function of the named contract, without
1706    // comment markers, NatSpec tags other than the bug tag, or inline config. A test without such a
1707    // comment has none, and the word `contract` in a comment declares no contract.
1708    #[test]
1709    fn test_claim_reads_the_comment_above_the_test() {
1710        let source = r#"
1711pragma solidity ^0.8.20;
1712contract FeeTest {
1713    // The fee is capped at 100 ether (src/Fee.sol:7).
1714    function testCap() public {}
1715
1716    /// @notice A zero amount is charged,
1717    /// src/Fee.sol:11.
1718    /// forge-config: default.fuzz.runs = 10
1719    function testZero(uint256 amount) public {}
1720
1721    function testBare() public {}
1722
1723    // @bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.
1724    function testTagged() public {}
1725
1726    /// @custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.
1727    function testCustomTag() public {}
1728
1729    /// @notice Large amounts.
1730    /// @custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether,
1731    /// not 100 ether.
1732    function testLateTag() public {}
1733
1734    // @bug Fee.pay src/Fee.sol:20 — the contract pays twice.
1735    function testPaysTwice() public {}
1736}
1737contract OtherTest {
1738    /* Rounds down: src/Fee.sol:6 */
1739    function testCap() public {}
1740}
1741"#;
1742        assert_eq!(
1743            test_claim(source, "FeeTest", "testCap").as_deref(),
1744            Some("The fee is capped at 100 ether (src/Fee.sol:7).")
1745        );
1746        assert_eq!(
1747            test_claim(source, "FeeTest", "testZero").as_deref(),
1748            Some("A zero amount is charged, src/Fee.sol:11.")
1749        );
1750        assert_eq!(test_claim(source, "FeeTest", "testBare"), None);
1751        assert_eq!(
1752            test_claim(source, "FeeTest", "testTagged").as_deref(),
1753            Some("@bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.")
1754        );
1755        assert_eq!(
1756            test_claim(source, "FeeTest", "testCustomTag").as_deref(),
1757            Some("@custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether.")
1758        );
1759        assert_eq!(
1760            test_claim(source, "FeeTest", "testLateTag").as_deref(),
1761            Some("@custom:bug Fee.fee src/Fee.sol:7 — the cap is 10 ether, not 100 ether.")
1762        );
1763        assert_eq!(
1764            test_claim(source, "FeeTest", "testPaysTwice").as_deref(),
1765            Some("@bug Fee.pay src/Fee.sol:20 — the contract pays twice.")
1766        );
1767        assert_eq!(
1768            test_claim(source, "OtherTest", "testCap").as_deref(),
1769            Some("Rounds down: src/Fee.sol:6")
1770        );
1771        assert_eq!(test_claim(source, "OtherTest", "testZero"), None);
1772        assert_eq!(test_claim(source, "MissingTest", "testCap"), None);
1773    }
1774
1775    // Forge collects every Solidity file under the output directory, also in subdirectories, in
1776    // path order. It skips other files and links, and rejects files that set a seed, `ffi`, or
1777    // `fs_permissions` in inline config.
1778    #[test]
1779    fn collect_tests_reads_solidity_files_under_the_output_directory() {
1780        let workspace = tempfile::tempdir().unwrap();
1781        let out = Path::new("test/generated/agent-1");
1782        let write = |path: &str, content: &str| {
1783            let path = workspace.path().join(out).join(path);
1784            fs::create_dir_all(path.parent().unwrap()).unwrap();
1785            fs::write(path, content).unwrap();
1786        };
1787        write("Fee.t.sol", "contract FeeTest {}");
1788        write("helpers/Helper.sol", "library Helper {}");
1789        write("notes.md", "notes");
1790        write("A{B,C}.t.sol", "contract GlobTest {}");
1791        // Windows does not allow `*` in file names.
1792        #[cfg(unix)]
1793        write("dir*/Star.t.sol", "contract StarTest {}");
1794        #[cfg(unix)]
1795        std::os::unix::fs::symlink(
1796            workspace.path().join(out).join("Fee.t.sol"),
1797            workspace.path().join(out).join("Link.t.sol"),
1798        )
1799        .unwrap();
1800
1801        let mut rejected = Vec::new();
1802        let files = collect_tests(workspace.path(), out, 1, &mut rejected).unwrap();
1803        // Paths compare by component, so the expected paths hold on Windows too.
1804        assert_eq!(
1805            files
1806                .iter()
1807                .map(|file| (file.path.as_path(), file.content.as_str()))
1808                .collect::<Vec<_>>(),
1809            [
1810                (Path::new("test/generated/agent-1/Fee.t.sol"), "contract FeeTest {}"),
1811                (Path::new("test/generated/agent-1/helpers/Helper.sol"), "library Helper {}"),
1812            ]
1813        );
1814        rejected.sort_by(|a, b| a.paths.cmp(&b.paths));
1815        assert_eq!(
1816            rejected
1817                .iter()
1818                .map(|rejected| (rejected.paths[0].as_path(), rejected.reason.as_str()))
1819                .collect::<Vec<_>>(),
1820            ["test/generated/agent-1/A{B,C}.t.sol"]
1821                .into_iter()
1822                .chain(cfg!(unix).then_some("test/generated/agent-1/dir*/Star.t.sol"))
1823                .map(|path| (Path::new(path), "the path has a glob character"))
1824                .collect::<Vec<_>>()
1825        );
1826
1827        // The output directory itself must not be a link.
1828        #[cfg(unix)]
1829        {
1830            let linked = Path::new("test/generated/agent-2");
1831            std::os::unix::fs::symlink(workspace.path().join(out), workspace.path().join(linked))
1832                .unwrap();
1833            let error = collect_tests(workspace.path(), linked, 2, &mut rejected).unwrap_err();
1834            assert!(error.to_string().contains("is not a directory"), "{error}");
1835        }
1836    }
1837
1838    #[test]
1839    fn normalize_test_content_keeps_string_literals_and_strips_comments() {
1840        let with_space = r#"
1841            // Fixture case.
1842            contract EchoSpecTest {
1843              function testEcho() public pure {
1844                require(keccak256(bytes(Echo.echo("a b"))) == keccak256(bytes("a b")), "echo changed");
1845              }
1846            }
1847        "#;
1848        let without_space = r#"
1849            // Other fixture.
1850            contract EchoSpecTest {
1851              function testEcho() public pure {
1852                require(keccak256(bytes(Echo.echo("ab"))) == keccak256(bytes("ab")), "echo changed");
1853              }
1854            }
1855        "#;
1856        let url_one = r#"require(Echo.echo("https://one.example") == "https://one.example");"#;
1857        let url_two = r#"require(Echo.echo("https://two.example") == "https://two.example");"#;
1858        let comment_only_a =
1859            "contract T { function testA() public pure { /* note */ require(true); } }";
1860        let comment_only_b =
1861            "contract T { function testA() public pure { require(true); // note\n } }";
1862
1863        assert_ne!(normalize_test_content(with_space), normalize_test_content(without_space));
1864        assert_ne!(normalize_test_content(url_one), normalize_test_content(url_two));
1865        assert_eq!(normalize_test_content(comment_only_a), normalize_test_content(comment_only_b));
1866        for literal in
1867            [r#""https://one.example""#, r#"'a\' /* b */'"#, r#"unicode"été //""#, r#"hex"abcd""#]
1868        {
1869            assert_eq!(normalize_test_content(literal), [literal]);
1870        }
1871        assert_ne!(normalize_test_content("a + ++b"), normalize_test_content("a++ + b"));
1872    }
1873}