Skip to main content

forge/cmd/
exploit.rs

1use clap::{Parser, ValueHint};
2use eyre::{Context, Result, eyre};
3use foundry_cli::{opts::BuildOpts, utils::LoadConfig};
4use foundry_common::{provider::mpp::transport::LazyAccountsProvider, shell};
5use foundry_compilers::utils::source_files_iter;
6use foundry_config::impl_figment_convert;
7use foundry_evm::constants::{CHEATCODE_ADDRESS, HARDHAT_CONSOLE_ADDRESS, MONAD_CHEATCODE_ADDRESS};
8use mpp::client::Fetch;
9use serde::Serialize;
10use serde_json::{Value, json};
11use std::{
12    fmt::Write as _,
13    io::Write as _,
14    path::{Path, PathBuf},
15};
16use url::Url;
17
18use crate::{
19    result::{TestResult, TestStatus},
20    sandbox::{Sandbox, TestFile, shorten, stderr, suite_results},
21    traces::{CallKind, TraceKind},
22    workspace::relative_to_root,
23};
24
25/// Tempo Gateway that serves OpenAI-compatible chat completions paid over MPP.
26const DEFAULT_ENDPOINT: &str = "https://gateway.tempo-dev.workers.dev";
27
28/// Model used when none is given.
29const DEFAULT_MODEL: &str = "@cf/zai-org/glm-5.3";
30
31/// Reply that ends an attempt without a finding.
32const NO_EXPLOIT: &str = "NO_EXPLOIT";
33
34/// Longest source file sent to the model.
35const MAX_SOURCE_BYTES: usize = 64_000;
36
37/// CLI arguments for `forge exploit`.
38///
39/// Asks a model served by the Tempo Gateway to find a vulnerability in a contract and prove it
40/// with a Foundry test. Forge compiles and runs every test the model writes and sends compiler
41/// errors and failing results back until a test passes. Requests are paid through the Machine
42/// Payments Protocol from the Tempo account that `cast tempo login` authorized.
43///
44/// The output of every test goes back to the model, so Forge runs the tests as `forge properties`
45/// runs generated tests: in a temporary copy of the project, without `ffi`, network access, RPC
46/// and Etherscan settings, or Forge's environment variables. A test that passes is written to
47/// `test/exploit/` under a name that no file there has yet.
48#[derive(Clone, Debug, Parser)]
49pub struct ExploitArgs {
50    /// The contract source to attack.
51    #[arg(long, value_hint = ValueHint::FilePath, value_name = "PATH")]
52    target: PathBuf,
53
54    /// Extra files for the model, such as a specification or related contracts.
55    #[arg(long, value_hint = ValueHint::FilePath, value_name = "PATH")]
56    context: Vec<PathBuf>,
57
58    /// Gateway serving `/v1/chat/completions` over MPP.
59    #[arg(long, env = "FORGE_EXPLOIT_ENDPOINT", default_value = DEFAULT_ENDPOINT)]
60    endpoint: Url,
61
62    /// Model to ask.
63    #[arg(long, env = "FORGE_EXPLOIT_MODEL", default_value = DEFAULT_MODEL)]
64    model: String,
65
66    /// Independent attempts; each later attempt is told which bugs were already proven.
67    #[arg(long, default_value_t = 3)]
68    attempts: usize,
69
70    /// Model replies per attempt, counting each retry after a compiler error or failing test.
71    #[arg(long, default_value_t = 4)]
72    turns: usize,
73
74    /// Most tokens per model reply.
75    #[arg(long, default_value_t = 8_192)]
76    max_tokens: u32,
77
78    #[command(flatten)]
79    build: BuildOpts,
80}
81
82impl_figment_convert!(ExploitArgs, build);
83
84/// A test the model wrote that passes, proving its claimed bug.
85#[derive(Debug, Serialize)]
86struct Finding {
87    attempt: usize,
88    turns: usize,
89    file: PathBuf,
90    tests: Vec<String>,
91    bugs: Vec<String>,
92}
93
94#[derive(Debug, Serialize)]
95struct Report<'a> {
96    target: &'a Path,
97    endpoint: &'a str,
98    model: &'a str,
99    findings: &'a [Finding],
100}
101
102enum Run {
103    Proven(Vec<String>),
104    Rejected(String),
105}
106
107impl ExploitArgs {
108    pub async fn run(self) -> Result<()> {
109        let config = self.load_config()?;
110        let root = config.root.clone();
111        let target = root.join(&self.target);
112        let target_rel = relative_to_root(&root, &target);
113        let target_source = read_source(&target)?;
114        let mut context = String::new();
115        for path in &self.context {
116            let path = root.join(path);
117            write!(
118                context,
119                "\n\n--- {} ---\n{}",
120                relative_to_root(&root, &path).display(),
121                read_source(&path)?
122            )?;
123        }
124        let remappings =
125            config.get_all_remappings().map(|r| r.to_string()).collect::<Vec<_>>().join("\n");
126        let mut sources = source_files_iter(&config.src, &["sol"])
127            .map(|path| relative_to_root(&root, &path).display().to_string())
128            .collect::<Vec<_>>();
129        sources.sort();
130        let sources = sources.join("\n");
131        let mut sandbox = Sandbox::new(&config, vec![], false, false)?;
132        sandbox.end_on_interrupt(Vec::new())?;
133        if let Some(errors) = sandbox.build()? {
134            return Err(eyre!(
135                "the project must compile before Forge looks for exploits: {errors}"
136            ));
137        }
138
139        let out_dir = config.test.join("exploit");
140        let chat = Chat::new(&self.endpoint, &self.model, self.max_tokens)?;
141        let mut findings = Vec::<Finding>::new();
142        let mut next = 1;
143
144        for attempt in 1..=self.attempts {
145            // Proofs of earlier runs keep their files: each attempt names its test after the first
146            // number that no file has.
147            while out_dir.join(format!("Exploit{next}Test.t.sol")).exists() {
148                next += 1;
149            }
150            let contract = format!("Exploit{next}Test");
151            let file = out_dir.join(format!("{contract}.t.sol"));
152            let file_rel = relative_to_root(&root, &file);
153            let mut prompt = format!(
154                "Target: {target}\n\n{target_source}{context}\n\nProject sources:\n{sources}\n\n\
155                 Remappings:\n{remappings}\n\nWrite the test file at {file}. Name the test \
156                 contract {contract}.",
157                target = target_rel.display(),
158                file = file_rel.display(),
159            );
160            if !findings.is_empty() {
161                prompt.push_str("\n\nThese bugs are already proven; find a different one:\n");
162                for bug in findings.iter().flat_map(|f| &f.bugs) {
163                    writeln!(prompt, "{bug}")?;
164                }
165            }
166            let mut messages = vec![
167                json!({"role": "system", "content": SYSTEM}),
168                json!({"role": "user", "content": prompt}),
169            ];
170
171            sh_status!("Attempt {attempt}/{}: asking {}", self.attempts, self.model)?;
172            for turn in 1..=self.turns {
173                let reply = chat.complete(&messages).await?;
174                messages.push(json!({"role": "assistant", "content": reply}));
175
176                let Some(code) = solidity_block(&reply) else {
177                    if reply.contains(NO_EXPLOIT) {
178                        sh_status!("Attempt {attempt}: the model found no exploit")?;
179                    } else {
180                        sh_warn!("Attempt {attempt}: the reply had no Solidity code block")?;
181                    }
182                    break;
183                };
184                let feedback = match run_test(&sandbox, &file_rel, code)? {
185                    Run::Proven(tests) => {
186                        std::fs::create_dir_all(&out_dir)?;
187                        std::fs::OpenOptions::new()
188                            .write(true)
189                            .create_new(true)
190                            .open(&file)
191                            .and_then(|mut target| target.write_all(code.as_bytes()))
192                            .wrap_err_with(|| format!("failed to write {}", file_rel.display()))?;
193                        let bugs = bug_tags(code);
194                        sh_status!("Attempt {attempt}: proven by {}", file_rel.display())?;
195                        findings.push(Finding {
196                            attempt,
197                            turns: turn,
198                            file: file_rel.clone(),
199                            tests,
200                            bugs,
201                        });
202                        break;
203                    }
204                    Run::Rejected(feedback) => feedback,
205                };
206                sh_status!("Attempt {attempt}, turn {turn}: not proven yet")?;
207                messages.push(json!({"role": "user", "content": feedback}));
208            }
209        }
210
211        let report_path = config.cache_path.join("exploit").join("report.json");
212        std::fs::create_dir_all(report_path.parent().unwrap())?;
213        let report = Report {
214            target: &target_rel,
215            endpoint: self.endpoint.as_str(),
216            model: &self.model,
217            findings: &findings,
218        };
219        std::fs::write(&report_path, serde_json::to_string_pretty(&report)?)?;
220
221        if shell::is_json() {
222            sh_println!("{}", serde_json::to_string(&report)?)?;
223        } else {
224            for finding in &findings {
225                for bug in &finding.bugs {
226                    sh_println!("{}\t{bug}", finding.file.display())?;
227                }
228            }
229            sh_status!(
230                "{} of {} attempts proved an exploit; report in {}",
231                findings.len(),
232                self.attempts,
233                relative_to_root(&root, &report_path).display()
234            )?;
235        }
236        Ok(())
237    }
238}
239
240const SYSTEM: &str = "You are a smart contract security researcher auditing a Foundry project. \
241Find one concrete, exploitable vulnerability in the target contract and prove it with a Foundry \
242test.
243
244Reply with exactly one solidity fenced code block holding a complete test file:
245- an SPDX line, a pragma compatible with the target, and imports of forge-std/Test.sol and of the \
246contracts you need, using the project's remappings or paths relative to the test file;
247- one contract with the name you are given, inheriting Test, that deploys everything it needs in \
248setUp();
249- one or more functions named test_exploit_* that pass only when the exploit works: assert what \
250the attacker gained or which guarantee broke, never something that holds anyway;
251- each such function successfully calls a function of a target contract that can change its \
252state; view calls and reverted calls alone do not count;
253- directly above each such function, a comment line \
254// @bug Contract.function path:line - one sentence saying what is wrong;
255- no ffi, no forks, no network access and no environment variables.
256
257When the test does not compile or does not pass you get the output; reply with the whole fixed \
258file. If you find no exploitable vulnerability, reply NO_EXPLOIT and nothing else.";
259
260struct Chat {
261    client: reqwest::Client,
262    provider: LazyAccountsProvider,
263    url: Url,
264    model: String,
265    max_tokens: u32,
266}
267
268impl Chat {
269    fn new(endpoint: &Url, model: &str, max_tokens: u32) -> Result<Self> {
270        let mut base = endpoint.clone();
271        if !base.path().ends_with('/') {
272            base.set_path(&format!("{}/", base.path()));
273        }
274        Ok(Self {
275            client: reqwest::Client::new(),
276            provider: LazyAccountsProvider::new(endpoint.to_string(), true),
277            url: base.join("v1/chat/completions")?,
278            model: model.to_string(),
279            max_tokens,
280        })
281    }
282
283    async fn complete(&self, messages: &[Value]) -> Result<String> {
284        let body = json!({
285            "model": self.model,
286            "messages": messages,
287            "max_tokens": self.max_tokens,
288        });
289        let response = self
290            .client
291            .post(self.url.clone())
292            .json(&body)
293            .send_with_payment(&self.provider)
294            .await
295            .map_err(|e| eyre!("request to {} failed: {e}", self.url))?;
296        let status = response.status();
297        let text = response.text().await?;
298        if !status.is_success() {
299            return Err(eyre!("{} answered {status}: {text}", self.url));
300        }
301        let value: Value =
302            serde_json::from_str(&text).wrap_err_with(|| format!("invalid response: {text}"))?;
303        let message = &value["choices"][0]["message"];
304        message["content"]
305            .as_str()
306            .filter(|content| !content.trim().is_empty())
307            .or_else(|| message["reasoning_content"].as_str())
308            .map(str::to_string)
309            .ok_or_else(|| eyre!("response has no message content: {text}"))
310    }
311}
312
313/// Runs the tests of a model-written file in a fresh copy of the project.
314fn run_test(sandbox: &Sandbox<'_>, file: &Path, code: &str) -> Result<Run> {
315    let test = TestFile { path: file.to_path_buf(), content: code.to_string() };
316    let files = [test];
317    let workspace = sandbox.workspace(&files)?;
318    let mut command = sandbox.test_command(workspace.path(), &files);
319    command
320        // The traces show which contracts each test calls.
321        .arg("-vvv")
322        // A fuzzed exploit must not pass on inputs that revert.
323        .env("FOUNDRY_FUZZ_FAIL_ON_REVERT", "true");
324    let Some(output) = sandbox.output(&mut command)? else {
325        return Ok(Run::Rejected(
326            "The tests reached the timeout; they must not loop without end.".to_string(),
327        ));
328    };
329    if !output.status.success() && output.stdout.is_empty() {
330        return Ok(Run::Rejected(format!(
331            "The test file did not compile or could not run:\n{}",
332            stderr(&output)
333        )));
334    }
335    let results = suite_results(&output)?;
336    let proven = results
337        .iter()
338        .filter(|(_, name, result)| {
339            result.status == TestStatus::Success
340                && name.starts_with("test_exploit")
341                && changes_state(result)
342        })
343        .map(|(_, name, _)| name.clone())
344        .collect::<Vec<_>>();
345    if !proven.is_empty() {
346        return Ok(Run::Proven(proven));
347    }
348
349    let mut feedback = String::from("No test_exploit_* test passed, so nothing is proven:\n");
350    for (_, name, result) in &results {
351        let _ = write!(feedback, "\n{name}: {:?}", result.status);
352        if let Some(reason) = &result.reason {
353            let _ = write!(feedback, " ({reason})");
354        } else if result.status == TestStatus::Success && !changes_state(result) {
355            feedback.push_str(
356                " (makes no successful call that can change state into another contract, so it \
357                 cannot show the exploit)",
358            );
359        }
360        for log in &result.decoded_logs {
361            let _ = write!(feedback, "\n  {log}");
362        }
363    }
364    if results.is_empty() {
365        feedback.push_str("\nNo tests ran; name the functions test_exploit_*.");
366    }
367    Ok(Run::Rejected(shorten(&feedback)))
368}
369
370/// Returns whether a test makes a successful plain call, which can change the callee's state,
371/// into a contract other than the test, the cheatcodes of any network, and the console. A test
372/// that does not, such as an empty one, one that only reads, or one whose calls all revert, passes
373/// whatever the target does. The tests run in another process, so the network's own list of
374/// cheatcode addresses is not at hand.
375fn changes_state(result: &TestResult) -> bool {
376    result.traces.iter().filter(|(kind, _)| *kind == TraceKind::Execution).any(|(_, arena)| {
377        let nodes = arena.arena.nodes();
378        let test = &nodes[0].trace;
379        test.success
380            && nodes[1..].iter().map(|node| &node.trace).any(|trace| {
381                trace.success
382                    && trace.kind == CallKind::Call
383                    && ![
384                        test.address,
385                        CHEATCODE_ADDRESS,
386                        MONAD_CHEATCODE_ADDRESS,
387                        HARDHAT_CONSOLE_ADDRESS,
388                    ]
389                    .contains(&trace.address)
390            })
391    })
392}
393
394fn solidity_block(reply: &str) -> Option<&str> {
395    let start = reply.find("```solidity").or_else(|| reply.find("```sol"))?;
396    let rest = &reply[start..];
397    let body = &rest[rest.find('\n')? + 1..];
398    let end = body.find("```").unwrap_or(body.len());
399    let code = body[..end].trim();
400    (!code.is_empty()).then_some(code)
401}
402
403fn bug_tags(code: &str) -> Vec<String> {
404    code.lines()
405        .filter_map(|line| line.trim().strip_prefix("//"))
406        .map(str::trim)
407        .filter(|line| line.starts_with("@bug"))
408        .map(str::to_string)
409        .collect()
410}
411
412fn read_source(path: &Path) -> Result<String> {
413    let mut source = std::fs::read_to_string(path)
414        .wrap_err_with(|| format!("failed to read {}", path.display()))?;
415    if source.len() > MAX_SOURCE_BYTES {
416        source.truncate(source.floor_char_boundary(MAX_SOURCE_BYTES));
417        source.push_str("\n// [truncated]");
418    }
419    Ok(source)
420}
421
422#[cfg(test)]
423mod tests {
424    use super::*;
425
426    #[test]
427    fn extracts_solidity_block_and_bug_tags() {
428        let reply = "Here:\n```solidity\n// SPDX-License-Identifier: UNLICENSED\ncontract A {\n    // @bug Vault.withdraw src/Vault.sol:12 - no reentrancy guard\n    function test_exploit_reenter() public {}\n}\n```\ndone";
429        let code = solidity_block(reply).unwrap();
430        assert!(code.starts_with("// SPDX"));
431        assert!(code.ends_with('}'));
432        assert_eq!(bug_tags(code), ["@bug Vault.withdraw src/Vault.sol:12 - no reentrancy guard"]);
433        assert_eq!(solidity_block("NO_EXPLOIT"), None);
434    }
435}