Skip to main content

forge/
sandbox.rs

1//! Temporary copies of a project where Forge builds and runs generated tests.
2
3use crate::{result::TestResult, workspace};
4use eyre::{Context, Result, eyre};
5use foundry_common::fs::canonicalize_path;
6use foundry_compilers::{cache::SOLIDITY_FILES_CACHE_FILENAME, solc::Solc};
7use foundry_config::{Config, DenyLevel};
8use serde::Deserialize;
9use std::{
10    fs,
11    path::{Path, PathBuf},
12    process::{Command, ExitStatus, Output, Stdio},
13    thread,
14    time::{Duration, Instant},
15};
16use tempfile::TempDir;
17use toml_edit::DocumentMut;
18
19#[cfg(unix)]
20use rustix::process::{Pid, Signal, kill_process_group};
21#[cfg(unix)]
22use std::{os::unix::process::CommandExt, sync::Mutex};
23
24/// How often Forge checks whether an agent session with a timeout has exited. Sessions take
25/// minutes, so this adds no noticeable delay.
26const SESSION_POLL: Duration = Duration::from_millis(100);
27/// How long one Forge build or test run in a workspace can take.
28pub(crate) const RUN_TIMEOUT: Duration = Duration::from_secs(10 * 60);
29/// Agent settings and instruction files that a workspace leaves out at every level. Settings can
30/// hold hooks that run commands on the host. `CLAUDE.md` and `AGENTS.md` are project docs, but
31/// they steer the agents that work in the copy, so a user who wants them passes them with
32/// `--context`. Entries of dependency directories are links, so they keep their own files.
33const AGENT_CONFIG: [&str; 14] = [
34    ".agents",
35    ".claude",
36    ".clinerules",
37    ".codex",
38    ".cursor",
39    ".cursorrules",
40    ".gemini",
41    ".windsurf",
42    ".windsurfrules",
43    "AGENT.md",
44    "AGENTS.md",
45    "CLAUDE.local.md",
46    "CLAUDE.md",
47    "GEMINI.md",
48];
49/// The environment variables that builds and test runs of generated code see, besides `HOME`,
50/// `FOUNDRY_PROFILE`, and the seed settings. Generated tests run outside the agent's sandbox, so
51/// they must not read the secrets of Forge's environment with the env cheatcodes.
52const PASSED_ENV: [&str; 7] = ["LANG", "LC_ALL", "PATH", "SYSTEMROOT", "TEMP", "TMP", "TMPDIR"];
53/// Overrides configured test filters, so they cannot hide generated tests. `$^` matches no
54/// contract or test name, and no project file is under `/forge-properties-none`.
55const SELECT_ALL_TESTS: [&str; 10] = [
56    "--match-contract",
57    ".*",
58    "--match-test",
59    ".*",
60    "--no-match-contract",
61    "$^",
62    "--no-match-test",
63    "$^",
64    "--no-match-path",
65    "/forge-properties-none/**",
66];
67
68/// The process groups of the running agent sessions, which Forge ends if it is interrupted.
69#[cfg(unix)]
70static SESSIONS: Mutex<Vec<Pid>> = Mutex::new(Vec::new());
71
72/// A Solidity file that an agent wrote, by its path relative to the project root.
73#[derive(Clone, Debug)]
74pub(crate) struct TestFile {
75    pub(crate) path: PathBuf,
76    pub(crate) content: String,
77}
78
79/// Builds and runs generated code in temporary copies of the project. Generated code runs outside
80/// the sandbox of whatever wrote it, so a copy leaves out hidden root entries such as `.env` and
81/// agent settings, and Forge runs in it with only a few environment variables.
82pub(crate) struct Sandbox<'a> {
83    config: &'a Config,
84    /// Whether generated tests can call `vm.ffi`, if the project enables `ffi`.
85    allow_ffi: bool,
86    /// Whether generated tests can use the network and the endpoints that the config names.
87    allow_network: bool,
88    forge: PathBuf,
89    /// Project paths that workspaces leave out, besides the build output and the cache.
90    skipped: Vec<PathBuf>,
91    /// The workspace whose build output every new workspace copies.
92    build_cache: Option<TempDir>,
93    /// The home directory of builds and test runs, whose `.svm` links to the installed compilers.
94    home: PathBuf,
95    /// The temporary directory that holds the home directory and every workspace, so one removal
96    /// cleans up after an interrupt. It drops last.
97    root: TempDir,
98}
99
100impl<'a> Sandbox<'a> {
101    /// Returns a sandbox for the project of `config`, whose workspaces leave out the `skipped`
102    /// project paths and allow `ffi` and network access only as given.
103    pub(crate) fn new(
104        config: &'a Config,
105        skipped: Vec<PathBuf>,
106        allow_ffi: bool,
107        allow_network: bool,
108    ) -> Result<Self> {
109        let forge = std::env::current_exe().wrap_err("failed to resolve Forge executable")?;
110        let root = tempfile::Builder::new().prefix("forge-properties-").tempdir()?;
111        let home = root.path().join("home");
112        fs::create_dir(&home)?;
113        if let Some(svm) = Solc::svm_home().filter(|svm| svm.is_dir()) {
114            workspace::symlink_entry(&svm, &home.join(".svm"), fs::metadata(&svm)?.file_type())?;
115        }
116        Ok(Self { config, allow_ffi, allow_network, forge, skipped, build_cache: None, home, root })
117    }
118
119    /// Makes an interrupt end every session and test run and remove the workspaces before Forge
120    /// exits. They run in their own process groups, so a terminal interrupt does not reach them.
121    pub(crate) fn end_on_interrupt(&self, reserved: Vec<PathBuf>) -> Result<()> {
122        let root = self.root.path().to_path_buf();
123        ctrlc::set_handler(move || {
124            // The lock stays held until Forge exits, so no thread starts another process.
125            #[cfg(unix)]
126            let sessions = SESSIONS.lock().unwrap();
127            #[cfg(unix)]
128            for group in sessions.iter() {
129                let _ = kill_process_group(*group, Signal::KILL);
130            }
131            let _ = fs::remove_dir_all(&root);
132            // Exiting skips the drops, so the reserved directories that stayed empty go here.
133            for dir in &reserved {
134                let _ = fs::remove_dir(dir);
135            }
136            std::process::exit(130);
137        })?;
138        Ok(())
139    }
140
141    /// Builds the project once in a workspace whose build output later workspaces copy, so a
142    /// test run compiles only the files that it adds. Returns the compiler errors of a project
143    /// that does not build; it gets no shared output, and each test run builds it in full.
144    pub(crate) fn build(&mut self) -> Result<Option<String>> {
145        let workspace = self.workspace(&[])?;
146        if let Some(errors) = self.build_errors(workspace.path())? {
147            return Ok(Some(errors));
148        }
149        self.build_cache = Some(workspace);
150        Ok(None)
151    }
152
153    /// Builds a workspace and returns the compiler errors, if any. If the project denies
154    /// warnings, the warnings are errors. A workspace starts from a build without such warnings,
155    /// so only the files that this build compiles can warn.
156    pub(crate) fn build_errors(&self, workspace: &Path) -> Result<Option<String>> {
157        // The lint step can fail where the compiler succeeds, and it does not change the tests.
158        let Some(output) = self.output(self.command(workspace).args(["build", "--no-lint"]))?
159        else {
160            return Ok(Some(format!("the build reached the timeout of {RUN_TIMEOUT:?}")));
161        };
162        if !output.status.success() {
163            return Ok(Some(stderr(&output)));
164        }
165        let stdout = String::from_utf8_lossy(&output.stdout);
166        Ok(stdout.find("Warning (").filter(|_| self.config.deny.warnings()).map(|start| {
167            format!("the project denies compiler warnings: {}", shorten(&stdout[start..]))
168        }))
169    }
170
171    /// Copies the project and the shared build output into a temporary workspace and adds the
172    /// files.
173    pub(crate) fn workspace(&self, files: &[TestFile]) -> Result<TempDir> {
174        let config = self.config;
175        let workspace =
176            tempfile::Builder::new().prefix("workspace-").tempdir_in(self.root.path())?;
177        let dependencies = self.dependencies();
178        let skipped = [config.out.clone(), config.cache_path.clone()]
179            .into_iter()
180            .chain(self.skipped.iter().cloned())
181            .collect::<Vec<_>>();
182        copy_project_tree(config, workspace.path(), Path::new(""), &dependencies, &skipped)?;
183        // Internal runs see the project's resolved config, including inherited `extends` values,
184        // absolute paths, and environment overrides, without reading files outside the workspace.
185        let mut rebased = workspace::rebase_config_paths(config, workspace.path());
186        rebased.extends = None;
187        restrict_config(&mut rebased, self.allow_ffi, self.allow_network);
188        // Foundry turns the compiler cache off when it denies warnings, so a workspace builds with
189        // `deny = "never"`, and `Sandbox::build_errors` checks the warnings instead.
190        rebased.deny = DenyLevel::Never;
191        // With dynamic test linking, every new source file makes the compiler rebuild all files, so
192        // a workspace compiles without it. It changes how tests deploy contracts, not what they
193        // assert.
194        rebased.dynamic_test_linking = false;
195        // Dependency entries link to the project's entries, and the compiler resolves links before
196        // it checks the allowed paths. A source that the compiler reads itself, such as an import
197        // that climbs above the root into a nested library, must stay readable.
198        rebased.allow_paths.extend(
199            dependencies
200                .iter()
201                .filter(|path| path.is_dir())
202                .filter_map(|path| canonicalize_path(path).ok()),
203        );
204        let mut toml = rebased.to_string_pretty()?.parse::<DocumentMut>()?;
205        // Inline config may name the project's other profiles, so they must still exist. The TOML
206        // writer quotes names such as `""` or `a.b` that are not bare keys.
207        let profiles = toml[Config::PROFILE_SECTION]
208            .as_table_mut()
209            .expect("the resolved config has a profile table");
210        for profile in config.profiles.iter().filter(|profile| **profile != config.profile) {
211            profiles.entry(profile.as_str().as_str()).or_insert_with(toml_edit::table);
212        }
213        fs::write(workspace.path().join(Config::FILE_NAME), toml.to_string())?;
214        // Remappings are already resolved and rebased in the workspace config. The Soldeer lock
215        // goes too: `forge build` then hashes each Soldeer dependency to check it, and the hash
216        // panics when the dependency is a link, as in a workspace.
217        for file in ["remappings.txt", "soldeer.lock"] {
218            let path = workspace.path().join(file);
219            if path.exists() {
220                fs::remove_file(path)?;
221            }
222        }
223        if let Some(build) = &self.build_cache {
224            copy_build_output(config, build.path(), workspace.path())?;
225        }
226        for file in files {
227            let path = workspace.path().join(&file.path);
228            ensure_inside(workspace.path(), &path, &file.path)?;
229            fs::create_dir_all(path.parent().expect("test path has a parent"))?;
230            fs::write(path, &file.content)?;
231        }
232        Ok(workspace)
233    }
234
235    /// Returns the dependency directories of the project, whose entries workspaces link instead of
236    /// copying them.
237    pub(crate) fn dependencies(&self) -> Vec<PathBuf> {
238        let config = self.config;
239        config
240            .libs
241            .iter()
242            .map(|lib| lib.as_path())
243            .chain(["node_modules".as_ref(), "dependencies".as_ref()])
244            .map(|path| config.root.join(path))
245            .collect()
246    }
247
248    /// Returns a Forge command in a workspace. A workspace has its own resolved `foundry.toml`, so
249    /// the command needs only the selected profile from the environment. Generated tests cannot
250    /// read other variables.
251    pub(crate) fn command(&self, workspace: &Path) -> Command {
252        let mut command = Command::new(&self.forge);
253        command
254            .current_dir(workspace)
255            .env_clear()
256            .envs(PASSED_ENV.iter().filter_map(|key| Some((key, std::env::var_os(key)?))))
257            .env("HOME", &self.home)
258            .env("FOUNDRY_PROFILE", self.config.profile.as_str().as_str());
259        command
260    }
261
262    /// Returns a command in a workspace that runs every test in the files with JSON output.
263    /// Configured filters cannot hide the tests, inline config can only lower runs, and the tests
264    /// use the network only if the sandbox allows it.
265    pub(crate) fn test_command(&self, workspace: &Path, files: &[TestFile]) -> Command {
266        // A path can hold glob characters, such as a comma in the test directory, which would
267        // split the alternatives.
268        let paths = files
269            .iter()
270            .map(|file| globset::escape(&file.path.display().to_string()).replace(',', "[,]"))
271            .collect::<Vec<_>>()
272            .join(",");
273        let mut command = self.command(workspace);
274        command
275            .args(["test", "--json", "--restrict-inline-config", "--match-path"])
276            .arg(format!("{{{paths}}}"))
277            .args(SELECT_ALL_TESTS);
278        if !self.allow_network {
279            command.arg("--no-network");
280        }
281        command
282    }
283
284    /// Runs a command of [`Sandbox::command`] and returns its output, or `None` if it reached
285    /// [`RUN_TIMEOUT`]. Generated tests can loop for as long as their gas lasts, so Forge ends the
286    /// command and every process that it started at the timeout.
287    pub(crate) fn output(&self, command: &mut Command) -> Result<Option<Output>> {
288        let dir = tempfile::Builder::new().prefix("output-").tempdir_in(self.root.path())?;
289        let (stdout, stderr) = (dir.path().join("stdout"), dir.path().join("stderr"));
290        command
291            .stdin(Stdio::null())
292            .stdout(fs::File::create(&stdout)?)
293            .stderr(fs::File::create(&stderr)?);
294        let Some(status) = run_session(command, Some(RUN_TIMEOUT.as_secs()), "forge")? else {
295            return Ok(None);
296        };
297        Ok(Some(Output { status, stdout: fs::read(stdout)?, stderr: fs::read(stderr)? }))
298    }
299}
300
301/// Turns off `ffi` unless `allow_ffi`, and removes the RPC and Etherscan settings unless
302/// `allow_network`. Every workspace of a [`Sandbox`] writes its config through this, so no
303/// generated test can call `vm.ffi` or reach an endpoint that the config names. Config ignores
304/// `FFI` environment variables.
305fn restrict_config(config: &mut Config, allow_ffi: bool, allow_network: bool) {
306    config.ffi &= allow_ffi;
307    // Generated tests could send data to any host, and the endpoints can hold API keys.
308    if !allow_network {
309        config.eth_rpc_url = None;
310        config.eth_rpc_jwt = None;
311        config.eth_rpc_headers = None;
312        config.etherscan_api_key = None;
313        config.etherscan = Default::default();
314        config.rpc_endpoints = Default::default();
315    }
316}
317
318/// Fails if the nearest existing ancestor of `path` resolves outside `root`, for example through a
319/// linked directory.
320pub(crate) fn ensure_inside(root: &Path, path: &Path, orig: &Path) -> Result<()> {
321    match path.ancestors().find(|ancestor| ancestor.exists()) {
322        Some(existing) => workspace::ensure_within_root(root, existing, "generated test", orig),
323        None => Ok(()),
324    }
325}
326
327/// Copies a project directory into a workspace, without Git data, the `skipped` paths, agent
328/// settings and instructions (`AGENT_CONFIG`), and, at the project root, hidden entries such as
329/// `.env`.
330///
331/// A dependency directory becomes a real directory whose entries link to the project's entries.
332/// A linked dependency directory would resolve outside the workspace, and the compiler then
333/// cannot resolve a dependency's own absolute imports, such as `src/lib/X.sol` in a pnpm package.
334/// A link to a path in the project leads to that path in the workspace, so it stays inside it.
335fn copy_project_tree(
336    config: &Config,
337    copy: &Path,
338    dir: &Path,
339    dependencies: &[PathBuf],
340    skipped: &[PathBuf],
341) -> Result<()> {
342    fs::create_dir_all(copy.join(dir))?;
343    for entry in fs::read_dir(config.root.join(dir))? {
344        let entry = entry?;
345        let path = entry.path();
346        let name = entry.file_name();
347        let rel = dir.join(&name);
348        let target = copy.join(&rel);
349        if name == ".git"
350            || AGENT_CONFIG.iter().any(|config| name == *config)
351            || (dir.as_os_str().is_empty() && name.to_string_lossy().starts_with('.'))
352            || skipped.contains(&path)
353        {
354            continue;
355        }
356        let file_type = entry.file_type()?;
357        if dependencies.contains(&path) && path.is_dir() {
358            fs::create_dir_all(&target)?;
359            for dependency in fs::read_dir(&path)? {
360                let dependency = dependency?;
361                let link = target.join(dependency.file_name());
362                workspace::symlink_entry(&dependency.path(), &link, dependency.file_type()?)?;
363            }
364        } else if file_type.is_symlink() {
365            let link = match path.canonicalize() {
366                Ok(resolved) => match resolved.strip_prefix(&config.root) {
367                    Ok(resolved) => copy.join(resolved),
368                    Err(_) => resolved,
369                },
370                // A dangling link stays as it is.
371                Err(_) => fs::read_link(&path)?,
372            };
373            workspace::symlink_entry(&link, &target, file_type)?;
374        } else if file_type.is_dir() {
375            copy_project_tree(config, copy, &rel, dependencies, skipped)?;
376        } else {
377            fs::copy(&path, &target)?;
378        }
379    }
380    Ok(())
381}
382
383/// Copies the build output and the compiler cache of a built workspace into another workspace.
384/// The copy clones files where the file system can (APFS, Btrfs, XFS).
385///
386/// The compiler cache keys sources by their path relative to the root, but it stores remappings
387/// with the absolute root, and other remappings invalidate it. Both workspaces have the same files
388/// at the same relative paths, so moving the remappings to the new root keeps the cache valid, and
389/// the compiler rebuilds only sources whose content changed.
390fn copy_build_output(config: &Config, build: &Path, workspace: &Path) -> Result<()> {
391    let out = workspace::relative_to_root(&config.root, &config.out);
392    let cache = workspace::relative_to_root(&config.root, &config.cache_path)
393        .join(SOLIDITY_FILES_CACHE_FILENAME);
394    // Shared output outside the project would mix the builds of all workspaces.
395    if out.is_absolute() || cache.is_absolute() || !build.join(&cache).exists() {
396        return Ok(());
397    }
398    workspace::copy_dir_recursive(&build.join(&out), &workspace.join(&out))?;
399    let mut text = fs::read_to_string(build.join(&cache))?;
400    // The compiler can see either spelling of a temporary path, such as `/private/var` on macOS.
401    for (from, to) in [
402        (canonicalize_path(build)?, canonicalize_path(workspace)?),
403        (build.to_path_buf(), workspace.to_path_buf()),
404    ] {
405        text = text.replace(&format!("{}/", from.display()), &format!("{}/", to.display()));
406    }
407    fs::create_dir_all(workspace.join(&cache).parent().expect("cache file has a parent"))?;
408    fs::write(workspace.join(&cache), text)?;
409    Ok(())
410}
411
412/// Returns every test result in Forge's JSON output with its contract and test name.
413pub(crate) fn suite_results(output: &Output) -> Result<Vec<(String, String, TestResult)>> {
414    let suites = serde_json::from_slice::<serde_json::Value>(&output.stdout)
415        .wrap_err_with(|| format!("invalid Forge JSON: {}", stderr(output)))?;
416    let suites = suites.as_object().ok_or_else(|| eyre!("Forge JSON is not an object"))?;
417    let mut results = Vec::new();
418    for (contract, suite) in suites {
419        let Some(tests) = suite.get("test_results").and_then(serde_json::Value::as_object) else {
420            continue;
421        };
422        for (test, result) in tests {
423            let result = TestResult::deserialize(result)
424                .wrap_err_with(|| format!("invalid result for {contract}::{test}"))?;
425            results.push((contract.clone(), test.clone(), result));
426        }
427    }
428    Ok(results)
429}
430
431/// Returns stderr without compiler warning blocks and keeps both ends when it is still long.
432/// Warnings can fill the limit before the decisive error, and many tools print the error last.
433/// Without a compiler error, as with `deny = "warnings"`, the warnings are the errors and stay.
434pub(crate) fn stderr(output: &Output) -> String {
435    let stderr = String::from_utf8_lossy(&output.stderr);
436    let has_error = stderr.lines().any(|line| line.starts_with("Error ("));
437    let mut in_warning = false;
438    let stderr = stderr
439        .lines()
440        .filter(|line| {
441            if !has_error {
442                return true;
443            }
444            if line.starts_with("Warning") {
445                in_warning = true;
446            } else if line.starts_with("Error") {
447                in_warning = false;
448            } else if in_warning && line.trim().is_empty() {
449                in_warning = false;
450                return false;
451            }
452            !in_warning
453        })
454        .collect::<Vec<_>>()
455        .join("\n");
456    shorten(&stderr)
457}
458
459/// Keeps both ends of a long tool output. Many tools print the decisive error last.
460pub(crate) fn shorten(text: &str) -> String {
461    const HALF: usize = 1_000;
462    let text = text.trim();
463    let chars = text.chars().count();
464    if chars <= 2 * HALF {
465        return text.to_string();
466    }
467    let head = text.chars().take(HALF).collect::<String>();
468    let tail = text.chars().skip(chars - HALF).collect::<String>();
469    format!("{head}\n[...]\n{tail}")
470}
471
472/// Runs a session command in its own process group until it exits or reaches the timeout in
473/// seconds, then ends the process group, so no process that the session started keeps running.
474/// Returns `None` if the session reached the timeout.
475pub(crate) fn run_session(
476    command: &mut Command,
477    timeout: Option<u64>,
478    session: &str,
479) -> Result<Option<ExitStatus>> {
480    #[cfg(unix)]
481    command.process_group(0);
482    // The lock spans the spawn, so an interrupt cannot miss a session that just started.
483    #[cfg(unix)]
484    let mut sessions = SESSIONS.lock().unwrap();
485    let mut child = command.spawn().wrap_err_with(|| format!("failed to run {session} command"))?;
486    #[cfg(unix)]
487    let group = Pid::from_child(&child);
488    #[cfg(unix)]
489    {
490        sessions.push(group);
491        drop(sessions);
492    }
493    let status = match timeout {
494        Some(timeout) => {
495            let deadline = Instant::now() + Duration::from_secs(timeout);
496            loop {
497                match child.try_wait() {
498                    Ok(None) if Instant::now() < deadline => thread::sleep(SESSION_POLL),
499                    status => break status,
500                }
501            }
502        }
503        None => child.wait().map(Some),
504    };
505    // The session ends with the command: no process that it started keeps running.
506    #[cfg(unix)]
507    {
508        let _ = kill_process_group(group, Signal::KILL);
509        SESSIONS.lock().unwrap().retain(|running| *running != group);
510    }
511    Ok(match status? {
512        Some(status) => Some(status),
513        None => {
514            #[cfg(not(unix))]
515            let _ = child.kill();
516            child.wait()?;
517            None
518        }
519    })
520}