foundry_evm_traces/identifier/
external.rs

1use super::{IdentifiedAddress, TraceIdentifier};
2use crate::debug::ContractSources;
3use alloy_primitives::{
4    Address,
5    map::{Entry, HashMap},
6};
7use eyre::WrapErr;
8use foundry_block_explorers::{contract::Metadata, errors::EtherscanError};
9use foundry_common::compile::etherscan_project;
10use foundry_config::{Chain, Config};
11use futures::{
12    future::join_all,
13    stream::{FuturesUnordered, Stream, StreamExt},
14    task::{Context, Poll},
15};
16use revm_inspectors::tracing::types::CallTraceNode;
17use serde::Deserialize;
18use std::{
19    borrow::Cow,
20    pin::Pin,
21    sync::{
22        Arc,
23        atomic::{AtomicBool, Ordering},
24    },
25};
26use tokio::time::{Duration, Interval};
27
28/// A trace identifier that tries to identify addresses using Etherscan.
29pub struct ExternalIdentifier {
30    fetchers: Vec<Arc<dyn ExternalFetcherT>>,
31    /// Cached contracts.
32    contracts: HashMap<Address, (FetcherKind, Option<Metadata>)>,
33}
34
35impl ExternalIdentifier {
36    /// Creates a new external identifier with the given client
37    pub fn new(config: &Config, mut chain: Option<Chain>) -> eyre::Result<Option<Self>> {
38        if config.offline {
39            return Ok(None);
40        }
41
42        let config = match config.get_etherscan_config_with_chain(chain) {
43            Ok(Some(config)) => {
44                chain = config.chain;
45                Some(config)
46            }
47            Ok(None) => {
48                warn!(target: "evm::traces::external", "etherscan config not found");
49                None
50            }
51            Err(err) => {
52                warn!(target: "evm::traces::external", ?err, "failed to get etherscan config");
53                None
54            }
55        };
56
57        let mut fetchers = Vec::<Arc<dyn ExternalFetcherT>>::new();
58        if let Some(chain) = chain {
59            debug!(target: "evm::traces::external", ?chain, "using sourcify identifier");
60            fetchers.push(Arc::new(SourcifyFetcher::new(chain)));
61        }
62        if let Some(config) = config {
63            debug!(target: "evm::traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier");
64            fetchers.push(Arc::new(EtherscanFetcher::new(config.into_client()?)));
65        }
66        if fetchers.is_empty() {
67            debug!(target: "evm::traces::external", "no fetchers enabled");
68            return Ok(None);
69        }
70
71        Ok(Some(Self { fetchers, contracts: Default::default() }))
72    }
73
74    /// Goes over the list of contracts we have pulled from the traces, clones their source from
75    /// Etherscan and compiles them locally, for usage in the debugger.
76    pub async fn get_compiled_contracts(&self) -> eyre::Result<ContractSources> {
77        // Collect contract info upfront so we can reference it in error messages
78        let contracts_info: Vec<_> = self
79            .contracts
80            .iter()
81            // filter out vyper files and contracts without metadata
82            .filter_map(|(addr, (_, metadata))| {
83                if let Some(metadata) = metadata.as_ref()
84                    && !metadata.is_vyper()
85                {
86                    Some((*addr, metadata))
87                } else {
88                    None
89                }
90            })
91            .collect();
92
93        let outputs_fut = contracts_info
94            .iter()
95            .map(|(addr, metadata)| async move {
96                sh_println!("Compiling: {} {addr}", metadata.contract_name)?;
97                let root = tempfile::tempdir()?;
98                let root_path = root.path();
99                let project = etherscan_project(metadata, root_path)?;
100                let output = project.compile()?;
101                if output.has_compiler_errors() {
102                    eyre::bail!("{output}")
103                }
104
105                Ok((project, output, root))
106            })
107            .collect::<Vec<_>>();
108
109        // poll all the futures concurrently
110        let outputs = join_all(outputs_fut).await;
111
112        let mut sources: ContractSources = Default::default();
113
114        // construct the map
115        for (idx, res) in outputs.into_iter().enumerate() {
116            let (addr, metadata) = &contracts_info[idx];
117            let name = &metadata.contract_name;
118            let (project, output, _) =
119                res.wrap_err_with(|| format!("Failed to compile contract {name} at {addr}"))?;
120            sources
121                .insert(&output, project.root(), None)
122                .wrap_err_with(|| format!("Failed to insert contract {name} at {addr}"))?;
123        }
124
125        Ok(sources)
126    }
127
128    fn identify_from_metadata(
129        &self,
130        address: Address,
131        metadata: &Metadata,
132    ) -> IdentifiedAddress<'static> {
133        let label = metadata.contract_name.clone();
134        let abi = metadata.abi().ok().map(Cow::Owned);
135        IdentifiedAddress {
136            address,
137            label: Some(label.clone()),
138            contract: Some(label),
139            abi,
140            artifact_id: None,
141        }
142    }
143}
144
145impl TraceIdentifier for ExternalIdentifier {
146    fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec<IdentifiedAddress<'_>> {
147        if nodes.is_empty() {
148            return Vec::new();
149        }
150
151        trace!(target: "evm::traces::external", "identify {} addresses", nodes.len());
152
153        let mut identities = Vec::new();
154        let mut to_fetch = Vec::new();
155
156        // Check cache first.
157        for &node in nodes {
158            let address = node.trace.address;
159            if let Some((_, metadata)) = self.contracts.get(&address) {
160                if let Some(metadata) = metadata {
161                    identities.push(self.identify_from_metadata(address, metadata));
162                } else {
163                    // Do nothing. We know that this contract was not verified.
164                }
165            } else {
166                to_fetch.push(address);
167            }
168        }
169
170        if to_fetch.is_empty() {
171            return identities;
172        }
173        trace!(target: "evm::traces::external", "fetching {} addresses", to_fetch.len());
174
175        let fetchers =
176            self.fetchers.iter().map(|fetcher| ExternalFetcher::new(fetcher.clone(), &to_fetch));
177        let fetched_identities = foundry_common::block_on(
178            futures::stream::select_all(fetchers)
179                .filter_map(|(address, value)| {
180                    let addr = value
181                        .1
182                        .as_ref()
183                        .map(|metadata| self.identify_from_metadata(address, metadata));
184                    match self.contracts.entry(address) {
185                        Entry::Occupied(mut occupied_entry) => {
186                            // Override if:
187                            // - new is from Etherscan and old is not
188                            // - new is Some and old is None, meaning verified only in one source
189                            if !matches!(occupied_entry.get().0, FetcherKind::Etherscan)
190                                || value.1.is_none()
191                            {
192                                occupied_entry.insert(value);
193                            }
194                        }
195                        Entry::Vacant(vacant_entry) => {
196                            vacant_entry.insert(value);
197                        }
198                    }
199                    async move { addr }
200                })
201                .collect::<Vec<IdentifiedAddress<'_>>>(),
202        );
203        trace!(target: "evm::traces::external", "fetched {} addresses: {fetched_identities:#?}", fetched_identities.len());
204
205        identities.extend(fetched_identities);
206        identities
207    }
208}
209
210type FetchFuture =
211    Pin<Box<dyn Future<Output = (Address, Result<Option<Metadata>, EtherscanError>)>>>;
212
213/// A rate limit aware fetcher.
214///
215/// Fetches information about multiple addresses concurrently, while respecting rate limits.
216struct ExternalFetcher {
217    /// The fetcher
218    fetcher: Arc<dyn ExternalFetcherT>,
219    /// The time we wait if we hit the rate limit
220    timeout: Duration,
221    /// The interval we are currently waiting for before making a new request
222    backoff: Option<Interval>,
223    /// The maximum amount of requests to send concurrently
224    concurrency: usize,
225    /// The addresses we have yet to make requests for
226    queue: Vec<Address>,
227    /// The in progress requests
228    in_progress: FuturesUnordered<FetchFuture>,
229}
230
231impl ExternalFetcher {
232    fn new(fetcher: Arc<dyn ExternalFetcherT>, to_fetch: &[Address]) -> Self {
233        Self {
234            timeout: fetcher.timeout(),
235            backoff: None,
236            concurrency: fetcher.concurrency(),
237            fetcher,
238            queue: to_fetch.to_vec(),
239            in_progress: FuturesUnordered::new(),
240        }
241    }
242
243    fn queue_next_reqs(&mut self) {
244        while self.in_progress.len() < self.concurrency {
245            let Some(addr) = self.queue.pop() else { break };
246            let fetcher = Arc::clone(&self.fetcher);
247            self.in_progress.push(Box::pin(async move {
248                trace!(target: "evm::traces::external", ?addr, "fetching info");
249                let res = fetcher.fetch(addr).await;
250                (addr, res)
251            }));
252        }
253    }
254}
255
256impl Stream for ExternalFetcher {
257    type Item = (Address, (FetcherKind, Option<Metadata>));
258
259    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
260        let pin = self.get_mut();
261
262        let _guard =
263            info_span!("evm::traces::external", kind=?pin.fetcher.kind(), "ExternalFetcher")
264                .entered();
265
266        if pin.fetcher.invalid_api_key().load(Ordering::Relaxed) {
267            return Poll::Ready(None);
268        }
269
270        loop {
271            if let Some(mut backoff) = pin.backoff.take()
272                && backoff.poll_tick(cx).is_pending()
273            {
274                pin.backoff = Some(backoff);
275                return Poll::Pending;
276            }
277
278            pin.queue_next_reqs();
279
280            let mut made_progress_this_iter = false;
281            match pin.in_progress.poll_next_unpin(cx) {
282                Poll::Pending => {}
283                Poll::Ready(None) => return Poll::Ready(None),
284                Poll::Ready(Some((addr, res))) => {
285                    made_progress_this_iter = true;
286                    match res {
287                        Ok(metadata) => {
288                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata))));
289                        }
290                        Err(EtherscanError::ContractCodeNotVerified(_)) => {
291                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
292                        }
293                        Err(EtherscanError::RateLimitExceeded) => {
294                            warn!(target: "evm::traces::external", "rate limit exceeded on attempt");
295                            pin.backoff = Some(tokio::time::interval(pin.timeout));
296                            pin.queue.push(addr);
297                        }
298                        Err(EtherscanError::InvalidApiKey) => {
299                            warn!(target: "evm::traces::external", "invalid api key");
300                            // mark key as invalid
301                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
302                            return Poll::Ready(None);
303                        }
304                        Err(EtherscanError::BlockedByCloudflare) => {
305                            warn!(target: "evm::traces::external", "blocked by cloudflare");
306                            // mark key as invalid
307                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
308                            return Poll::Ready(None);
309                        }
310                        Err(err) => {
311                            warn!(target: "evm::traces::external", ?err, "could not get info");
312                        }
313                    }
314                }
315            }
316
317            if !made_progress_this_iter {
318                return Poll::Pending;
319            }
320        }
321    }
322}
323
324#[derive(Debug, Clone, Copy, PartialEq, Eq)]
325enum FetcherKind {
326    Etherscan,
327    Sourcify,
328}
329
330#[async_trait::async_trait]
331trait ExternalFetcherT: Send + Sync {
332    fn kind(&self) -> FetcherKind;
333    fn timeout(&self) -> Duration;
334    fn concurrency(&self) -> usize;
335    fn invalid_api_key(&self) -> &AtomicBool;
336    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError>;
337}
338
339struct EtherscanFetcher {
340    client: foundry_block_explorers::Client,
341    invalid_api_key: AtomicBool,
342}
343
344impl EtherscanFetcher {
345    fn new(client: foundry_block_explorers::Client) -> Self {
346        Self { client, invalid_api_key: AtomicBool::new(false) }
347    }
348}
349
350#[async_trait::async_trait]
351impl ExternalFetcherT for EtherscanFetcher {
352    fn kind(&self) -> FetcherKind {
353        FetcherKind::Etherscan
354    }
355
356    fn timeout(&self) -> Duration {
357        Duration::from_secs(1)
358    }
359
360    fn concurrency(&self) -> usize {
361        5
362    }
363
364    fn invalid_api_key(&self) -> &AtomicBool {
365        &self.invalid_api_key
366    }
367
368    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
369        self.client.contract_source_code(address).await.map(|mut metadata| metadata.items.pop())
370    }
371}
372
373struct SourcifyFetcher {
374    client: reqwest::Client,
375    url: String,
376    invalid_api_key: AtomicBool,
377}
378
379impl SourcifyFetcher {
380    fn new(chain: Chain) -> Self {
381        Self {
382            client: reqwest::Client::new(),
383            url: format!("https://sourcify.dev/server/v2/contract/{}", chain.id()),
384            invalid_api_key: AtomicBool::new(false),
385        }
386    }
387}
388
389#[async_trait::async_trait]
390impl ExternalFetcherT for SourcifyFetcher {
391    fn kind(&self) -> FetcherKind {
392        FetcherKind::Sourcify
393    }
394
395    fn timeout(&self) -> Duration {
396        Duration::from_secs(1)
397    }
398
399    fn concurrency(&self) -> usize {
400        5
401    }
402
403    fn invalid_api_key(&self) -> &AtomicBool {
404        &self.invalid_api_key
405    }
406
407    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
408        let url = format!("{url}/{address}?fields=abi,compilation", url = self.url);
409        let response = self.client.get(url).send().await?;
410        let code = response.status();
411        let response: SourcifyResponse = response.json().await?;
412        trace!(target: "evm::traces::external", "Sourcify response for {address}: {response:#?}");
413        match code.as_u16() {
414            // Not verified.
415            404 => return Err(EtherscanError::ContractCodeNotVerified(address)),
416            // Too many requests.
417            429 => return Err(EtherscanError::RateLimitExceeded),
418            _ => {}
419        }
420        match response {
421            SourcifyResponse::Success(metadata) => Ok(Some(metadata.into())),
422            SourcifyResponse::Error(error) => Err(EtherscanError::Unknown(format!("{error:#?}"))),
423        }
424    }
425}
426
427/// Sourcify API response for `/v2/contract/{chainId}/{address}`.
428#[derive(Debug, Clone, Deserialize)]
429#[serde(untagged)]
430enum SourcifyResponse {
431    Success(SourcifyMetadata),
432    Error(SourcifyError),
433}
434
435#[derive(Debug, Clone, Deserialize)]
436#[serde(rename_all = "camelCase")]
437#[expect(dead_code)] // Used in Debug.
438struct SourcifyError {
439    custom_code: String,
440    message: String,
441    error_id: String,
442}
443
444#[derive(Debug, Clone, Deserialize)]
445#[serde(rename_all = "camelCase")]
446struct SourcifyMetadata {
447    #[serde(default)]
448    abi: Option<Box<serde_json::value::RawValue>>,
449    #[serde(default)]
450    compilation: Option<Compilation>,
451}
452
453#[derive(Debug, Clone, Deserialize)]
454#[serde(rename_all = "camelCase")]
455struct Compilation {
456    #[serde(default)]
457    compiler_version: String,
458    #[serde(default)]
459    name: String,
460}
461
462impl From<SourcifyMetadata> for Metadata {
463    fn from(metadata: SourcifyMetadata) -> Self {
464        let SourcifyMetadata { abi, compilation } = metadata;
465        let (contract_name, compiler_version) = compilation
466            .map(|c| (c.name, c.compiler_version))
467            .unwrap_or_else(|| (String::new(), String::new()));
468        // Defaulted fields may be fetched from sourcify but we don't make use of them.
469        Self {
470            source_code: foundry_block_explorers::contract::SourceCodeMetadata::Sources(
471                Default::default(),
472            ),
473            abi: Box::<str>::from(abi.unwrap_or_default()).into(),
474            contract_name,
475            compiler_version,
476            optimization_used: 0,
477            runs: 0,
478            constructor_arguments: Default::default(),
479            evm_version: String::new(),
480            library: String::new(),
481            license_type: String::new(),
482            proxy: 0,
483            implementation: None,
484            swarm_source: String::new(),
485        }
486    }
487}
foundry_evm_traces/identifier/external.rs

foundry_evm_traces/identifier/
external.rs
