foundry_evm_traces/identifier/
external.rs

1use super::{IdentifiedAddress, TraceIdentifier};
2use crate::debug::ContractSources;
3use alloy_primitives::{
4    Address,
5    map::{Entry, HashMap, HashSet},
6};
7use eyre::WrapErr;
8use foundry_block_explorers::{contract::Metadata, errors::EtherscanError};
9use foundry_common::compile::etherscan_project;
10use foundry_config::{Chain, Config};
11use futures::{
12    future::join_all,
13    stream::{FuturesUnordered, Stream, StreamExt},
14    task::{Context, Poll},
15};
16use revm_inspectors::tracing::types::CallTraceNode;
17use serde::Deserialize;
18use std::{
19    borrow::Cow,
20    pin::Pin,
21    sync::{
22        Arc,
23        atomic::{AtomicBool, Ordering},
24    },
25};
26use tokio::time::{Duration, Interval};
27
28/// A trace identifier that tries to identify addresses using Etherscan.
29pub struct ExternalIdentifier {
30    fetchers: Vec<Arc<dyn ExternalFetcherT>>,
31    /// Cached contracts.
32    contracts: HashMap<Address, (FetcherKind, Option<Metadata>)>,
33}
34
35impl ExternalIdentifier {
36    /// Creates a new external identifier with the given client
37    pub fn new(config: &Config, mut chain: Option<Chain>) -> eyre::Result<Option<Self>> {
38        if config.offline {
39            return Ok(None);
40        }
41
42        let config = match config.get_etherscan_config_with_chain(chain) {
43            Ok(Some(config)) => {
44                chain = config.chain;
45                Some(config)
46            }
47            Ok(None) => {
48                warn!(target: "evm::traces::external", "etherscan config not found");
49                None
50            }
51            Err(err) => {
52                warn!(target: "evm::traces::external", ?err, "failed to get etherscan config");
53                None
54            }
55        };
56
57        let mut fetchers = Vec::<Arc<dyn ExternalFetcherT>>::new();
58        if let Some(chain) = chain {
59            debug!(target: "evm::traces::external", ?chain, "using sourcify identifier");
60            fetchers.push(Arc::new(SourcifyFetcher::new(chain)));
61        }
62        if let Some(config) = config {
63            debug!(target: "evm::traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier");
64            match config.into_client() {
65                Ok(client) => {
66                    fetchers.push(Arc::new(EtherscanFetcher::new(client)));
67                }
68                Err(err) => {
69                    warn!(target: "evm::traces::external", ?err, "failed to create etherscan client");
70                }
71            }
72        }
73        if fetchers.is_empty() {
74            debug!(target: "evm::traces::external", "no fetchers enabled");
75            return Ok(None);
76        }
77
78        Ok(Some(Self { fetchers, contracts: Default::default() }))
79    }
80
81    /// Goes over the list of contracts we have pulled from the traces, clones their source from
82    /// Etherscan and compiles them locally, for usage in the debugger.
83    pub async fn get_compiled_contracts(&self) -> eyre::Result<ContractSources> {
84        // Collect contract info upfront so we can reference it in error messages
85        let contracts_info: Vec<_> = self
86            .contracts
87            .iter()
88            // filter out vyper files and contracts without metadata
89            .filter_map(|(addr, (_, metadata))| {
90                if let Some(metadata) = metadata.as_ref()
91                    && !metadata.is_vyper()
92                {
93                    Some((*addr, metadata))
94                } else {
95                    None
96                }
97            })
98            .collect();
99
100        let outputs_fut = contracts_info
101            .iter()
102            .map(|(addr, metadata)| async move {
103                sh_println!("Compiling: {} {addr}", metadata.contract_name)?;
104                let root = tempfile::tempdir()?;
105                let root_path = root.path();
106                let project = etherscan_project(metadata, root_path)?;
107                let output = project.compile()?;
108                if output.has_compiler_errors() {
109                    eyre::bail!("{output}")
110                }
111
112                Ok((project, output, root))
113            })
114            .collect::<Vec<_>>();
115
116        // poll all the futures concurrently
117        let outputs = join_all(outputs_fut).await;
118
119        let mut sources: ContractSources = Default::default();
120
121        // construct the map
122        for (idx, res) in outputs.into_iter().enumerate() {
123            let (addr, metadata) = &contracts_info[idx];
124            let name = &metadata.contract_name;
125            let (project, output, _) =
126                res.wrap_err_with(|| format!("Failed to compile contract {name} at {addr}"))?;
127            sources
128                .insert(&output, project.root(), None)
129                .wrap_err_with(|| format!("Failed to insert contract {name} at {addr}"))?;
130        }
131
132        Ok(sources)
133    }
134
135    fn identify_from_metadata(
136        &self,
137        address: Address,
138        metadata: &Metadata,
139    ) -> IdentifiedAddress<'static> {
140        let label = metadata.contract_name.clone();
141        let abi = metadata.abi().ok().map(Cow::Owned);
142        IdentifiedAddress {
143            address,
144            label: Some(label.clone()),
145            contract: Some(label),
146            abi,
147            artifact_id: None,
148        }
149    }
150}
151
152impl TraceIdentifier for ExternalIdentifier {
153    fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec<IdentifiedAddress<'_>> {
154        if nodes.is_empty() {
155            return Vec::new();
156        }
157
158        trace!(target: "evm::traces::external", "identify {} addresses", nodes.len());
159
160        let mut identities = Vec::new();
161        let mut to_fetch = HashSet::new();
162
163        // Check cache first.
164        for &node in nodes {
165            let address = node.trace.address;
166            if let Some((_, metadata)) = self.contracts.get(&address) {
167                if let Some(metadata) = metadata {
168                    identities.push(self.identify_from_metadata(address, metadata));
169                } else {
170                    // Do nothing. We know that this contract was not verified.
171                }
172            } else {
173                to_fetch.insert(address);
174            }
175        }
176
177        if to_fetch.is_empty() {
178            return identities;
179        }
180        trace!(target: "evm::traces::external", "fetching {} addresses", to_fetch.len());
181
182        let to_fetch = to_fetch.into_iter().collect::<Vec<_>>();
183        let fetchers =
184            self.fetchers.iter().map(|fetcher| ExternalFetcher::new(fetcher.clone(), &to_fetch));
185        let fetched_identities = foundry_common::block_on(
186            futures::stream::select_all(fetchers)
187                .filter_map(|(address, value)| {
188                    let addr = value
189                        .1
190                        .as_ref()
191                        .map(|metadata| self.identify_from_metadata(address, metadata));
192                    match self.contracts.entry(address) {
193                        Entry::Occupied(mut occupied_entry) => {
194                            let old = occupied_entry.get();
195                            // Only override when the new result is strictly better:
196                            // - new has metadata and old doesn't, OR
197                            // - both have metadata but new is from Etherscan and old is not.
198                            // Never downgrade a successful lookup to None.
199                            let should_replace = match (&old.1, &value.1) {
200                                (None, Some(_)) => true,
201                                (Some(_), None) => false,
202                                _ => {
203                                    matches!(value.0, FetcherKind::Etherscan)
204                                        && !matches!(old.0, FetcherKind::Etherscan)
205                                }
206                            };
207                            if should_replace {
208                                occupied_entry.insert(value);
209                            }
210                        }
211                        Entry::Vacant(vacant_entry) => {
212                            vacant_entry.insert(value);
213                        }
214                    }
215                    async move { addr }
216                })
217                .collect::<Vec<IdentifiedAddress<'_>>>(),
218        );
219        trace!(target: "evm::traces::external", "fetched {} addresses: {fetched_identities:#?}", fetched_identities.len());
220
221        identities.extend(fetched_identities);
222        identities
223    }
224}
225
226type FetchFuture =
227    Pin<Box<dyn Future<Output = (Address, Result<Option<Metadata>, EtherscanError>)>>>;
228
229/// A rate limit aware fetcher.
230///
231/// Fetches information about multiple addresses concurrently, while respecting rate limits.
232struct ExternalFetcher {
233    /// The fetcher
234    fetcher: Arc<dyn ExternalFetcherT>,
235    /// The time we wait if we hit the rate limit
236    timeout: Duration,
237    /// The interval we are currently waiting for before making a new request
238    backoff: Option<Interval>,
239    /// The maximum amount of requests to send concurrently
240    concurrency: usize,
241    /// The addresses we have yet to make requests for
242    queue: Vec<Address>,
243    /// The in progress requests
244    in_progress: FuturesUnordered<FetchFuture>,
245}
246
247impl ExternalFetcher {
248    fn new(fetcher: Arc<dyn ExternalFetcherT>, to_fetch: &[Address]) -> Self {
249        Self {
250            timeout: fetcher.timeout(),
251            backoff: None,
252            concurrency: fetcher.concurrency(),
253            fetcher,
254            queue: to_fetch.to_vec(),
255            in_progress: FuturesUnordered::new(),
256        }
257    }
258
259    fn queue_next_reqs(&mut self) {
260        while self.in_progress.len() < self.concurrency {
261            let Some(addr) = self.queue.pop() else { break };
262            let fetcher = Arc::clone(&self.fetcher);
263            self.in_progress.push(Box::pin(async move {
264                trace!(target: "evm::traces::external", ?addr, "fetching info");
265                let res = fetcher.fetch(addr).await;
266                (addr, res)
267            }));
268        }
269    }
270}
271
272impl Stream for ExternalFetcher {
273    type Item = (Address, (FetcherKind, Option<Metadata>));
274
275    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
276        let pin = self.get_mut();
277
278        let _guard =
279            info_span!("evm::traces::external", kind=?pin.fetcher.kind(), "ExternalFetcher")
280                .entered();
281
282        if pin.fetcher.invalid_api_key().load(Ordering::Relaxed) {
283            return Poll::Ready(None);
284        }
285
286        loop {
287            if let Some(mut backoff) = pin.backoff.take()
288                && backoff.poll_tick(cx).is_pending()
289            {
290                pin.backoff = Some(backoff);
291                return Poll::Pending;
292            }
293
294            pin.queue_next_reqs();
295
296            let mut made_progress_this_iter = false;
297            match pin.in_progress.poll_next_unpin(cx) {
298                Poll::Pending => {}
299                Poll::Ready(None) => return Poll::Ready(None),
300                Poll::Ready(Some((addr, res))) => {
301                    made_progress_this_iter = true;
302                    match res {
303                        Ok(metadata) => {
304                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata))));
305                        }
306                        Err(EtherscanError::ContractCodeNotVerified(_)) => {
307                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
308                        }
309                        Err(EtherscanError::RateLimitExceeded) => {
310                            warn!(target: "evm::traces::external", "rate limit exceeded on attempt");
311                            pin.backoff = Some(tokio::time::interval(pin.timeout));
312                            pin.queue.push(addr);
313                        }
314                        Err(EtherscanError::InvalidApiKey) => {
315                            warn!(target: "evm::traces::external", "invalid api key");
316                            // mark key as invalid
317                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
318                            return Poll::Ready(None);
319                        }
320                        Err(EtherscanError::BlockedByCloudflare) => {
321                            warn!(target: "evm::traces::external", "blocked by cloudflare");
322                            // mark key as invalid
323                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
324                            return Poll::Ready(None);
325                        }
326                        Err(err) => {
327                            warn!(target: "evm::traces::external", ?err, "could not get info");
328                            // Cache the failure so we don't re-fetch on subsequent arenas.
329                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
330                        }
331                    }
332                }
333            }
334
335            if !made_progress_this_iter {
336                return Poll::Pending;
337            }
338        }
339    }
340}
341
342#[derive(Debug, Clone, Copy, PartialEq, Eq)]
343enum FetcherKind {
344    Etherscan,
345    Sourcify,
346}
347
348#[async_trait::async_trait]
349trait ExternalFetcherT: Send + Sync {
350    fn kind(&self) -> FetcherKind;
351    fn timeout(&self) -> Duration;
352    fn concurrency(&self) -> usize;
353    fn invalid_api_key(&self) -> &AtomicBool;
354    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError>;
355}
356
357struct EtherscanFetcher {
358    client: foundry_block_explorers::Client,
359    invalid_api_key: AtomicBool,
360}
361
362impl EtherscanFetcher {
363    fn new(client: foundry_block_explorers::Client) -> Self {
364        Self { client, invalid_api_key: AtomicBool::new(false) }
365    }
366}
367
368#[async_trait::async_trait]
369impl ExternalFetcherT for EtherscanFetcher {
370    fn kind(&self) -> FetcherKind {
371        FetcherKind::Etherscan
372    }
373
374    fn timeout(&self) -> Duration {
375        Duration::from_secs(1)
376    }
377
378    fn concurrency(&self) -> usize {
379        5
380    }
381
382    fn invalid_api_key(&self) -> &AtomicBool {
383        &self.invalid_api_key
384    }
385
386    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
387        self.client.contract_source_code(address).await.map(|mut metadata| metadata.items.pop())
388    }
389}
390
391struct SourcifyFetcher {
392    client: reqwest::Client,
393    url: String,
394    invalid_api_key: AtomicBool,
395}
396
397impl SourcifyFetcher {
398    fn new(chain: Chain) -> Self {
399        Self {
400            client: reqwest::Client::new(),
401            url: format!("https://sourcify.dev/server/v2/contract/{}", chain.id()),
402            invalid_api_key: AtomicBool::new(false),
403        }
404    }
405}
406
407#[async_trait::async_trait]
408impl ExternalFetcherT for SourcifyFetcher {
409    fn kind(&self) -> FetcherKind {
410        FetcherKind::Sourcify
411    }
412
413    fn timeout(&self) -> Duration {
414        Duration::from_secs(1)
415    }
416
417    fn concurrency(&self) -> usize {
418        5
419    }
420
421    fn invalid_api_key(&self) -> &AtomicBool {
422        &self.invalid_api_key
423    }
424
425    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
426        let url = format!("{url}/{address}?fields=abi,compilation", url = self.url);
427        let response = self
428            .client
429            .get(url)
430            .send()
431            .await
432            .map_err(|e| EtherscanError::Unknown(e.to_string()))?;
433        let code = response.status();
434        match code.as_u16() {
435            // Not verified.
436            404 => return Err(EtherscanError::ContractCodeNotVerified(address)),
437            // Too many requests.
438            429 => return Err(EtherscanError::RateLimitExceeded),
439            _ => {}
440        }
441        let response: SourcifyResponse =
442            response.json().await.map_err(|e| EtherscanError::Unknown(e.to_string()))?;
443        trace!(target: "evm::traces::external", "Sourcify response for {address}: {response:#?}");
444        match response {
445            SourcifyResponse::Success(metadata) => Ok(Some(metadata.into())),
446            SourcifyResponse::Error(error) => Err(EtherscanError::Unknown(format!("{error:#?}"))),
447        }
448    }
449}
450
451/// Sourcify API response for `/v2/contract/{chainId}/{address}`.
452#[derive(Debug, Clone, Deserialize)]
453#[serde(untagged)]
454enum SourcifyResponse {
455    Success(SourcifyMetadata),
456    Error(SourcifyError),
457}
458
459#[derive(Debug, Clone, Deserialize)]
460#[serde(rename_all = "camelCase")]
461#[expect(dead_code)] // Used in Debug.
462struct SourcifyError {
463    custom_code: String,
464    message: String,
465    error_id: String,
466}
467
468#[derive(Debug, Clone, Deserialize)]
469#[serde(rename_all = "camelCase")]
470struct SourcifyMetadata {
471    #[serde(default)]
472    abi: Option<Box<serde_json::value::RawValue>>,
473    #[serde(default)]
474    compilation: Option<Compilation>,
475}
476
477#[derive(Debug, Clone, Deserialize)]
478#[serde(rename_all = "camelCase")]
479struct Compilation {
480    #[serde(default)]
481    compiler_version: String,
482    #[serde(default)]
483    name: String,
484}
485
486impl From<SourcifyMetadata> for Metadata {
487    fn from(metadata: SourcifyMetadata) -> Self {
488        let SourcifyMetadata { abi, compilation } = metadata;
489        let (contract_name, compiler_version) = compilation
490            .map(|c| (c.name, c.compiler_version))
491            .unwrap_or_else(|| (String::new(), String::new()));
492        // Defaulted fields may be fetched from sourcify but we don't make use of them.
493        Self {
494            source_code: foundry_block_explorers::contract::SourceCodeMetadata::Sources(
495                Default::default(),
496            ),
497            abi: Box::<str>::from(abi.unwrap_or_default()).into(),
498            contract_name,
499            compiler_version,
500            optimization_used: 0,
501            runs: 0,
502            constructor_arguments: Default::default(),
503            evm_version: String::new(),
504            library: String::new(),
505            license_type: String::new(),
506            proxy: 0,
507            implementation: None,
508            swarm_source: String::new(),
509        }
510    }
511}
foundry_evm_traces/identifier/external.rs

foundry_evm_traces/identifier/
external.rs
