foundry_evm_traces/identifier/
external.rs

1use super::{IdentifiedAddress, TraceIdentifier};
2use crate::debug::ContractSources;
3use alloy_primitives::{
4    Address,
5    map::{Entry, HashMap, HashSet},
6};
7use eyre::WrapErr;
8use foundry_block_explorers::{contract::Metadata, errors::EtherscanError};
9use foundry_common::compile::etherscan_project;
10use foundry_config::{Chain, Config};
11use futures::{
12    future::join_all,
13    stream::{FuturesUnordered, Stream, StreamExt},
14    task::{Context, Poll},
15};
16use revm_inspectors::tracing::types::CallTraceNode;
17use serde::Deserialize;
18use std::{
19    borrow::Cow,
20    pin::Pin,
21    sync::{
22        Arc,
23        atomic::{AtomicBool, Ordering},
24    },
25};
26use tokio::time::{Duration, Interval};
27
28/// A trace identifier that tries to identify addresses using Etherscan.
29pub struct ExternalIdentifier {
30    fetchers: Vec<Arc<dyn ExternalFetcherT>>,
31    /// Cached contracts.
32    contracts: HashMap<Address, (FetcherKind, Option<Metadata>)>,
33}
34
35impl ExternalIdentifier {
36    /// Creates a new external identifier with the given client
37    pub fn new(config: &Config, mut chain: Option<Chain>) -> eyre::Result<Option<Self>> {
38        if config.offline {
39            return Ok(None);
40        }
41
42        let config = match config.get_etherscan_config_with_chain(chain) {
43            Ok(Some(config)) => {
44                chain = config.chain;
45                Some(config)
46            }
47            Ok(None) => {
48                warn!(target: "evm::traces::external", "etherscan config not found");
49                None
50            }
51            Err(err) => {
52                warn!(target: "evm::traces::external", ?err, "failed to get etherscan config");
53                None
54            }
55        };
56
57        let mut fetchers = Vec::<Arc<dyn ExternalFetcherT>>::new();
58        if let Some(chain) = chain {
59            debug!(target: "evm::traces::external", ?chain, "using sourcify identifier");
60            fetchers.push(Arc::new(SourcifyFetcher::new(chain)));
61        }
62        if let Some(config) = config {
63            debug!(target: "evm::traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier");
64            match config.into_client() {
65                Ok(client) => {
66                    fetchers.push(Arc::new(EtherscanFetcher::new(client)));
67                }
68                Err(err) => {
69                    warn!(target: "evm::traces::external", ?err, "failed to create etherscan client");
70                }
71            }
72        }
73        if fetchers.is_empty() {
74            debug!(target: "evm::traces::external", "no fetchers enabled");
75            return Ok(None);
76        }
77
78        Ok(Some(Self { fetchers, contracts: Default::default() }))
79    }
80
81    /// Goes over the list of contracts we have pulled from the traces, clones their source from
82    /// Etherscan and compiles them locally, for usage in the debugger.
83    pub async fn get_compiled_contracts(&self) -> eyre::Result<ContractSources> {
84        // Collect contract info upfront so we can reference it in error messages
85        let contracts_info: Vec<_> = self
86            .contracts
87            .iter()
88            // filter out vyper files and contracts without metadata
89            .filter_map(|(addr, (_, metadata))| {
90                if let Some(metadata) = metadata.as_ref()
91                    && !metadata.is_vyper()
92                {
93                    Some((*addr, metadata))
94                } else {
95                    None
96                }
97            })
98            .collect();
99
100        let outputs_fut = contracts_info
101            .iter()
102            .map(|(addr, metadata)| async move {
103                sh_println!("Compiling: {} {addr}", metadata.contract_name)?;
104                let root = tempfile::tempdir()?;
105                let root_path = root.path();
106                let project = etherscan_project(metadata, root_path)?;
107                let output = project.compile()?;
108                if output.has_compiler_errors() {
109                    eyre::bail!("{output}")
110                }
111
112                Ok((project, output, root))
113            })
114            .collect::<Vec<_>>();
115
116        // poll all the futures concurrently
117        let outputs = join_all(outputs_fut).await;
118
119        let mut sources: ContractSources = Default::default();
120
121        // construct the map
122        for (idx, res) in outputs.into_iter().enumerate() {
123            let (addr, metadata) = &contracts_info[idx];
124            let name = &metadata.contract_name;
125            let (project, output, _) =
126                res.wrap_err_with(|| format!("Failed to compile contract {name} at {addr}"))?;
127            sources
128                .insert(&output, project.root(), None)
129                .wrap_err_with(|| format!("Failed to insert contract {name} at {addr}"))?;
130        }
131
132        Ok(sources)
133    }
134
135    fn identify_from_metadata(
136        &self,
137        address: Address,
138        metadata: &Metadata,
139    ) -> IdentifiedAddress<'static> {
140        let label = metadata.contract_name.clone();
141        let abi = metadata.abi().ok().map(Cow::Owned);
142        IdentifiedAddress {
143            address,
144            label: Some(label.clone()),
145            contract: Some(label),
146            abi,
147            artifact_id: None,
148        }
149    }
150}
151
152impl TraceIdentifier for ExternalIdentifier {
153    fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec<IdentifiedAddress<'_>> {
154        if nodes.is_empty() {
155            return Vec::new();
156        }
157
158        trace!(target: "evm::traces::external", "identify {} addresses", nodes.len());
159
160        let mut identities = Vec::new();
161        let mut to_fetch = HashSet::new();
162
163        // Check cache first.
164        for &node in nodes {
165            let address = node.trace.address;
166            if let Some((_, metadata)) = self.contracts.get(&address) {
167                if let Some(metadata) = metadata {
168                    identities.push(self.identify_from_metadata(address, metadata));
169                } else {
170                    // Do nothing. We know that this contract was not verified.
171                }
172            } else {
173                to_fetch.insert(address);
174            }
175        }
176
177        if to_fetch.is_empty() {
178            return identities;
179        }
180        trace!(target: "evm::traces::external", "fetching {} addresses", to_fetch.len());
181
182        let to_fetch = to_fetch.into_iter().collect::<Vec<_>>();
183        let fetchers =
184            self.fetchers.iter().map(|fetcher| ExternalFetcher::new(fetcher.clone(), &to_fetch));
185        let fetched_identities = foundry_common::block_on(
186            futures::stream::select_all(fetchers)
187                .filter_map(|(address, value)| {
188                    let addr = value
189                        .1
190                        .as_ref()
191                        .map(|metadata| self.identify_from_metadata(address, metadata));
192                    match self.contracts.entry(address) {
193                        Entry::Occupied(mut occupied_entry) => {
194                            // Override if:
195                            // - new is from Etherscan and old is not
196                            // - new is Some and old is None, meaning verified only in one source
197                            if !matches!(occupied_entry.get().0, FetcherKind::Etherscan)
198                                || value.1.is_none()
199                            {
200                                occupied_entry.insert(value);
201                            }
202                        }
203                        Entry::Vacant(vacant_entry) => {
204                            vacant_entry.insert(value);
205                        }
206                    }
207                    async move { addr }
208                })
209                .collect::<Vec<IdentifiedAddress<'_>>>(),
210        );
211        trace!(target: "evm::traces::external", "fetched {} addresses: {fetched_identities:#?}", fetched_identities.len());
212
213        identities.extend(fetched_identities);
214        identities
215    }
216}
217
218type FetchFuture =
219    Pin<Box<dyn Future<Output = (Address, Result<Option<Metadata>, EtherscanError>)>>>;
220
221/// A rate limit aware fetcher.
222///
223/// Fetches information about multiple addresses concurrently, while respecting rate limits.
224struct ExternalFetcher {
225    /// The fetcher
226    fetcher: Arc<dyn ExternalFetcherT>,
227    /// The time we wait if we hit the rate limit
228    timeout: Duration,
229    /// The interval we are currently waiting for before making a new request
230    backoff: Option<Interval>,
231    /// The maximum amount of requests to send concurrently
232    concurrency: usize,
233    /// The addresses we have yet to make requests for
234    queue: Vec<Address>,
235    /// The in progress requests
236    in_progress: FuturesUnordered<FetchFuture>,
237}
238
239impl ExternalFetcher {
240    fn new(fetcher: Arc<dyn ExternalFetcherT>, to_fetch: &[Address]) -> Self {
241        Self {
242            timeout: fetcher.timeout(),
243            backoff: None,
244            concurrency: fetcher.concurrency(),
245            fetcher,
246            queue: to_fetch.to_vec(),
247            in_progress: FuturesUnordered::new(),
248        }
249    }
250
251    fn queue_next_reqs(&mut self) {
252        while self.in_progress.len() < self.concurrency {
253            let Some(addr) = self.queue.pop() else { break };
254            let fetcher = Arc::clone(&self.fetcher);
255            self.in_progress.push(Box::pin(async move {
256                trace!(target: "evm::traces::external", ?addr, "fetching info");
257                let res = fetcher.fetch(addr).await;
258                (addr, res)
259            }));
260        }
261    }
262}
263
264impl Stream for ExternalFetcher {
265    type Item = (Address, (FetcherKind, Option<Metadata>));
266
267    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
268        let pin = self.get_mut();
269
270        let _guard =
271            info_span!("evm::traces::external", kind=?pin.fetcher.kind(), "ExternalFetcher")
272                .entered();
273
274        if pin.fetcher.invalid_api_key().load(Ordering::Relaxed) {
275            return Poll::Ready(None);
276        }
277
278        loop {
279            if let Some(mut backoff) = pin.backoff.take()
280                && backoff.poll_tick(cx).is_pending()
281            {
282                pin.backoff = Some(backoff);
283                return Poll::Pending;
284            }
285
286            pin.queue_next_reqs();
287
288            let mut made_progress_this_iter = false;
289            match pin.in_progress.poll_next_unpin(cx) {
290                Poll::Pending => {}
291                Poll::Ready(None) => return Poll::Ready(None),
292                Poll::Ready(Some((addr, res))) => {
293                    made_progress_this_iter = true;
294                    match res {
295                        Ok(metadata) => {
296                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata))));
297                        }
298                        Err(EtherscanError::ContractCodeNotVerified(_)) => {
299                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
300                        }
301                        Err(EtherscanError::RateLimitExceeded) => {
302                            warn!(target: "evm::traces::external", "rate limit exceeded on attempt");
303                            pin.backoff = Some(tokio::time::interval(pin.timeout));
304                            pin.queue.push(addr);
305                        }
306                        Err(EtherscanError::InvalidApiKey) => {
307                            warn!(target: "evm::traces::external", "invalid api key");
308                            // mark key as invalid
309                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
310                            return Poll::Ready(None);
311                        }
312                        Err(EtherscanError::BlockedByCloudflare) => {
313                            warn!(target: "evm::traces::external", "blocked by cloudflare");
314                            // mark key as invalid
315                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
316                            return Poll::Ready(None);
317                        }
318                        Err(err) => {
319                            warn!(target: "evm::traces::external", ?err, "could not get info");
320                        }
321                    }
322                }
323            }
324
325            if !made_progress_this_iter {
326                return Poll::Pending;
327            }
328        }
329    }
330}
331
332#[derive(Debug, Clone, Copy, PartialEq, Eq)]
333enum FetcherKind {
334    Etherscan,
335    Sourcify,
336}
337
338#[async_trait::async_trait]
339trait ExternalFetcherT: Send + Sync {
340    fn kind(&self) -> FetcherKind;
341    fn timeout(&self) -> Duration;
342    fn concurrency(&self) -> usize;
343    fn invalid_api_key(&self) -> &AtomicBool;
344    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError>;
345}
346
347struct EtherscanFetcher {
348    client: foundry_block_explorers::Client,
349    invalid_api_key: AtomicBool,
350}
351
352impl EtherscanFetcher {
353    fn new(client: foundry_block_explorers::Client) -> Self {
354        Self { client, invalid_api_key: AtomicBool::new(false) }
355    }
356}
357
358#[async_trait::async_trait]
359impl ExternalFetcherT for EtherscanFetcher {
360    fn kind(&self) -> FetcherKind {
361        FetcherKind::Etherscan
362    }
363
364    fn timeout(&self) -> Duration {
365        Duration::from_secs(1)
366    }
367
368    fn concurrency(&self) -> usize {
369        5
370    }
371
372    fn invalid_api_key(&self) -> &AtomicBool {
373        &self.invalid_api_key
374    }
375
376    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
377        self.client.contract_source_code(address).await.map(|mut metadata| metadata.items.pop())
378    }
379}
380
381struct SourcifyFetcher {
382    client: reqwest::Client,
383    url: String,
384    invalid_api_key: AtomicBool,
385}
386
387impl SourcifyFetcher {
388    fn new(chain: Chain) -> Self {
389        Self {
390            client: reqwest::Client::new(),
391            url: format!("https://sourcify.dev/server/v2/contract/{}", chain.id()),
392            invalid_api_key: AtomicBool::new(false),
393        }
394    }
395}
396
397#[async_trait::async_trait]
398impl ExternalFetcherT for SourcifyFetcher {
399    fn kind(&self) -> FetcherKind {
400        FetcherKind::Sourcify
401    }
402
403    fn timeout(&self) -> Duration {
404        Duration::from_secs(1)
405    }
406
407    fn concurrency(&self) -> usize {
408        5
409    }
410
411    fn invalid_api_key(&self) -> &AtomicBool {
412        &self.invalid_api_key
413    }
414
415    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
416        let url = format!("{url}/{address}?fields=abi,compilation", url = self.url);
417        let response = self.client.get(url).send().await?;
418        let code = response.status();
419        match code.as_u16() {
420            // Not verified.
421            404 => return Err(EtherscanError::ContractCodeNotVerified(address)),
422            // Too many requests.
423            429 => return Err(EtherscanError::RateLimitExceeded),
424            _ => {}
425        }
426        let response: SourcifyResponse = response.json().await?;
427        trace!(target: "evm::traces::external", "Sourcify response for {address}: {response:#?}");
428        match response {
429            SourcifyResponse::Success(metadata) => Ok(Some(metadata.into())),
430            SourcifyResponse::Error(error) => Err(EtherscanError::Unknown(format!("{error:#?}"))),
431        }
432    }
433}
434
435/// Sourcify API response for `/v2/contract/{chainId}/{address}`.
436#[derive(Debug, Clone, Deserialize)]
437#[serde(untagged)]
438enum SourcifyResponse {
439    Success(SourcifyMetadata),
440    Error(SourcifyError),
441}
442
443#[derive(Debug, Clone, Deserialize)]
444#[serde(rename_all = "camelCase")]
445#[expect(dead_code)] // Used in Debug.
446struct SourcifyError {
447    custom_code: String,
448    message: String,
449    error_id: String,
450}
451
452#[derive(Debug, Clone, Deserialize)]
453#[serde(rename_all = "camelCase")]
454struct SourcifyMetadata {
455    #[serde(default)]
456    abi: Option<Box<serde_json::value::RawValue>>,
457    #[serde(default)]
458    compilation: Option<Compilation>,
459}
460
461#[derive(Debug, Clone, Deserialize)]
462#[serde(rename_all = "camelCase")]
463struct Compilation {
464    #[serde(default)]
465    compiler_version: String,
466    #[serde(default)]
467    name: String,
468}
469
470impl From<SourcifyMetadata> for Metadata {
471    fn from(metadata: SourcifyMetadata) -> Self {
472        let SourcifyMetadata { abi, compilation } = metadata;
473        let (contract_name, compiler_version) = compilation
474            .map(|c| (c.name, c.compiler_version))
475            .unwrap_or_else(|| (String::new(), String::new()));
476        // Defaulted fields may be fetched from sourcify but we don't make use of them.
477        Self {
478            source_code: foundry_block_explorers::contract::SourceCodeMetadata::Sources(
479                Default::default(),
480            ),
481            abi: Box::<str>::from(abi.unwrap_or_default()).into(),
482            contract_name,
483            compiler_version,
484            optimization_used: 0,
485            runs: 0,
486            constructor_arguments: Default::default(),
487            evm_version: String::new(),
488            library: String::new(),
489            license_type: String::new(),
490            proxy: 0,
491            implementation: None,
492            swarm_source: String::new(),
493        }
494    }
495}
foundry_evm_traces/identifier/external.rs

foundry_evm_traces/identifier/
external.rs
