foundry_evm_traces/identifier/
external.rs

1use super::{IdentifiedAddress, TraceIdentifier};
2use crate::debug::ContractSources;
3use alloy_primitives::{
4    Address,
5    map::{Entry, HashMap, HashSet},
6};
7use eyre::WrapErr;
8use foundry_block_explorers::{contract::Metadata, errors::EtherscanError};
9use foundry_common::compile::etherscan_project;
10use foundry_config::{Chain, Config};
11use futures::{
12    future::join_all,
13    stream::{FuturesUnordered, Stream, StreamExt},
14    task::{Context, Poll},
15};
16use revm_inspectors::tracing::types::CallTraceNode;
17use serde::Deserialize;
18use std::{
19    borrow::Cow,
20    pin::Pin,
21    sync::{
22        Arc,
23        atomic::{AtomicBool, Ordering},
24    },
25};
26use tokio::time::{Duration, Interval};
27
28/// A trace identifier that tries to identify addresses using Etherscan.
29pub struct ExternalIdentifier {
30    fetchers: Vec<Arc<dyn ExternalFetcherT>>,
31    /// Cached contracts.
32    contracts: HashMap<Address, (FetcherKind, Option<Metadata>)>,
33}
34
35impl ExternalIdentifier {
36    /// Creates a new external identifier with the given client
37    pub fn new(config: &Config, mut chain: Option<Chain>) -> eyre::Result<Option<Self>> {
38        if config.offline {
39            return Ok(None);
40        }
41
42        let no_proxy = config.eth_rpc_no_proxy;
43        let config = match config.get_etherscan_config_with_chain(chain) {
44            Ok(Some(config)) => {
45                chain = config.chain;
46                Some(config)
47            }
48            Ok(None) => {
49                warn!(target: "evm::traces::external", "etherscan config not found");
50                None
51            }
52            Err(err) => {
53                warn!(target: "evm::traces::external", ?err, "failed to get etherscan config");
54                None
55            }
56        };
57
58        let mut fetchers = Vec::<Arc<dyn ExternalFetcherT>>::new();
59        if let Some(chain) = chain {
60            debug!(target: "evm::traces::external", ?chain, "using sourcify identifier");
61            fetchers.push(Arc::new(SourcifyFetcher::new(chain)));
62        }
63        if let Some(config) = config {
64            debug!(target: "evm::traces::external", chain=?config.chain, url=?config.api_url, "using etherscan identifier");
65            match config.into_client_with_no_proxy(no_proxy) {
66                Ok(client) => {
67                    fetchers.push(Arc::new(EtherscanFetcher::new(client)));
68                }
69                Err(err) => {
70                    warn!(target: "evm::traces::external", ?err, "failed to create etherscan client");
71                }
72            }
73        }
74        if fetchers.is_empty() {
75            debug!(target: "evm::traces::external", "no fetchers enabled");
76            return Ok(None);
77        }
78
79        Ok(Some(Self { fetchers, contracts: Default::default() }))
80    }
81
82    /// Goes over the list of contracts we have pulled from the traces, clones their source from
83    /// Etherscan and compiles them locally, for usage in the debugger.
84    pub async fn get_compiled_contracts(&self) -> eyre::Result<ContractSources> {
85        // Collect contract info upfront so we can reference it in error messages
86        let contracts_info: Vec<_> = self
87            .contracts
88            .iter()
89            // filter out vyper files and contracts without metadata
90            .filter_map(|(addr, (_, metadata))| {
91                if let Some(metadata) = metadata.as_ref()
92                    && !metadata.is_vyper()
93                {
94                    Some((*addr, metadata))
95                } else {
96                    None
97                }
98            })
99            .collect();
100
101        let outputs_fut = contracts_info
102            .iter()
103            .map(|(addr, metadata)| async move {
104                sh_println!("Compiling: {} {addr}", metadata.contract_name)?;
105                let root = tempfile::tempdir()?;
106                let root_path = root.path();
107                let project = etherscan_project(metadata, root_path)?;
108                let output = project.compile()?;
109                if output.has_compiler_errors() {
110                    eyre::bail!("{output}");
111                }
112
113                Ok((project, output, root))
114            })
115            .collect::<Vec<_>>();
116
117        // poll all the futures concurrently
118        let outputs = join_all(outputs_fut).await;
119
120        let mut sources: ContractSources = Default::default();
121
122        // construct the map
123        for (idx, res) in outputs.into_iter().enumerate() {
124            let (addr, metadata) = &contracts_info[idx];
125            let name = &metadata.contract_name;
126            let (project, output, _) =
127                res.wrap_err_with(|| format!("Failed to compile contract {name} at {addr}"))?;
128            sources
129                .insert(&output, project.root(), None)
130                .wrap_err_with(|| format!("Failed to insert contract {name} at {addr}"))?;
131        }
132
133        Ok(sources)
134    }
135
136    fn identify_from_metadata(
137        &self,
138        address: Address,
139        metadata: &Metadata,
140    ) -> IdentifiedAddress<'static> {
141        let label = metadata.contract_name.clone();
142        let abi = metadata.abi().ok().map(Cow::Owned);
143        IdentifiedAddress {
144            address,
145            label: Some(label.clone()),
146            contract: Some(label),
147            abi,
148            constructor_args_offset: None,
149            artifact_id: None,
150        }
151    }
152}
153
154impl TraceIdentifier for ExternalIdentifier {
155    fn identify_addresses(&mut self, nodes: &[&CallTraceNode]) -> Vec<IdentifiedAddress<'_>> {
156        if nodes.is_empty() {
157            return Vec::new();
158        }
159
160        trace!(target: "evm::traces::external", "identify {} addresses", nodes.len());
161
162        let mut identities = Vec::new();
163        let mut to_fetch = HashSet::new();
164
165        // Check cache first.
166        for &node in nodes {
167            let address = node.trace.address;
168            if let Some((_, metadata)) = self.contracts.get(&address) {
169                if let Some(metadata) = metadata {
170                    identities.push(self.identify_from_metadata(address, metadata));
171                } else {
172                    // Do nothing. We know that this contract was not verified.
173                }
174            } else {
175                to_fetch.insert(address);
176            }
177        }
178
179        if to_fetch.is_empty() {
180            return identities;
181        }
182        trace!(target: "evm::traces::external", "fetching {} addresses", to_fetch.len());
183
184        let to_fetch = to_fetch.into_iter().collect::<Vec<_>>();
185        let fetchers =
186            self.fetchers.iter().map(|fetcher| ExternalFetcher::new(fetcher.clone(), &to_fetch));
187        let fetched_identities = foundry_common::block_on(
188            futures::stream::select_all(fetchers)
189                .filter_map(|(address, value)| {
190                    let addr = value
191                        .1
192                        .as_ref()
193                        .map(|metadata| self.identify_from_metadata(address, metadata));
194                    match self.contracts.entry(address) {
195                        Entry::Occupied(mut occupied_entry) => {
196                            let old = occupied_entry.get();
197                            // Only override when the new result is strictly better:
198                            // - new has metadata and old doesn't, OR
199                            // - both have metadata but new is from Etherscan and old is not.
200                            // Never downgrade a successful lookup to None.
201                            let should_replace = match (&old.1, &value.1) {
202                                (None, Some(_)) => true,
203                                (Some(_), None) => false,
204                                _ => {
205                                    matches!(value.0, FetcherKind::Etherscan)
206                                        && !matches!(old.0, FetcherKind::Etherscan)
207                                }
208                            };
209                            if should_replace {
210                                occupied_entry.insert(value);
211                            }
212                        }
213                        Entry::Vacant(vacant_entry) => {
214                            vacant_entry.insert(value);
215                        }
216                    }
217                    async move { addr }
218                })
219                .collect::<Vec<IdentifiedAddress<'_>>>(),
220        );
221        trace!(target: "evm::traces::external", "fetched {} addresses: {fetched_identities:#?}", fetched_identities.len());
222
223        identities.extend(fetched_identities);
224        identities
225    }
226}
227
228type FetchFuture =
229    Pin<Box<dyn Future<Output = (Address, Result<Option<Metadata>, EtherscanError>)>>>;
230
231/// Maximum number of times a single address is retried through a transient Cloudflare
232/// block before we give up on it. Bounded so a persistent block can't loop forever.
233const MAX_CLOUDFLARE_RETRIES: u32 = 5;
234
235/// A rate limit aware fetcher.
236///
237/// Fetches information about multiple addresses concurrently, while respecting rate limits.
238struct ExternalFetcher {
239    /// The fetcher
240    fetcher: Arc<dyn ExternalFetcherT>,
241    /// The time we wait if we hit the rate limit
242    timeout: Duration,
243    /// The interval we are currently waiting for before making a new request
244    backoff: Option<Interval>,
245    /// The maximum amount of requests to send concurrently
246    concurrency: usize,
247    /// The addresses we have yet to make requests for
248    queue: Vec<Address>,
249    /// The in progress requests
250    in_progress: FuturesUnordered<FetchFuture>,
251    /// Per-address retry counter for transient Cloudflare blocks.
252    attempts: HashMap<Address, u32>,
253}
254
255impl ExternalFetcher {
256    fn new(fetcher: Arc<dyn ExternalFetcherT>, to_fetch: &[Address]) -> Self {
257        Self {
258            timeout: fetcher.timeout(),
259            backoff: None,
260            concurrency: fetcher.concurrency(),
261            fetcher,
262            queue: to_fetch.to_vec(),
263            in_progress: FuturesUnordered::new(),
264            attempts: HashMap::default(),
265        }
266    }
267
268    fn queue_next_reqs(&mut self) {
269        while self.in_progress.len() < self.concurrency {
270            let Some(addr) = self.queue.pop() else { break };
271            let fetcher = Arc::clone(&self.fetcher);
272            self.in_progress.push(Box::pin(async move {
273                trace!(target: "evm::traces::external", ?addr, "fetching info");
274                let res = fetcher.fetch(addr).await;
275                (addr, res)
276            }));
277        }
278    }
279}
280
281impl Stream for ExternalFetcher {
282    type Item = (Address, (FetcherKind, Option<Metadata>));
283
284    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
285        let pin = self.get_mut();
286
287        let _guard =
288            info_span!("evm::traces::external", kind=?pin.fetcher.kind(), "ExternalFetcher")
289                .entered();
290
291        if pin.fetcher.invalid_api_key().load(Ordering::Relaxed) {
292            return Poll::Ready(None);
293        }
294
295        loop {
296            if let Some(mut backoff) = pin.backoff.take()
297                && backoff.poll_tick(cx).is_pending()
298            {
299                pin.backoff = Some(backoff);
300                return Poll::Pending;
301            }
302
303            pin.queue_next_reqs();
304
305            let mut made_progress_this_iter = false;
306            match pin.in_progress.poll_next_unpin(cx) {
307                Poll::Pending => {}
308                Poll::Ready(None) => return Poll::Ready(None),
309                Poll::Ready(Some((addr, res))) => {
310                    made_progress_this_iter = true;
311                    match res {
312                        Ok(metadata) => {
313                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), metadata))));
314                        }
315                        Err(EtherscanError::ContractCodeNotVerified(_)) => {
316                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
317                        }
318                        Err(EtherscanError::RateLimitExceeded) => {
319                            warn!(target: "evm::traces::external", "rate limit exceeded on attempt");
320                            pin.backoff = Some(tokio::time::interval(pin.timeout));
321                            pin.queue.push(addr);
322                        }
323                        Err(EtherscanError::InvalidApiKey) => {
324                            warn!(target: "evm::traces::external", "invalid api key");
325                            // mark key as invalid
326                            pin.fetcher.invalid_api_key().store(true, Ordering::Relaxed);
327                            return Poll::Ready(None);
328                        }
329                        Err(EtherscanError::BlockedByCloudflare) => {
330                            // A Cloudflare block is transient rate limiting (often triggered
331                            // by request bursts), not a permanent failure like an invalid key.
332                            // Back off and retry the address a bounded number of times instead
333                            // of aborting the whole stream, which would abandon every still-
334                            // queued address and leave traces only partially decoded (#9880).
335                            let attempts = {
336                                let entry = pin.attempts.entry(addr).or_default();
337                                *entry += 1;
338                                *entry
339                            };
340                            if attempts <= MAX_CLOUDFLARE_RETRIES {
341                                warn!(target: "evm::traces::external", attempts, "blocked by cloudflare, backing off");
342                                pin.backoff = Some(tokio::time::interval(pin.timeout));
343                                pin.queue.push(addr);
344                            } else {
345                                warn!(target: "evm::traces::external", "blocked by cloudflare, giving up on address");
346                                return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
347                            }
348                        }
349                        Err(err) => {
350                            warn!(target: "evm::traces::external", ?err, "could not get info");
351                            // Cache the failure so we don't re-fetch on subsequent arenas.
352                            return Poll::Ready(Some((addr, (pin.fetcher.kind(), None))));
353                        }
354                    }
355                }
356            }
357
358            if !made_progress_this_iter {
359                return Poll::Pending;
360            }
361        }
362    }
363}
364
365#[derive(Debug, Clone, Copy, PartialEq, Eq)]
366enum FetcherKind {
367    Etherscan,
368    Sourcify,
369}
370
371#[async_trait::async_trait]
372trait ExternalFetcherT: Send + Sync {
373    fn kind(&self) -> FetcherKind;
374    fn timeout(&self) -> Duration;
375    fn concurrency(&self) -> usize;
376    fn invalid_api_key(&self) -> &AtomicBool;
377    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError>;
378}
379
380struct EtherscanFetcher {
381    client: foundry_block_explorers::Client,
382    invalid_api_key: AtomicBool,
383}
384
385impl EtherscanFetcher {
386    const fn new(client: foundry_block_explorers::Client) -> Self {
387        Self { client, invalid_api_key: AtomicBool::new(false) }
388    }
389}
390
391#[async_trait::async_trait]
392impl ExternalFetcherT for EtherscanFetcher {
393    fn kind(&self) -> FetcherKind {
394        FetcherKind::Etherscan
395    }
396
397    fn timeout(&self) -> Duration {
398        Duration::from_secs(1)
399    }
400
401    fn concurrency(&self) -> usize {
402        5
403    }
404
405    fn invalid_api_key(&self) -> &AtomicBool {
406        &self.invalid_api_key
407    }
408
409    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
410        self.client.contract_source_code(address).await.map(|mut metadata| metadata.items.pop())
411    }
412}
413
414struct SourcifyFetcher {
415    client: reqwest::Client,
416    url: String,
417    invalid_api_key: AtomicBool,
418}
419
420impl SourcifyFetcher {
421    fn new(chain: Chain) -> Self {
422        Self {
423            client: reqwest::Client::new(),
424            url: format!("https://sourcify.dev/server/v2/contract/{}", chain.id()),
425            invalid_api_key: AtomicBool::new(false),
426        }
427    }
428}
429
430#[async_trait::async_trait]
431impl ExternalFetcherT for SourcifyFetcher {
432    fn kind(&self) -> FetcherKind {
433        FetcherKind::Sourcify
434    }
435
436    fn timeout(&self) -> Duration {
437        Duration::from_secs(1)
438    }
439
440    fn concurrency(&self) -> usize {
441        5
442    }
443
444    fn invalid_api_key(&self) -> &AtomicBool {
445        &self.invalid_api_key
446    }
447
448    async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
449        let url = format!("{url}/{address}?fields=abi,compilation", url = self.url);
450        let response = self
451            .client
452            .get(url)
453            .send()
454            .await
455            .map_err(|e| EtherscanError::Unknown(e.to_string()))?;
456        let code = response.status();
457        match code.as_u16() {
458            // Not verified.
459            404 => return Err(EtherscanError::ContractCodeNotVerified(address)),
460            // Too many requests.
461            429 => return Err(EtherscanError::RateLimitExceeded),
462            _ => {}
463        }
464        let response: SourcifyResponse =
465            response.json().await.map_err(|e| EtherscanError::Unknown(e.to_string()))?;
466        trace!(target: "evm::traces::external", "Sourcify response for {address}: {response:#?}");
467        match response {
468            SourcifyResponse::Success(metadata) => Ok(Some(metadata.into())),
469            SourcifyResponse::Error(error) => Err(EtherscanError::Unknown(format!("{error:#?}"))),
470        }
471    }
472}
473
474/// Sourcify API response for `/v2/contract/{chainId}/{address}`.
475#[derive(Debug, Clone, Deserialize)]
476#[serde(untagged)]
477enum SourcifyResponse {
478    Success(SourcifyMetadata),
479    Error(SourcifyError),
480}
481
482#[derive(Debug, Clone, Deserialize)]
483#[serde(rename_all = "camelCase")]
484#[expect(dead_code)] // Used in Debug.
485struct SourcifyError {
486    custom_code: String,
487    message: String,
488    error_id: String,
489}
490
491#[derive(Debug, Clone, Deserialize)]
492#[serde(rename_all = "camelCase")]
493struct SourcifyMetadata {
494    #[serde(default)]
495    abi: Option<Box<serde_json::value::RawValue>>,
496    #[serde(default)]
497    compilation: Option<Compilation>,
498}
499
500#[derive(Debug, Clone, Deserialize)]
501#[serde(rename_all = "camelCase")]
502struct Compilation {
503    #[serde(default)]
504    compiler_version: String,
505    #[serde(default)]
506    name: String,
507}
508
509impl From<SourcifyMetadata> for Metadata {
510    fn from(metadata: SourcifyMetadata) -> Self {
511        let SourcifyMetadata { abi, compilation } = metadata;
512        let (contract_name, compiler_version) = compilation
513            .map(|c| (c.name, c.compiler_version))
514            .unwrap_or_else(|| (String::new(), String::new()));
515        // Defaulted fields may be fetched from sourcify but we don't make use of them.
516        Self {
517            source_code: foundry_block_explorers::contract::SourceCodeMetadata::Sources(
518                Default::default(),
519            ),
520            abi: Box::<str>::from(abi.unwrap_or_default()).into(),
521            contract_name,
522            compiler_version,
523            optimization_used: 0,
524            runs: 0,
525            constructor_arguments: Default::default(),
526            evm_version: String::new(),
527            library: String::new(),
528            license_type: String::new(),
529            proxy: 0,
530            implementation: None,
531            swarm_source: String::new(),
532        }
533    }
534}
535
536#[cfg(test)]
537mod tests {
538    use super::*;
539    use std::{collections::HashSet as StdHashSet, sync::Mutex};
540
541    /// Fetcher that returns a transient Cloudflare block the first time it sees an address, then
542    /// succeeds. Mirrors Etherscan/Cloudflare throttling a burst of concurrent requests.
543    struct FlakyCloudflareFetcher {
544        seen: Mutex<StdHashSet<Address>>,
545        invalid: AtomicBool,
546    }
547
548    #[async_trait::async_trait]
549    impl ExternalFetcherT for FlakyCloudflareFetcher {
550        fn kind(&self) -> FetcherKind {
551            FetcherKind::Etherscan
552        }
553        fn timeout(&self) -> Duration {
554            Duration::from_millis(1)
555        }
556        fn concurrency(&self) -> usize {
557            1
558        }
559        fn invalid_api_key(&self) -> &AtomicBool {
560            &self.invalid
561        }
562        async fn fetch(&self, address: Address) -> Result<Option<Metadata>, EtherscanError> {
563            let first_time = self.seen.lock().unwrap().insert(address);
564            if first_time { Err(EtherscanError::BlockedByCloudflare) } else { Ok(None) }
565        }
566    }
567
568    /// Regression test for #9880: a transient Cloudflare block on one address must not abandon the
569    /// rest of the queue. Before the fix the fetcher returned `Poll::Ready(None)` on the first
570    /// block, ending the stream and leaving later addresses unidentified (partial trace decoding).
571    #[tokio::test]
572    async fn cloudflare_block_retries_instead_of_abandoning_queue() {
573        let addrs: Vec<Address> = (1u8..=4).map(Address::with_last_byte).collect();
574        let fetcher: Arc<dyn ExternalFetcherT> = Arc::new(FlakyCloudflareFetcher {
575            seen: Mutex::new(StdHashSet::new()),
576            invalid: AtomicBool::new(false),
577        });
578
579        let collected: Vec<_> = ExternalFetcher::new(fetcher, &addrs).collect().await;
580
581        let got: StdHashSet<Address> = collected.into_iter().map(|(addr, _)| addr).collect();
582        let want: StdHashSet<Address> = addrs.into_iter().collect();
583        assert_eq!(got, want, "every address must be yielded despite a transient cloudflare block");
584    }
585}
foundry_evm_traces/identifier/external.rs

foundry_evm_traces/identifier/
external.rs
