diff --git a/api-server/CHANGELOG.md b/api-server/CHANGELOG.md index b1436f9ea8..d5fee60258 100644 --- a/api-server/CHANGELOG.md +++ b/api-server/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to the API server will be documented in this file. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Fixed +- The scanner's fast catch-up mode no longer leaves the address and token transaction history permanently incomplete: a backfill pass now rebuilds the deferred history rows from the locally stored blocks after catching up, without touching balances or UTXOs. Databases written by builds predating the history watermark are seeded at startup so their history is rebuilt too. + ## [1.4.5] - 2026-11-10 ### Fixed diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index c3bd9417c8..1f3ce43c88 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -101,6 +101,7 @@ struct ApiServerInMemoryStorage { genesis_block: Arc>, number_of_coin_decimals: u8, storage_version: u32, + secondary_indexed_height: Option, // kept so that order/pool identifiers can be encoded exactly like the postgres // backend does (bech32 address strings), keeping tie-break orders identical chain_config: ChainConfig, @@ -131,6 +132,7 @@ impl ApiServerInMemoryStorage { genesis_block: chain_config.genesis_block().clone(), number_of_coin_decimals: chain_config.coin_decimals(), storage_version: CURRENT_STORAGE_VERSION, + secondary_indexed_height: None, chain_config: chain_config.clone(), } } @@ -406,6 +408,37 @@ impl ApiServerInMemoryStorage { Ok(version_table_handle) } + fn secondary_indexed_height(&self) -> Result, ApiServerStorageError> { + Ok(self.secondary_indexed_height) + } + + fn get_block_transaction_indices( + &self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError> { + let Some(block_id) = self.main_chain_blocks_table.get(&block_height) else { + return Ok(Vec::new()); + }; + Ok(self + .ordered_transaction_table + .iter() + .filter(|(_, tx_id)| { + self.transaction_table + .get(*tx_id) + .is_some_and(|(owning_block_id, _)| owning_block_id == block_id) + }) + .map(|(index, tx_id)| (*tx_id, *index)) + .collect()) + } + + fn set_secondary_indexed_height( + &mut self, + block_height: BlockHeight, + ) -> Result<(), ApiServerStorageError> { + self.secondary_indexed_height = Some(block_height); + Ok(()) + } + fn get_best_block(&self) -> Result { let result = self.main_chain_blocks_table.last_key_value().map_or_else( || { diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs index 7ca6e41bbb..0066d9544c 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs @@ -215,6 +215,17 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { self.transaction.get_best_block() } + async fn secondary_indexed_height(&self) -> Result, ApiServerStorageError> { + self.transaction.secondary_indexed_height() + } + + async fn get_block_transaction_indices( + &self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError> { + self.transaction.get_block_transaction_indices(block_height) + } + async fn get_block_aux_data( &self, block_id: Id, diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs index 0bdc964e9e..de0f4626d6 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs @@ -36,6 +36,13 @@ use super::ApiServerInMemoryStorageTransactionalRw; #[async_trait::async_trait] impl ApiServerStorageWrite for ApiServerInMemoryStorageTransactionalRw<'_> { + async fn set_secondary_indexed_height( + &mut self, + block_height: BlockHeight, + ) -> Result<(), ApiServerStorageError> { + self.transaction.set_secondary_indexed_height(block_height) + } + async fn reinitialize_storage( &mut self, chain_config: &ChainConfig, @@ -381,6 +388,17 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { self.transaction.get_best_block() } + async fn secondary_indexed_height(&self) -> Result, ApiServerStorageError> { + self.transaction.secondary_indexed_height() + } + + async fn get_block_transaction_indices( + &self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError> { + self.transaction.get_block_transaction_indices(block_height) + } + async fn get_block( &self, block_id: Id, diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 7850e90820..803a083af2 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -1651,6 +1651,92 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { Ok(()) } + /// The height through which the secondary history tables are complete; + /// `None` when no fast catch-up has deferred them. See the scanner-lib + /// backfill pass. + pub async fn secondary_indexed_height( + &mut self, + ) -> Result, ApiServerStorageError> { + let row = self + .tx + .query_opt( + "SELECT value FROM ml.misc_data WHERE name = 'secondary_indexed_height';", + &[], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + let Some(bytes) = row.map(|row| row.get::<_, Vec>(0)) else { + return Ok(None); + }; + let bytes: [u8; 8] = bytes.try_into().map_err(|_| { + ApiServerStorageError::DeserializationError( + "Invalid secondary indexed height stored in the database".to_owned(), + ) + })?; + Ok(Some(BlockHeight::new(u64::from_be_bytes(bytes)))) + } + + /// Records the height through which the secondary history tables are + /// complete. + pub async fn set_secondary_indexed_height( + &mut self, + block_height: BlockHeight, + ) -> Result<(), ApiServerStorageError> { + let bytes = u64::to_be_bytes(block_height.into_int()).to_vec(); + self.tx + .execute( + "INSERT INTO ml.misc_data (name, value) VALUES ('secondary_indexed_height', $1) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;", + &[&bytes], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) + } + + /// The `(transaction id, global index)` pairs of all transactions owned by + /// the main-chain block at `block_height`, ascending by index. + pub async fn get_block_transaction_indices( + &mut self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError> { + let height = Self::block_height_to_postgres_friendly(block_height); + let rows = self + .tx + .query( + "SELECT t.transaction_id, t.tx_global_index + FROM ml.transactions t + JOIN ml.blocks b ON t.owning_block_id = b.block_id + WHERE b.block_height = $1 + ORDER BY t.tx_global_index ASC;", + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + let mut result = Vec::with_capacity(rows.len()); + for row in rows { + let tx_id_bytes: Vec = row.get(0); + let index: i64 = row.get(1); + let tx_id = + Id::::decode_all(&mut tx_id_bytes.as_slice()).map_err(|_| { + ApiServerStorageError::DeserializationError( + "Invalid transaction id in transactions table".to_owned(), + ) + })?; + result.push(( + tx_id, + u64::try_from(index).map_err(|_| { + ApiServerStorageError::DeserializationError( + "Invalid transaction global index".to_owned(), + ) + })?, + )); + } + Ok(result) + } + pub async fn initialize_database( &mut self, chain_config: &ChainConfig, diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs index 9ba1c9ab1c..6e37da1572 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs @@ -15,7 +15,7 @@ use common::{ chain::{ - DelegationId, Destination, OrderId, PoolId, block::timestamp::BlockTimestamp, + DelegationId, Destination, OrderId, PoolId, Transaction, block::timestamp::BlockTimestamp, tokens::TokenId, }, primitives::{Amount, BlockHeight, CoinOrTokenId, Id}, @@ -143,6 +143,25 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn secondary_indexed_height(&self) -> Result, ApiServerStorageError> { + let mut conn = + QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR), &self.statements); + let res = conn.secondary_indexed_height().await?; + + Ok(res) + } + + async fn get_block_transaction_indices( + &self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError> { + let mut conn = + QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR), &self.statements); + let res = conn.get_block_transaction_indices(block_height).await?; + + Ok(res) + } + async fn get_block( &self, block_id: Id, diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs index b93742d790..9952916ae9 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs @@ -41,6 +41,15 @@ use super::{ApiServerPostgresTransactionalRw, CONN_ERR}; #[async_trait::async_trait] impl ApiServerStorageWrite for ApiServerPostgresTransactionalRw<'_> { + async fn set_secondary_indexed_height( + &mut self, + block_height: BlockHeight, + ) -> Result<(), ApiServerStorageError> { + let mut conn = + QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR), &self.statements); + conn.set_secondary_indexed_height(block_height).await + } + async fn reinitialize_storage( &mut self, chain_config: &ChainConfig, @@ -578,6 +587,25 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn secondary_indexed_height(&self) -> Result, ApiServerStorageError> { + let mut conn = + QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR), &self.statements); + let res = conn.secondary_indexed_height().await?; + + Ok(res) + } + + async fn get_block_transaction_indices( + &self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError> { + let mut conn = + QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR), &self.statements); + let res = conn.get_block_transaction_indices(block_height).await?; + + Ok(res) + } + async fn get_block( &self, block_id: Id, diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 2061cf235d..8ff92a3dcf 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -666,6 +666,21 @@ pub trait ApiServerStorageRead: Sync { async fn get_best_block(&self) -> Result; + /// The height through which the secondary history tables (address and + /// token transaction listings) are known to be complete. `None` means no + /// scan has deferred them, i.e. they are complete wherever they exist. + /// See `set_secondary_indexed_height`. + async fn secondary_indexed_height(&self) -> Result, ApiServerStorageError>; + + /// The transaction global indices of all transactions whose owning block + /// is the main-chain block at `block_height`, in ascending index order. + /// Used by the secondary-index backfill to stamp token transaction rows + /// with the same indices the live scan assigned. + async fn get_block_transaction_indices( + &self, + block_height: BlockHeight, + ) -> Result, u64)>, ApiServerStorageError>; + async fn get_latest_blocktimestamps( &self, ) -> Result, ApiServerStorageError>; @@ -949,6 +964,15 @@ pub trait ApiServerStorageWrite: ApiServerStorageRead { block_height: BlockHeight, ) -> Result<(), ApiServerStorageError>; + /// Records that the secondary history tables are complete through + /// `block_height` inclusive. Maintained by the scan (lagging while fast + /// catch-up defers history rows) and by the backfill pass; rolled back on + /// reorg disconnects together with the rows it describes. + async fn set_secondary_indexed_height( + &mut self, + block_height: BlockHeight, + ) -> Result<(), ApiServerStorageError>; + /// Stores the balance under the string encoding carried by the given address: /// scanning constructs addresses via `Address::new`, so the stored encoding is /// the canonical one, which the byte-ordered tie-break comparison of the diff --git a/api-server/scanner-daemon/src/lib.rs b/api-server/scanner-daemon/src/lib.rs index cbd56303f5..49a2e9045b 100644 --- a/api-server/scanner-daemon/src/lib.rs +++ b/api-server/scanner-daemon/src/lib.rs @@ -152,6 +152,12 @@ pub async fn run( BlockchainState::new(Arc::clone(chain_config), storage) } }; + // Seed the history-table watermark for databases written by builds that + // predate it; a no-op on fresh databases (see the method docs). + local_block + .seed_secondary_watermark_for_upgrade() + .await + .unwrap_or_else(|e| panic!("Seeding the secondary history watermark failed: {e}")); supervise_sync( chain_config, @@ -323,10 +329,9 @@ async fn supervise_sync( // only reset after a successful sync, see below. // (experimental step 10) Fast catch-up decision: while the scanner is // more than CATCH_UP_THRESHOLD blocks behind the node, run in - // CatchingUp mode (secondary indexes deferred); within the threshold, - // run Full and complete any deferred backfill. The threshold is - // deliberately simple; the backfill pass is a follow-up tracked in the - // experimental notes. + // CatchingUp mode (secondary history rows deferred); within the + // threshold, run Full. The deferred history rows are rebuilt by the + // backfill pass that runs alongside Full-mode syncing, below. const CATCH_UP_THRESHOLD: u64 = 2_000; // Note: the RemoteNode wrapper already maps connection errors; reuse // the sync error handling by folding the decision into the sync cycle @@ -369,19 +374,17 @@ async fn supervise_sync( }; if local_block.scan_mode() != mode { // Blocks scanned in CatchingUp mode skip the secondary history - // and statistics tables, and the backfill pass that rebuilds - // them is not implemented yet, so both transitions are - // surfaced loudly instead of failing silently. + // tables; the backfill pass rebuilds those rows afterwards. if mode == api_blockchain_scanner_lib::blockchain_state::ScanMode::CatchingUp { - logging::log::warn!( - "(experimental) scanner entering fast catch-up mode: address/token \ - history and statistics will not be indexed for blocks scanned in this mode" + logging::log::info!( + "scanner entering fast catch-up mode: address/token history rows \ + are deferred for blocks scanned in this mode and rebuilt by the \ + backfill pass afterwards" ); } else { - logging::log::warn!( - "(experimental) scanner leaving fast catch-up mode: any range indexed \ - while catching up lacks history/statistics rows until a backfill pass \ - (not yet implemented) is provided" + logging::log::info!( + "scanner leaving fast catch-up mode: deferred history rows are \ + being rebuilt by the backfill pass" ); } local_block.set_scan_mode(mode); @@ -394,6 +397,21 @@ async fn supervise_sync( // Note: the backoffs and the outage diagnostics are only reset here (after a // successful sync), not after a successful connect: a flapping node must not // pin the retry rate at the initial delay or reset the outage counters. + // Post-catch-up backfill: while in Full mode, rebuild the + // deferred secondary history tables for a chunk of the lagging + // range each cycle; live indexing keeps priority, so the pass + // is skipped while still catching up. The watermark makes + // this a no-op once the history tables are complete. + if local_block.scan_mode() + == api_blockchain_scanner_lib::blockchain_state::ScanMode::Full + { + const SECONDARY_BACKFILL_CHUNK: usize = 1_000; + if let Err(err) = + local_block.backfill_secondary_indexes(SECONDARY_BACKFILL_CHUNK).await + { + logging::log::warn!("Secondary history backfill failed: {err}"); + } + } backoff.reset(); sync_error_backoff.reset(); outage = Outage::new(); diff --git a/api-server/scanner-lib/src/blockchain_state/mod.rs b/api-server/scanner-lib/src/blockchain_state/mod.rs index c11d19a416..b22634d771 100644 --- a/api-server/scanner-lib/src/blockchain_state/mod.rs +++ b/api-server/scanner-lib/src/blockchain_state/mod.rs @@ -82,10 +82,10 @@ pub enum BlockchainStateError { pub enum ScanMode { /// Full indexing: everything is written as blocks are scanned. Full, - /// Fast catch-up: secondary indexes (address/token transaction history and - /// statistics) are deferred; core data (balances, utxos, blocks, - /// transactions, stream events) is still indexed. A backfill pass after - /// catching up rebuilds the deferred tables. + /// Fast catch-up: the secondary history tables (address and token + /// transaction listings) are deferred; core data (balances, utxos, + /// statistics, blocks, transactions, stream events) is still indexed. A + /// backfill pass after catching up rebuilds the deferred history rows. CatchingUp, } @@ -124,6 +124,145 @@ impl BlockchainState { &self.storage } + /// Mutable access to the storage, for callers that need to run + /// transactions outside the scan paths (used by tests). + #[cfg(test)] + pub fn storage_mut(&mut self) -> &mut S { + &mut self.storage + } + + /// Consumes the state, returning the storage. + #[cfg(test)] + pub fn consume_storage(self) -> S { + self.storage + } + + /// Rebuild the deferred secondary history tables (address and token + /// transaction listings) for up to `max_blocks` blocks above the recorded + /// watermark, from the locally stored blocks. Returns the new watermark + /// height, or `None` when there is nothing to backfill. + /// + /// This is the post-catch-up backfill pass: blocks scanned while the + /// scanner was more than `CATCH_UP_THRESHOLD` blocks behind skipped the + /// history rows; this pass rebuilds exactly those rows. It is read-only + /// with respect to every core table (balances, utxos, orders, pools, + /// statistics): it cannot double-count because it never applies a balance + /// or utxo mutation, only per-height history inserts. + pub async fn backfill_secondary_indexes( + &mut self, + max_blocks: usize, + ) -> Result, BlockchainStateError> { + let mut db_tx = self.storage.transaction_rw().await?; + + let best_height = db_tx.get_best_block().await?.block_height(); + let Some(watermark) = db_tx.secondary_indexed_height().await? else { + // History was never deferred; the tables are complete. + return Ok(None); + }; + if watermark >= best_height { + return Ok(None); + } + + let remaining = best_height.into_int() - watermark.into_int(); + let chunk = std::cmp::min(remaining, u64::try_from(max_blocks).unwrap_or(remaining)); + let requested_end_height = BlockHeight::new(watermark.into_int() + chunk); + + // The watermark only ever advances to the last height that was fully + // processed: an inconsistent local database (a missing block or + // transaction row) fails loudly instead of silently marking the + // skipped range complete. + let mut processed_end_height = watermark; + for height_int in (watermark.into_int() + 1)..=(requested_end_height.into_int()) { + let height = BlockHeight::new(height_int); + let Some(block_id) = db_tx.get_main_chain_block_id(height).await? else { + return Err(BlockchainStateError::StorageError( + ApiServerStorageError::LowLevelStorageError(format!( + "Backfill: no main-chain block id at height {height}" + )), + )); + }; + let Some(block_info) = db_tx.get_block(block_id).await? else { + return Err(BlockchainStateError::StorageError( + ApiServerStorageError::LowLevelStorageError(format!( + "Backfill: no block row for {block_id} at height {height}" + )), + )); + }; + let indices: BTreeMap, u64> = + db_tx.get_block_transaction_indices(height).await?.into_iter().collect(); + + for tx in block_info.block.block.transactions() { + let (address_transactions, transaction_tokens) = + collect_secondary_history(&self.chain_config, &db_tx, tx, height).await?; + for (address, transactions) in address_transactions { + db_tx + .set_address_transactions_at_height(address.as_str(), transactions, height) + .await + .map_err(|_| { + ApiServerStorageError::LowLevelStorageError( + "Unable to set address transactions".to_owned(), + ) + })?; + } + let tx_id = tx.transaction().get_id(); + for token_id in transaction_tokens { + let tx_global_index = indices.get(&tx_id).copied().ok_or_else(|| { + ApiServerStorageError::LowLevelStorageError(format!( + "Backfill: no transaction global index for {tx_id} at height {height}" + )) + })?; + db_tx + .set_token_transaction_at_height(token_id, tx_id, height, tx_global_index) + .await + .map_err(|_| { + ApiServerStorageError::LowLevelStorageError( + "Unable to set token transactions".to_owned(), + ) + })?; + } + } + processed_end_height = height; + } + + db_tx.set_secondary_indexed_height(processed_end_height).await?; + db_tx.commit().await?; + if processed_end_height < best_height { + logging::log::info!( + "Secondary index backfill progressed to height {processed_end_height} (best {best_height})" + ); + } else { + logging::log::info!( + "Secondary index backfill complete: history tables rebuilt through height {processed_end_height}" + ); + } + + Ok(Some(processed_end_height)) + } + + /// One-time startup seeding of the history-table watermark for databases + /// written by builds predating the watermark. A `None` watermark with a + /// non-genesis tip means history rows may be missing from an earlier + /// fast catch-up under those builds; seeding to genesis forces the + /// backfill to rebuild the full history (idempotent, correct in all + /// cases). Fresh databases keep `None`; the first Full batch records it. + pub async fn seed_secondary_watermark_for_upgrade( + &mut self, + ) -> Result<(), BlockchainStateError> { + let mut db_tx = self.storage.transaction_rw().await?; + if db_tx.secondary_indexed_height().await?.is_none() { + let best_height = db_tx.get_best_block().await?.block_height(); + if best_height > BlockHeight::zero() { + logging::log::info!( + "Seeding the secondary history watermark at genesis: an older build may \ + have deferred history rows during fast catch-up" + ); + db_tx.set_secondary_indexed_height(BlockHeight::zero()).await?; + } + } + db_tx.commit().await?; + Ok(()) + } + pub async fn scan_genesis(&mut self, genesis: &Genesis) -> Result<(), BlockchainStateError> { let mut db_tx = self.storage.transaction_rw().await?; @@ -268,7 +407,8 @@ impl LocalBlockchainState for BlockchainState } // Connect the new blocks in the new chain - count_blocks(blocks.len()); + let batch_len = blocks.len(); + count_blocks(batch_len); for (index, block) in blocks.into_iter().map(WithId::new).enumerate() { let block_height = BlockHeight::new(common_block_height.into_int() + index as u64 + 1); let block_timestamp = block.timestamp(); @@ -310,11 +450,17 @@ impl LocalBlockchainState for BlockchainState let block_trusted_fees = trusted_fees.get(index).and_then(|f| f.clone()); // (experimental step 10) In fast catch-up mode, address/token - // transaction history and statistics tables are deferred to the - // post-catch-up backfill pass. Balances, utxos, blocks, - // transactions and stream events keep indexing so the core API - // stays complete. + // transaction history tables are deferred to the post-catch-up + // backfill pass. Balances, utxos, blocks, transactions and stream + // events keep indexing so the core API stays complete. let defer_secondary = self.scan_mode == ScanMode::CatchingUp; + if defer_secondary && db_tx.secondary_indexed_height().await?.is_none() { + // First deferred batch: the history tables are complete up to + // and including the common ancestor height. If a row is + // already present (a previous catch-up round), keep the + // smaller, lagging value. + db_tx.set_secondary_indexed_height(common_block_height).await?; + } count_txs(block.transactions().len()); for (idx, tx) in block.transactions().iter().enumerate() { @@ -436,6 +582,24 @@ impl LocalBlockchainState for BlockchainState db_tx.notify_new_stream_events(last_event_id).await?; } + // The history tables are complete through the tip of this batch + // unless it was scanned in fast catch-up mode, where the deferred + // range is left to the backfill pass (the watermark then lags). + // A full batch only extends completeness if the range below it is + // already complete; if a deferred gap sits below the batch, the + // watermark must stay put so the backfill still sees the gap. + if self.scan_mode != ScanMode::CatchingUp { + let complete_below_batch = db_tx + .secondary_indexed_height() + .await? + .is_none_or(|watermark| watermark >= common_block_height); + if complete_below_batch { + let tip_height = + BlockHeight::new(common_block_height.into_int() + batch_len as u64); + db_tx.set_secondary_indexed_height(tip_height).await?; + } + } + { let _t = TimingGuard::stage(Stage::Commit); db_tx.commit().await?; @@ -839,6 +1003,17 @@ async fn disconnect_tables_above_height( db_tx.del_orders_above_height(block_height).await?; + // The history-table watermark describes rows that are being disconnected + // above this height, so it must move down with them. A `None` watermark + // (nothing ever deferred) stays untouched. + if let Some(_watermark) = db_tx + .secondary_indexed_height() + .await? + .filter(|watermark| *watermark > block_height) + { + db_tx.set_secondary_indexed_height(block_height).await?; + } + Ok(()) } @@ -1154,6 +1329,222 @@ async fn calculate_tx_fee_and_collect_token_info( Ok((fee, tx_aditional_infos)) } +/// Read-only re-derivation of the secondary history rows (address -> +/// transaction, token -> transaction) for one transaction, mirroring what +/// the live scan collects inside `update_tables_from_transaction_inputs` +/// and `update_tables_from_transaction_outputs`, arm by arm. Used by the +/// post-catch-up backfill; it must not mutate anything, and its +/// equivalence with the live path is what the backfill tests assert. +async fn collect_secondary_history( + chain_config: &ChainConfig, + db_tx: &T, + tx: &SignedTransaction, + block_height: BlockHeight, +) -> Result< + ( + BTreeMap, BTreeSet>>, + BTreeSet, + ), + ApiServerStorageError, +> { + let tx_id = tx.transaction().get_id(); + let mut address_transactions: BTreeMap, BTreeSet>> = + BTreeMap::new(); + let mut transaction_tokens: BTreeSet = BTreeSet::new(); + let insert_address = |destination: &Destination, + address_transactions: &mut BTreeMap< + Address, + BTreeSet>, + >| { + let address = Address::::new(chain_config, destination.clone()) + .expect("Unable to encode destination"); + address_transactions.entry(address).or_default().insert(tx_id); + }; + + // --- outputs (mirrors update_tables_from_transaction_outputs) --- + for output in tx.transaction().outputs() { + match output { + TxOutput::Burn(value) => { + if let OutputValue::TokenV1(token_id, _) = value { + transaction_tokens.insert(*token_id); + } + } + TxOutput::DataDeposit(_) + | TxOutput::CreateDelegationId(_, _) + | TxOutput::ProduceBlockFromStake(_, _) => {} + TxOutput::IssueFungibleToken(_) => { + let token_id = + make_token_id(chain_config, block_height, tx.transaction().inputs())?; + transaction_tokens.insert(token_id); + } + TxOutput::IssueNft(token_id, _, destination) => { + insert_address(destination, &mut address_transactions); + transaction_tokens.insert(*token_id); + } + TxOutput::CreateStakePool(_, stake_pool_data) => { + insert_address( + stake_pool_data.decommission_key(), + &mut address_transactions, + ); + insert_address(stake_pool_data.staker(), &mut address_transactions); + } + TxOutput::DelegateStaking(_, delegation_id) => { + let delegation = db_tx + .get_delegation_at_or_below_height(*delegation_id, block_height) + .await? + .expect("Delegation should exist"); + insert_address(delegation.spend_destination(), &mut address_transactions); + } + TxOutput::Transfer(output_value, destination) + | TxOutput::LockThenTransfer(output_value, destination, _) => { + insert_address(destination, &mut address_transactions); + if let OutputValue::TokenV1(token_id, _) = output_value { + transaction_tokens.insert(*token_id); + } + } + TxOutput::Htlc(output_value, htlc) => { + insert_address(&htlc.spend_key, &mut address_transactions); + insert_address(&htlc.refund_key, &mut address_transactions); + if let OutputValue::TokenV1(token_id, _) = output_value { + transaction_tokens.insert(*token_id); + } + } + TxOutput::CreateOrder(order_data) => { + let currencies = [order_data.ask(), order_data.give()]; + for currency in currencies { + if let OutputValue::TokenV1(token_id, _) = currency { + transaction_tokens.insert(*token_id); + } + } + } + } + } + + // --- inputs (mirrors update_tables_from_transaction_inputs) --- + let sigs = tx.signatures(); + for (input, sig) in tx.transaction().inputs().iter().zip(sigs) { + match input { + TxInput::AccountCommand(_, cmd) => match cmd { + AccountCommand::MintTokens(token_id, _) + | AccountCommand::UnmintTokens(token_id) + | AccountCommand::FreezeToken(token_id, _) + | AccountCommand::UnfreezeToken(token_id) + | AccountCommand::LockTokenSupply(token_id) + | AccountCommand::ChangeTokenAuthority(token_id, _) + | AccountCommand::ChangeTokenMetadataUri(token_id, _) => { + transaction_tokens.insert(*token_id); + } + AccountCommand::FillOrder(order_id, _, _) => { + if let Some(order) = db_tx.get_order(*order_id).await? { + collect_order_tokens(&order, &mut transaction_tokens); + } + } + AccountCommand::ConcludeOrder(order_id) => { + if let Some(order) = db_tx.get_order(*order_id).await? { + collect_order_tokens(&order, &mut transaction_tokens); + } + } + }, + TxInput::OrderAccountCommand(cmd) => match cmd { + OrderAccountCommand::FillOrder(order_id, _) + | OrderAccountCommand::ConcludeOrder(order_id) + | OrderAccountCommand::FreezeOrder(order_id) => { + if let Some(order) = db_tx.get_order(*order_id).await? { + collect_order_tokens(&order, &mut transaction_tokens); + } + } + }, + TxInput::Account(_) => {} + TxInput::Utxo(outpoint) => { + let utxo = db_tx.get_utxo(outpoint.clone()).await?.expect("must be present"); + // The live scan treats most output kinds of a spent input as + // history no-ops when the input is block-reward-sourced; only + // the pool-decommission pool ids record an address there. + // Mirror that per-source split. + let block_reward_sourced = + matches!(outpoint.source_id(), OutPointSourceId::BlockReward(_)); + match utxo.into_output() { + TxOutput::Burn(_) + | TxOutput::CreateDelegationId(_, _) + | TxOutput::DelegateStaking(_, _) + | TxOutput::DataDeposit(_) + | TxOutput::IssueFungibleToken(_) + | TxOutput::CreateOrder(_) => {} + TxOutput::CreateStakePool(pool_id, _) + | TxOutput::ProduceBlockFromStake(_, pool_id) => { + // Only the block-reward-sourced variant records the + // decommission destination in the history tables; the + // pool data is read at or below this height; the row + // at this height is the post-decommission record, + // whose decommission destination is preserved. + if matches!(outpoint.source_id(), OutPointSourceId::BlockReward(_)) { + let pool_data = db_tx + .get_pool_data_at_or_below_height(pool_id, block_height) + .await? + .expect("pool should exist"); + insert_address( + pool_data.decommission_destination(), + &mut address_transactions, + ); + } + } + TxOutput::IssueNft(token_id, _, destination) => { + if !block_reward_sourced { + insert_address(&destination, &mut address_transactions); + transaction_tokens.insert(token_id); + } + } + TxOutput::Htlc(output_value, htlc) => { + // The live scan treats block-reward-sourced HTLC + // inputs as history no-ops (no witness decode). + if !block_reward_sourced { + let destination = if let InputWitness::Standard(sig) = sig { + let htlc_sig = AuthorizedHashedTimelockContractSpend::decode_all( + &mut sig.raw_signature(), + ) + .expect("proper signature"); + match htlc_sig { + AuthorizedHashedTimelockContractSpend::Spend(_, _) => { + htlc.spend_key.clone() + } + AuthorizedHashedTimelockContractSpend::Refund(_) => { + htlc.refund_key.clone() + } + } + } else { + panic!("Empty signature for htlc") + }; + insert_address(&destination, &mut address_transactions); + if let OutputValue::TokenV1(token_id, _) = &output_value { + transaction_tokens.insert(*token_id); + } + } + } + TxOutput::LockThenTransfer(output_value, destination, _) + | TxOutput::Transfer(output_value, destination) => { + if !block_reward_sourced { + insert_address(&destination, &mut address_transactions); + if let OutputValue::TokenV1(token_id, _) = &output_value { + transaction_tokens.insert(*token_id); + } + } + } + } + } + } + } + + Ok((address_transactions, transaction_tokens)) +} + +fn collect_order_tokens(order: &Order, transaction_tokens: &mut BTreeSet) { + for currency in [&order.ask_currency, &order.give_currency] { + if let CoinOrTokenId::TokenId(token_id) = currency { + transaction_tokens.insert(*token_id); + } + } +} + /// (experimental) step 4: UTXO reads go through the batch-level prefetch /// cache first; a cache miss (e.g. a UTXO created by a tx earlier in the same /// batch, which the prefetch couldn't see) falls back to a direct read, which @@ -2696,6 +3087,374 @@ fn get_tx_output_destination(txo: &TxOutput) -> Option<&Destination> { } } +#[cfg(test)] +mod backfill_tests { + use super::*; + + use api_server_common::storage::{ + impls::in_memory::transactional::TransactionalApiServerInMemoryStorage, + storage_api::{ApiServerStorageRead, Transactional}, + }; + use chainstate::BlockSource; + use chainstate_test_framework::{TestFramework, TransactionBuilder}; + use common::{ + chain::{ + CoinUnit, Destination, OutPointSourceId, TxInput, TxOutput, stakelock::StakePoolData, + }, + primitives::{H256, per_thousand::PerThousand}, + }; + use crypto::{ + key::{KeyKind, PrivateKey}, + vrf::{VRFKeyKind, VRFPrivateKey}, + }; + use rstest::rstest; + use test_utils::random::{Seed, make_seedable_rng}; + + async fn initialized_state( + chain_config: &Arc, + ) -> BlockchainState { + let storage = TransactionalApiServerInMemoryStorage::new(chain_config); + let mut state = BlockchainState::new(Arc::clone(chain_config), storage); + state.scan_genesis(chain_config.genesis_block().as_ref()).await.unwrap(); + state + } + + async fn scan_block( + local: &mut BlockchainState, + block: Block, + ) { + let common_height = local + .storage() + .transaction_ro() + .await + .unwrap() + .get_best_block() + .await + .unwrap() + .block_height(); + local.scan_blocks(common_height, vec![block]).await.unwrap(); + } + + /// The equivalence test: a chain scanned entirely in Full mode must end up + /// with exactly the same history rows as the same chain scanned mostly in + /// CatchingUp mode followed by the backfill pass. Balances and UTXO rows + /// must be untouched by the backfill (the double-count hazard guard). + #[rstest] + #[trace] + #[case(test_utils::random::Seed::from_entropy())] + #[tokio::test] + async fn backfill_matches_full_scan(#[case] seed: Seed) { + let mut rng = make_seedable_rng(seed); + + let initial_pledge = 40_000 * CoinUnit::ATOMS_PER_COIN; + let (staking_sk, pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr); + let (vrf_sk, _vrf_pk) = VRFPrivateKey::new_from_rng(&mut rng, VRFKeyKind::Schnorrkel); + let staking_key = Destination::PublicKey(pk.clone()); + let pool_data = StakePoolData::new( + Amount::from_atoms(initial_pledge), + staking_key.clone(), + vrf_sk.to_public_key(), + staking_key.clone(), + PerThousand::new_from_rng(&mut rng), + Amount::from_atoms(0), + ); + let pool_id = PoolId::new(H256::random_using(&mut rng)); + + let chain_config = chainstate_test_framework::create_chain_config_with_staking_pool( + &mut rng, + Amount::from_atoms(initial_pledge * 2), + pool_id, + pool_data, + ) + .build(); + let mut tf = + TestFramework::builder(&mut rng).with_chain_config(chain_config.clone()).build(); + let chain_config = Arc::clone(tf.chainstate.get_chain_config()); + + // Build three blocks; the first two will be scanned in CatchingUp + // mode, the third in Full mode. + let block_spacing = chain_config.target_block_spacing(); + let mut prev = chain_config.genesis_block_id(); + let mut blocks = Vec::new(); + // A chain of spends: each block's transaction spends the single + // output of the previous block's transaction. + let mut spend_source = OutPointSourceId::BlockReward(chain_config.genesis_block_id()); + for _block_index in 0..3 { + tf.progress_time_seconds_since_epoch(block_spacing.as_secs()); + let spend_tx = TransactionBuilder::new() + .add_input( + TxInput::from_utxo(spend_source.clone(), 0), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::Transfer( + OutputValue::Coin(Amount::from_atoms(100)), + Destination::AnyoneCanSpend, + )) + .build(); + spend_source = OutPointSourceId::Transaction(spend_tx.transaction().get_id()); + let txs = vec![spend_tx]; + let block = tf + .make_pos_block_builder() + .with_parent(prev) + .with_stake_spending_key(staking_sk.clone()) + .with_vrf_key(vrf_sk.clone()) + .with_stake_pool_id(pool_id) + .with_transactions(txs) + .build(&mut rng); + prev = block.get_id().into(); + // The block builder derives the kernel from the parent's + // chainstate index, so each parent must be processed before the + // next block is built. + tf.process_block(block.clone(), BlockSource::Local).unwrap(); + blocks.push(block); + } + + // Reference: full scan. + let mut full = initialized_state(&chain_config).await; + for block in blocks.clone() { + scan_block(&mut full, block).await; + } + + // Fast catch-up: first two blocks deferred, third in Full, then + // backfill in single-block chunks with a fresh state wrapper each + // time, simulating restarts mid-backfill. + let mut catching_up = initialized_state(&chain_config).await; + catching_up.set_scan_mode(ScanMode::CatchingUp); + for block in blocks.iter().take(2).cloned().collect::>() { + scan_block(&mut catching_up, block).await; + } + // While deferred, the history rows are missing. + let anyone = Address::::new(&chain_config, Destination::AnyoneCanSpend) + .expect("encoding"); + { + let tx = catching_up.storage().transaction_ro().await.unwrap(); + let history = tx.get_address_transactions(anyone.as_str()).await.unwrap(); + assert!( + history.is_empty(), + "history rows must be deferred in catch-up mode" + ); + // The watermark pins the common ancestor where deferral began + // (genesis here), not the last deferred block. + assert_eq!( + tx.secondary_indexed_height().await.unwrap(), + Some(BlockHeight::new(0)) + ); + } + catching_up.set_scan_mode(ScanMode::Full); + scan_block(&mut catching_up, blocks[2].clone()).await; + + // Simulate restarts mid-backfill: each chunk runs through a fresh + // state wrapper over the same storage. + loop { + let storage = catching_up.consume_storage(); + let mut state = BlockchainState::new(Arc::clone(&chain_config), storage); + let progressed = state.backfill_secondary_indexes(1).await.unwrap(); + catching_up = state; + if progressed.is_none() { + break; + } + } + + // Equivalence: identical history rows. + let (full_history, catchup_history) = { + let full_tx = full.storage().transaction_ro().await.unwrap(); + let cu_tx = catching_up.storage().transaction_ro().await.unwrap(); + ( + full_tx.get_address_transactions(anyone.as_str()).await.unwrap(), + cu_tx.get_address_transactions(anyone.as_str()).await.unwrap(), + ) + }; + assert_eq!(full_history, catchup_history); + assert_eq!(full_history.len(), 3, "one history row per transaction"); + + // The watermark reached the tip. + { + let tx = catching_up.storage().transaction_ro().await.unwrap(); + assert_eq!( + tx.secondary_indexed_height().await.unwrap(), + Some(BlockHeight::new(3)) + ); + } + + // Hazard guard: balances identical between the two scans, and the + // utxo row of the last transaction identical (proving the backfill + // left the core tables untouched). + let (full_balances, catchup_balances, full_utxo, catchup_utxo) = { + let full_tx = full.storage().transaction_ro().await.unwrap(); + let cu_tx = catching_up.storage().transaction_ro().await.unwrap(); + let last_tx_outpoint = UtxoOutPoint::new( + OutPointSourceId::Transaction(blocks[2].transactions()[0].transaction().get_id()), + 0, + ); + ( + full_tx.get_address_balance(anyone.as_str(), CoinOrTokenId::Coin).await.unwrap(), + cu_tx.get_address_balance(anyone.as_str(), CoinOrTokenId::Coin).await.unwrap(), + full_tx.get_utxo(last_tx_outpoint.clone()).await.unwrap(), + cu_tx.get_utxo(last_tx_outpoint).await.unwrap(), + ) + }; + assert_eq!(full_balances, catchup_balances); + assert_eq!(full_utxo, catchup_utxo); + + let (full_tip, catchup_tip) = { + let full_tx = full.storage().transaction_ro().await.unwrap(); + let cu_tx = catching_up.storage().transaction_ro().await.unwrap(); + ( + full_tx.get_best_block().await.unwrap(), + cu_tx.get_best_block().await.unwrap(), + ) + }; + assert_eq!(full_tip.block_height(), catchup_tip.block_height()); + } + + /// The disconnect path must clamp the watermark together with the rows it + /// describes; a `None` watermark stays untouched. + #[tokio::test] + async fn reorg_clamps_watermark() { + let chain_config = Arc::new(common::chain::config::create_mainnet()); + let mut state = initialized_state(&chain_config).await; + { + let mut db_tx = state.storage_mut().transaction_rw().await.unwrap(); + db_tx.set_secondary_indexed_height(BlockHeight::new(10)).await.unwrap(); + db_tx.commit().await.unwrap(); + } + let mut db_tx = state.storage_mut().transaction_rw().await.unwrap(); + disconnect_tables_above_height(&mut db_tx, BlockHeight::new(5)).await.unwrap(); + assert_eq!( + db_tx.secondary_indexed_height().await.unwrap(), + Some(BlockHeight::new(5)) + ); + db_tx.commit().await.unwrap(); + + // A `None` watermark (nothing ever deferred) survives a disconnect. + let mut state = initialized_state(&chain_config).await; + let mut db_tx = state.storage_mut().transaction_rw().await.unwrap(); + disconnect_tables_above_height(&mut db_tx, BlockHeight::new(5)).await.unwrap(); + assert_eq!(db_tx.secondary_indexed_height().await.unwrap(), None); + db_tx.commit().await.unwrap(); + } + + /// The upgrade seed only fires for databases that predate the watermark + /// and have a non-genesis tip; fresh databases keep `None`, and a second + /// call is a no-op. + #[tokio::test] + async fn upgrade_seed_targets_only_pre_watermark_databases() { + let chain_config = Arc::new(common::chain::config::create_mainnet()); + + // Fresh database: the seed is a no-op. + let mut fresh = initialized_state(&chain_config).await; + fresh.seed_secondary_watermark_for_upgrade().await.unwrap(); + assert_eq!( + fresh + .storage() + .transaction_ro() + .await + .unwrap() + .secondary_indexed_height() + .await + .unwrap(), + None + ); + + // A database with a tip above genesis and no watermark row (the + // pre-watermark-build population) is seeded at genesis. + let mut rng = make_seedable_rng(test_utils::random::Seed::from_entropy()); + let mut tf = TestFramework::builder(&mut rng).build(); + let tf_config = Arc::clone(tf.chainstate.get_chain_config()); + let tf_block = tf + .make_block_builder() + .with_parent(tf_config.genesis_block_id()) + .build(&mut rng); + let mut pre_watermark = initialized_state(&tf_config).await; + { + let mut db_tx = pre_watermark.storage_mut().transaction_rw().await.unwrap(); + db_tx + .set_mainchain_block( + tf_block.get_id(), + BlockHeight::new(1), + &BlockWithExtraData { + block: tf_block.clone(), + tx_additional_infos: Vec::new(), + }, + ) + .await + .unwrap(); + db_tx.commit().await.unwrap(); + } + pre_watermark.seed_secondary_watermark_for_upgrade().await.unwrap(); + assert_eq!( + pre_watermark + .storage() + .transaction_ro() + .await + .unwrap() + .secondary_indexed_height() + .await + .unwrap(), + Some(BlockHeight::new(0)) + ); + // Idempotent on re-run. + pre_watermark.seed_secondary_watermark_for_upgrade().await.unwrap(); + assert_eq!( + pre_watermark + .storage() + .transaction_ro() + .await + .unwrap() + .secondary_indexed_height() + .await + .unwrap(), + Some(BlockHeight::new(0)) + ); + } + + /// A missing block row in the backfill range fails loudly and leaves the + /// watermark (and the history tables) unchanged. + #[tokio::test] + async fn backfill_fails_loudly_on_missing_block() { + let mut rng = make_seedable_rng(test_utils::random::Seed::from_entropy()); + let mut tf = TestFramework::builder(&mut rng).build(); + let tf_config = Arc::clone(tf.chainstate.get_chain_config()); + let tf_block = tf + .make_block_builder() + .with_parent(tf_config.genesis_block_id()) + .build(&mut rng); + let chain_config = tf_config; + let mut state = initialized_state(&chain_config).await; + { + let mut db_tx = state.storage_mut().transaction_rw().await.unwrap(); + db_tx.set_secondary_indexed_height(BlockHeight::new(0)).await.unwrap(); + db_tx + .set_mainchain_block( + tf_block.get_id(), + BlockHeight::new(10), + &BlockWithExtraData { + block: tf_block.clone(), + tx_additional_infos: Vec::new(), + }, + ) + .await + .unwrap(); + db_tx.commit().await.unwrap(); + } + // The tip is at height 10 but height 1 has no main-chain block id: + // the pass must error, not advance the watermark. + let err = state.backfill_secondary_indexes(100).await.unwrap_err(); + assert!(err.to_string().contains("no main-chain block id")); + assert_eq!( + state + .storage() + .transaction_ro() + .await + .unwrap() + .secondary_indexed_height() + .await + .unwrap(), + Some(BlockHeight::new(0)) + ); + } +} + #[cfg(test)] mod replay_regression_tests { use super::*;