From ae05c22b2f2a128ea50c19ee5996d49cdba3c34a Mon Sep 17 00:00:00 2001 From: junderw Date: Sat, 14 Feb 2026 13:56:43 +0900 Subject: [PATCH] WIP --- src/chain.rs | 12 + src/daemon.rs | 4 +- src/new_index/mempool.rs | 105 +++++- src/new_index/mod.rs | 1 + src/new_index/query.rs | 47 ++- src/rest.rs | 12 +- src/util/fee_estimation.rs | 376 ++++++++++++++++++++ src/util/fees.rs | 19 +- src/util/gbt.rs | 702 +++++++++++++++++++++++++++++++++++++ src/util/mod.rs | 2 + 10 files changed, 1250 insertions(+), 30 deletions(-) create mode 100644 src/util/fee_estimation.rs create mode 100644 src/util/gbt.rs diff --git a/src/chain.rs b/src/chain.rs index 510a5d6..8635e4b 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -130,6 +130,18 @@ impl Network { } } + #[cfg(not(feature = "liquid"))] + #[inline(always)] + pub const fn is_liquid(self) -> bool { + false + } + + #[cfg(feature = "liquid")] + #[inline(always)] + pub const fn is_liquid(self) -> bool { + true + } + #[cfg(feature = "liquid")] pub fn address_params(self) -> &'static address::AddressParams { // Liquid regtest uses elements's address params diff --git a/src/daemon.rs b/src/daemon.rs index 9eff11a..123592d 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -110,6 +110,8 @@ pub struct BlockchainInfo { #[derive(Serialize, Deserialize, Debug)] pub struct MempoolInfo { pub loaded: bool, + #[serde(default)] + pub mempoolminfee: f64, // in BTC/kB } #[derive(Serialize, Deserialize, Debug)] @@ -531,7 +533,7 @@ impl Daemon { from_value(info).chain_err(|| "invalid blockchain info") } - fn getmempoolinfo(&self) -> Result { + pub fn getmempoolinfo(&self) -> Result { let info: Value = self.request("getmempoolinfo", json!([]))?; from_value(info).chain_err(|| "invalid mempool info") } diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 5992481..e8292df 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -21,12 +21,19 @@ use crate::new_index::{ compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats, SpendingInfo, SpendingInput, TxHistoryInfo, Utxo, }; +use crate::util::fee_estimation::{FeeEstimator, RecommendedFees}; use crate::util::fees::{make_fee_histogram, TxFeeInfo}; +use crate::util::gbt::{ + build_projected_blocks, GbtTransaction, MempoolBlock, DEFAULT_BLOCK_WEIGHT, +}; use crate::util::{extract_tx_prevouts, full_hash, has_prevout, is_spendable, Bytes}; #[cfg(feature = "liquid")] use crate::elements::asset; +/// Maximum number of projected blocks to build for fee estimation +pub const MAX_PROJECTED_BLOCKS: usize = 8; + pub struct Mempool { chain: Arc, config: Arc, @@ -36,6 +43,9 @@ pub struct Mempool { edges: HashMap, // OutPoint -> (spending_txid, spending_vin) recent: BoundedVecDeque, // The N most recent txs to enter the mempool backlog_stats: (BacklogStats, Instant), + projected_blocks: (Vec, Instant), // Cached projected blocks + recommended_fees: (RecommendedFees, Instant), // Cached recommended fees + mempool_min_fee: f64, // Cached mempoolminfee in BTC/kB // monitoring latency: HistogramVec, // mempool requests latency @@ -61,6 +71,7 @@ pub struct TxOverview { impl Mempool { pub fn new(chain: Arc, metrics: &Metrics, config: Arc) -> Self { + let ttl = config.mempool_backlog_stats_ttl; Mempool { chain, txstore: BTreeMap::new(), @@ -70,8 +81,14 @@ impl Mempool { recent: BoundedVecDeque::new(config.mempool_recent_txs_size), backlog_stats: ( BacklogStats::default(), - Instant::now() - Duration::from_secs(config.mempool_backlog_stats_ttl), + Instant::now() - Duration::from_secs(ttl), ), + projected_blocks: (Vec::new(), Instant::now() - Duration::from_secs(ttl)), + recommended_fees: ( + RecommendedFees::default(), + Instant::now() - Duration::from_secs(ttl), + ), + mempool_min_fee: 1.0, // Default: 1 sat/vB latency: metrics.histogram_vec( HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"), &["part"], @@ -386,6 +403,16 @@ impl Mempool { &self.backlog_stats.0 } + /// Get the projected mempool blocks + pub fn projected_blocks(&self) -> &[MempoolBlock] { + &self.projected_blocks.0 + } + + /// Get the recommended fees based on projected blocks + pub fn recommended_fees(&self) -> &RecommendedFees { + &self.recommended_fees.0 + } + pub fn unique_txids(&self) -> HashSet { HashSet::from_iter(self.txstore.keys().cloned()) } @@ -409,6 +436,13 @@ impl Mempool { let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect(); let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect(); + // Get mempoolminfee for fee estimation + // [LOCK] No lock taken. Wait for RPC request. + let mempool_min_fee = daemon + .getmempoolinfo() + .map(|info| info.mempoolminfee * 100_000.0) // Convert from BTC/kB to sat/vB + .unwrap_or(1.0); // Default: 1 sat/vB + // 3. Remove missing transactions. Even if we are unable to download new transactions from // the daemon, we still want to remove the transactions that are no longer in the mempool. // [LOCK] Write lock is released at the end of the call to remove(). @@ -420,7 +454,7 @@ impl Mempool { .gettransactions(&txids_to_add) .chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?; - // 4. Update local mempool to match daemon's state + // 5. Update local mempool to match daemon's state // [LOCK] Takes Write lock for whole scope. { let mut mempool = mempool.write().unwrap(); @@ -429,15 +463,17 @@ impl Mempool { debug!("Mempool update added less transactions than expected"); } + // Update mempoolminfee + mempool.mempool_min_fee = mempool_min_fee; + mempool .count .with_label_values(&["txs"]) .set(mempool.txstore.len() as f64); // Update cached backlog stats (if expired) - if mempool.backlog_stats.1.elapsed() - > Duration::from_secs(mempool.config.mempool_backlog_stats_ttl) - { + let ttl = Duration::from_secs(mempool.config.mempool_backlog_stats_ttl); + if mempool.backlog_stats.1.elapsed() > ttl { let _timer = mempool .latency .with_label_values(&["update_backlog_stats"]) @@ -445,6 +481,15 @@ impl Mempool { mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now()); } + // Update projected blocks and recommended fees (if expired) + if mempool.projected_blocks.1.elapsed() > ttl { + let _timer = mempool + .latency + .with_label_values(&["update_projected_blocks"]) + .start_timer(); + mempool.update_projected_blocks(); + } + Ok(()) } } @@ -692,6 +737,56 @@ impl Mempool { .retain(|_outpoint, (txid, _vin)| !to_remove.contains(txid)); } + /// Build projected mempool blocks and calculate recommended fees. + /// + /// This method builds block templates using the GBT algorithm and + /// calculates recommended transaction fees based on the projected blocks. + fn update_projected_blocks(&mut self) { + // Build GBT transactions from mempool + let gbt_txs: Vec = self + .txstore + .iter() + .filter_map(|(txid, tx)| { + let fee_info = self.feeinfo.get(txid)?; + + // Get parent txids (inputs that are unconfirmed) + let parents: Vec = tx + .input + .iter() + .filter(|txin| has_prevout(txin)) + .filter_map(|txin| { + let parent_txid = txin.previous_output.txid; + if self.txstore.contains_key(&parent_txid) { + Some(parent_txid) + } else { + None + } + }) + .collect(); + + Some(GbtTransaction::new(*txid, fee_info, parents)) + }) + .collect(); + + // Build projected blocks (up to MAX_PROJECTED_BLOCKS blocks) + let result = build_projected_blocks(&gbt_txs, DEFAULT_BLOCK_WEIGHT, MAX_PROJECTED_BLOCKS); + + self.projected_blocks = (result.block_stats, Instant::now()); + + // Calculate recommended fees using cached mempoolminfee + let estimator = FeeEstimator::for_network(self.config.network_type); + let fees = + estimator.calculate_recommended_fees(&self.projected_blocks.0, self.mempool_min_fee); + + self.recommended_fees = (fees, Instant::now()); + + debug!( + "Updated projected blocks: {} blocks, recommended fastest fee: {} sat/vB", + self.projected_blocks.0.len(), + self.recommended_fees.0.fastest_fee + ); + } + #[cfg(feature = "liquid")] pub fn asset_history(&self, asset_id: &AssetId, limit: usize) -> Vec { let _timer = self diff --git a/src/new_index/mod.rs b/src/new_index/mod.rs index bc16a1b..aa8f21c 100644 --- a/src/new_index/mod.rs +++ b/src/new_index/mod.rs @@ -8,6 +8,7 @@ pub mod schema; pub use self::db::{DBRow, DB}; pub use self::fetch::{BlockEntry, FetchFrom}; pub use self::mempool::Mempool; +pub use self::mempool::MAX_PROJECTED_BLOCKS; pub use self::query::Query; pub use self::schema::{ compute_script_hash, parse_hash, ChainQuery, FundingInfo, Indexer, ScriptStats, SpendingInfo, diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 94016aa..0fd966d 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -210,26 +210,39 @@ impl Query { .copied() } - pub fn estimate_fee_map(&self) -> HashMap { - if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() { - if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) { - return cache.clone(); - } - } - - self.update_fee_estimates(); - self.cached_estimates.read().unwrap().0.clone() - } - fn update_fee_estimates(&self) { - match self.daemon.estimatesmartfee_batch(&CONF_TARGETS) { - Ok(estimates) => { - *self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now())); - } - Err(err) => { - warn!("failed estimating feerates: {:?}", err); + let mempool = self.mempool.read().unwrap(); + let projected_blocks = mempool.projected_blocks(); + + if projected_blocks.is_empty() { + // Fallback to Bitcoin Core RPC if no projected blocks available + drop(mempool); + match self.daemon.estimatesmartfee_batch(&CONF_TARGETS) { + Ok(estimates) => { + *self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now())); + } + Err(err) => { + warn!("failed estimating feerates: {:?}", err); + } } + return; } + + let mut estimates: HashMap = HashMap::with_capacity(CONF_TARGETS.len()); + let last_block_fee = projected_blocks.last().map(|b| b.median_fee).unwrap_or(1.0); + + for target in CONF_TARGETS { + let fee = if (target as usize) <= projected_blocks.len() { + // Use the median fee from the corresponding projected block (target-1 for 0-indexed) + projected_blocks[(target as usize) - 1].median_fee + } else { + // For targets beyond available blocks, use the last block's fee + last_block_fee + }; + estimates.insert(target, fee); + } + + *self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now())); } pub fn get_relayfee(&self) -> Result { diff --git a/src/rest.rs b/src/rest.rs index 93f18af..7cab658 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1771,8 +1771,16 @@ fn handle_request( json_response(recent, TTL_MEMPOOL_RECENT) } - (&Method::GET, Some(&"fee-estimates"), None, None, None, None) => { - json_response(query.estimate_fee_map(), TTL_SHORT) + // Recommended fees endpoint (mempool-style fee estimation) + (&Method::GET, Some(&"v1"), Some(&"fees"), Some(&"recommended"), None, None) => { + let mempool = query.mempool(); + json_response(mempool.recommended_fees(), TTL_SHORT) + } + + // Mempool blocks endpoint (projected blocks) + (&Method::GET, Some(&"v1"), Some(&"fees"), Some(&"mempool-blocks"), None, None) => { + let mempool = query.mempool(); + json_response(mempool.projected_blocks(), TTL_SHORT) } #[cfg(feature = "liquid")] diff --git a/src/util/fee_estimation.rs b/src/util/fee_estimation.rs new file mode 100644 index 0000000..93d4095 --- /dev/null +++ b/src/util/fee_estimation.rs @@ -0,0 +1,376 @@ +//! Fee estimation based on projected mempool blocks. +//! +//! This module calculates recommended transaction fees based on the fee statistics +//! of projected mempool blocks (created by the GBT algorithm). +//! +//! Ported from mempool's fee-api.ts. + +use crate::chain::Network; +use crate::util::gbt::MempoolBlock; + +/// Recommended fee rates for different confirmation time targets +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecommendedFees { + /// Fee rate for confirmation in the next block (sat/vB) + pub fastest_fee: f64, + /// Fee rate for confirmation within ~30 minutes / 3 blocks (sat/vB) + pub half_hour_fee: f64, + /// Fee rate for confirmation within ~1 hour / 6 blocks (sat/vB) + pub hour_fee: f64, + /// Economy fee rate (sat/vB) + pub economy_fee: f64, + /// Minimum relay fee rate (sat/vB) + pub minimum_fee: f64, +} + +impl Default for RecommendedFees { + fn default() -> Self { + Self { + fastest_fee: 1.0, + half_hour_fee: 1.0, + hour_fee: 1.0, + economy_fee: 1.0, + minimum_fee: 1.0, + } + } +} + +/// Fee estimation configuration +#[derive(Debug, Clone)] +pub struct FeeEstimationConfig { + /// Minimum fee increment for rounding (sat/vB) + pub minimum_increment: f64, + /// Minimum fastest fee (sat/vB) + pub min_fastest_fee: f64, + /// Minimum half hour fee (sat/vB) + pub min_half_hour_fee: f64, + /// Priority factor added to highest priority recommendations (sat/vB) + pub priority_factor: f64, +} + +impl FeeEstimationConfig { + /// Configuration for Bitcoin mainnet/testnet + pub fn bitcoin() -> Self { + Self { + minimum_increment: 1.0, + min_fastest_fee: 1.0, + min_half_hour_fee: 0.5, + priority_factor: 0.5, + } + } + + /// Configuration for Liquid network + pub fn liquid() -> Self { + Self { + minimum_increment: 0.1, + min_fastest_fee: 0.1, + min_half_hour_fee: 0.1, + priority_factor: 0.0, + } + } + + /// Create configuration based on network type + pub fn for_network(network: Network) -> Self { + if network.is_liquid() { + Self::liquid() + } else { + Self::bitcoin() + } + } +} + +/// Fee estimator that calculates recommended fees from projected blocks +pub struct FeeEstimator { + config: FeeEstimationConfig, +} + +impl FeeEstimator { + pub fn new(config: FeeEstimationConfig) -> Self { + Self { config } + } + + /// Create a fee estimator for the given network + pub fn for_network(network: Network) -> Self { + Self::new(FeeEstimationConfig::for_network(network)) + } + + /// Calculate recommended fees from projected mempool blocks. + /// + /// # Arguments + /// * `projected_blocks` - Projected mempool blocks from GBT algorithm + /// * `mempool_min_fee` - Minimum fee to get into mempool (BTC/kvB from getmempoolinfo) + pub fn calculate_recommended_fees( + &self, + projected_blocks: &[MempoolBlock], + mempool_min_fee: f64, + ) -> RecommendedFees { + self.calculate_recommended_fees_with_increment( + projected_blocks, + mempool_min_fee, + self.config.minimum_increment, + ) + } + + /// Calculate precise recommended fees with sub-satoshi precision. + /// + /// # Arguments + /// * `projected_blocks` - Projected mempool blocks from GBT algorithm + /// * `mempool_min_fee` - Minimum fee to get into mempool (BTC/kvB from getmempoolinfo) + pub fn calculate_precise_recommended_fees( + &self, + projected_blocks: &[MempoolBlock], + mempool_min_fee: f64, + ) -> RecommendedFees { + // Use 0.001 sat/vB precision (minimum non-zero minrelaytxfee/incrementalrelayfee) + let mut recommendations = self.calculate_recommended_fees_with_increment( + projected_blocks, + mempool_min_fee, + 0.001, + ); + + // Enforce floor & offset for highest priority recommendations + recommendations.fastest_fee = (recommendations.fastest_fee + self.config.priority_factor) + .max(self.config.min_fastest_fee); + recommendations.half_hour_fee = (recommendations.half_hour_fee + + self.config.priority_factor / 2.0) + .max(self.config.min_half_hour_fee); + + // Round to 3 decimal places + RecommendedFees { + fastest_fee: (recommendations.fastest_fee * 1000.0).round() / 1000.0, + half_hour_fee: (recommendations.half_hour_fee * 1000.0).round() / 1000.0, + hour_fee: (recommendations.hour_fee * 1000.0).round() / 1000.0, + economy_fee: (recommendations.economy_fee * 1000.0).round() / 1000.0, + minimum_fee: (recommendations.minimum_fee * 1000.0).round() / 1000.0, + } + } + + /// Internal fee calculation with configurable increment. + fn calculate_recommended_fees_with_increment( + &self, + projected_blocks: &[MempoolBlock], + mempool_min_fee: f64, + min_increment: f64, + ) -> RecommendedFees { + let purge_rate = round_up_to_nearest(mempool_min_fee, min_increment); + let minimum_fee = purge_rate.max(min_increment); + + if projected_blocks.is_empty() { + return RecommendedFees { + fastest_fee: minimum_fee, + half_hour_fee: minimum_fee, + hour_fee: minimum_fee, + economy_fee: minimum_fee, + minimum_fee, + }; + } + + // Calculate median fees for first 3 blocks + let first_median_fee = self.optimize_median_fee( + &projected_blocks[0], + projected_blocks.get(1), + None, + minimum_fee, + min_increment, + ); + + let second_median_fee = projected_blocks.get(1).map_or(minimum_fee, |block| { + self.optimize_median_fee( + block, + projected_blocks.get(2), + Some(first_median_fee), + minimum_fee, + min_increment, + ) + }); + + let third_median_fee = projected_blocks.get(2).map_or(minimum_fee, |block| { + self.optimize_median_fee( + block, + projected_blocks.get(3), + Some(second_median_fee), + minimum_fee, + min_increment, + ) + }); + + // Enforce minimum fee on all recommendations + let mut fastest_fee = first_median_fee.max(minimum_fee); + let mut half_hour_fee = second_median_fee.max(minimum_fee); + let mut hour_fee = third_median_fee.max(minimum_fee); + let economy_fee = (2.0 * minimum_fee).min(third_median_fee).max(minimum_fee); + + // Ensure recommendations always increase with priority + fastest_fee = fastest_fee + .max(half_hour_fee) + .max(hour_fee) + .max(economy_fee); + half_hour_fee = half_hour_fee.max(hour_fee).max(economy_fee); + hour_fee = hour_fee.max(economy_fee); + + RecommendedFees { + fastest_fee: round_to_nearest(fastest_fee, min_increment), + half_hour_fee: round_to_nearest(half_hour_fee, min_increment), + hour_fee: round_to_nearest(hour_fee, min_increment), + economy_fee: round_to_nearest(economy_fee, min_increment), + minimum_fee: round_to_nearest(minimum_fee, min_increment), + } + } + + /// Optimize median fee based on block fullness. + /// + /// For partially full blocks, the fee is scaled down proportionally. + fn optimize_median_fee( + &self, + block: &MempoolBlock, + next_block: Option<&MempoolBlock>, + previous_fee: Option, + min_fee: f64, + min_increment: f64, + ) -> f64 { + let use_fee = match previous_fee { + Some(prev) => (block.median_fee + prev) / 2.0, + None => block.median_fee, + }; + + // If block is less than half full or median fee is below minimum, use minimum + if block.block_vsize <= 500_000.0 || block.median_fee < min_fee { + return min_fee; + } + + // If block is between 50-95% full and there's no next block, + // scale the fee proportionally + if block.block_vsize <= 950_000.0 && next_block.is_none() { + let multiplier = (block.block_vsize - 500_000.0) / 500_000.0; + return round_to_nearest(use_fee * multiplier, min_increment).max(min_fee); + } + + round_up_to_nearest(use_fee, min_increment).max(min_fee) + } +} + +/// Round up to the nearest increment +fn round_up_to_nearest(value: f64, nearest: f64) -> f64 { + if nearest != 0.0 { + (value / nearest).ceil() * nearest + } else { + value + } +} + +/// Round to the nearest increment +fn round_to_nearest(value: f64, nearest: f64) -> f64 { + if nearest != 0.0 { + (value / nearest).round() * nearest + } else { + value + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_block(vsize: f64, median_fee: f64) -> MempoolBlock { + MempoolBlock { + block_size: vsize as u64, + block_vsize: vsize, + n_tx: 1000, + total_fees: 1000000, + median_fee, + fee_range: vec![1.0, 2.0, 3.0, median_fee, 5.0, 6.0, 7.0], + } + } + + #[test] + fn test_empty_mempool() { + let estimator = FeeEstimator::new(FeeEstimationConfig::bitcoin()); + let fees = estimator.calculate_recommended_fees(&[], 0.00001); + + assert_eq!(fees.fastest_fee, 1.0); + assert_eq!(fees.half_hour_fee, 1.0); + assert_eq!(fees.hour_fee, 1.0); + assert_eq!(fees.economy_fee, 1.0); + assert_eq!(fees.minimum_fee, 1.0); + } + + #[test] + fn test_sub_sat_mempool() { + let estimator = FeeEstimator::new(FeeEstimationConfig::bitcoin()); + + // Use median fee slightly above 1.0 (like the real mempool data: 1.002...) + // This tests the rounding behavior + let blocks = vec![ + create_test_block(997953.25, 1.002), // Rounds up to 2 + create_test_block(997963.0, 0.6), + create_test_block(997821.25, 0.52), + ]; + + let fees = estimator.calculate_recommended_fees(&blocks, 0.000001); + + assert_eq!(fees.fastest_fee, 2.0); + assert_eq!(fees.half_hour_fee, 1.0); + assert_eq!(fees.hour_fee, 1.0); + assert_eq!(fees.economy_fee, 1.0); + assert_eq!(fees.minimum_fee, 1.0); + } + + #[test] + fn test_low_fee_mempool() { + let estimator = FeeEstimator::new(FeeEstimationConfig::bitcoin()); + + let blocks = vec![ + create_test_block(997953.25, 2.0), + create_test_block(997963.0, 1.5), + create_test_block(997821.25, 1.0), + ]; + + let fees = estimator.calculate_recommended_fees(&blocks, 0.00001); + + assert_eq!(fees.fastest_fee, 2.0); + assert_eq!(fees.half_hour_fee, 2.0); + assert_eq!(fees.hour_fee, 2.0); + assert_eq!(fees.economy_fee, 2.0); + assert_eq!(fees.minimum_fee, 1.0); + } + + #[test] + fn test_partially_full_block() { + let estimator = FeeEstimator::new(FeeEstimationConfig::bitcoin()); + + // Block that's 75% full (750000 vsize) + let blocks = vec![create_test_block(750_000.0, 10.0)]; + + let fees = estimator.calculate_recommended_fees(&blocks, 0.00001); + + // Fee should be scaled down because block isn't full and there's no next block + // multiplier = (750000 - 500000) / 500000 = 0.5 + // So fee should be 10 * 0.5 = 5, rounded to 5 + assert_eq!(fees.fastest_fee, 5.0); + } + + #[test] + fn test_liquid_config() { + let estimator = FeeEstimator::new(FeeEstimationConfig::liquid()); + let fees = estimator.calculate_recommended_fees(&[], 0.000001); + + // Liquid uses 0.1 as minimum + assert_eq!(fees.minimum_fee, 0.1); + } + + #[test] + fn test_round_up_to_nearest() { + assert_eq!(round_up_to_nearest(1.1, 1.0), 2.0); + assert_eq!(round_up_to_nearest(1.0, 1.0), 1.0); + assert_eq!(round_up_to_nearest(0.15, 0.1), 0.2); + assert_eq!(round_up_to_nearest(5.0, 0.0), 5.0); + } + + #[test] + fn test_round_to_nearest() { + assert_eq!(round_to_nearest(1.4, 1.0), 1.0); + assert_eq!(round_to_nearest(1.6, 1.0), 2.0); + assert_eq!(round_to_nearest(0.14, 0.1), 0.1); + assert_eq!(round_to_nearest(0.16, 0.1), 0.2); + } +} diff --git a/src/util/fees.rs b/src/util/fees.rs index 43f0bf2..5984bbe 100644 --- a/src/util/fees.rs +++ b/src/util/fees.rs @@ -1,11 +1,14 @@ use crate::chain::{Network, Transaction, TxOut}; +use crate::util::transaction::sigops::transaction_sigop_count; use std::collections::HashMap; const VSIZE_BIN_WIDTH: u32 = 50_000; // in vbytes pub struct TxFeeInfo { - pub fee: u64, // in satoshis - pub vsize: u32, // in virtual bytes (= weight/4) + pub fee: u64, // in satoshis + pub vsize: u32, // in virtual bytes (= weight/4) + pub weight: u32, // transaction weight + pub sigops: u32, // signature operations count pub fee_per_vbyte: f32, } @@ -13,13 +16,19 @@ impl TxFeeInfo { pub fn new(tx: &Transaction, prevouts: &HashMap, network: Network) -> Self { let fee = get_tx_fee(tx, prevouts, network); #[cfg(not(feature = "liquid"))] - let vsize = tx.weight().to_wu() / 4; + let weight = tx.weight().to_wu() as u32; #[cfg(feature = "liquid")] - let vsize = tx.weight() / 4; + let weight = tx.weight() as u32; + let vsize = weight / 4; + + // Calculate sigops, defaulting to 0 on error (e.g., coinbase) + let sigops = transaction_sigop_count(tx, prevouts).unwrap_or(0) as u32; TxFeeInfo { fee, - vsize: vsize as u32, + vsize, + weight, + sigops, fee_per_vbyte: fee as f32 / vsize as f32, } } diff --git a/src/util/gbt.rs b/src/util/gbt.rs new file mode 100644 index 0000000..f0547cd --- /dev/null +++ b/src/util/gbt.rs @@ -0,0 +1,702 @@ +//! Block template construction (GetBlockTemplate) algorithm. +//! +//! This module implements an approximation of the transaction selection algorithm +//! from Bitcoin Core's BlockAssembler to create projected mempool blocks. +//! +//! Ported from mempool's Rust GBT implementation. + +use std::collections::{BinaryHeap, HashMap, HashSet}; + +use crate::chain::Txid; +use crate::util::fees::TxFeeInfo; + +/// Default block weight limit (4MB weight = 1MB vsize for worst case) +pub const DEFAULT_BLOCK_WEIGHT: u32 = 4_000_000; +/// Maximum sigops per block +const BLOCK_SIGOPS: u32 = 80_000; +/// Reserved weight for coinbase +const BLOCK_RESERVED_WEIGHT: u32 = 4_000; +/// Reserved sigops for coinbase +const BLOCK_RESERVED_SIGOPS: u32 = 400; + +/// A projected mempool block with fee statistics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MempoolBlock { + /// Total size of transactions in bytes + pub block_size: u64, + /// Total virtual size of transactions (weight/4) + pub block_vsize: f64, + /// Number of transactions + pub n_tx: usize, + /// Total fees in satoshis + pub total_fees: u64, + /// Median fee rate in sat/vB + pub median_fee: f64, + /// Fee rate range [min, 10th, 25th, 50th, 75th, 90th, max] in sat/vB + pub fee_range: Vec, +} + +impl Default for MempoolBlock { + fn default() -> Self { + Self { + block_size: 0, + block_vsize: 0.0, + n_tx: 0, + total_fees: 0, + median_fee: 0.0, + fee_range: vec![0.0; 7], + } + } +} + +/// Transaction data for block template construction +#[derive(Debug, Clone)] +pub struct GbtTransaction { + pub txid: Txid, + pub fee: u64, + pub weight: u32, + pub sigops: u32, + /// Indices of parent transactions in the mempool (by txid) + pub parents: Vec, +} + +impl GbtTransaction { + pub fn new(txid: Txid, fee_info: &TxFeeInfo, parents: Vec) -> Self { + Self { + txid, + fee: fee_info.fee, + weight: fee_info.weight, + sigops: fee_info.sigops, + parents, + } + } + + #[inline] + pub fn vsize(&self) -> u32 { + self.weight.div_ceil(4) + } + + #[inline] + pub fn fee_rate(&self) -> f64 { + self.fee as f64 / self.vsize() as f64 + } + + /// Calculate sigop-adjusted vsize (rounded up) + #[inline] + pub fn sigop_adjusted_vsize(&self) -> u32 { + self.vsize().max(self.sigops * 5) + } + + /// Calculate sigop-adjusted weight + #[inline] + pub fn sigop_adjusted_weight(&self) -> u32 { + self.weight.max(self.sigops * 20) + } +} + +/// Internal audit transaction for GBT algorithm +#[derive(Debug, Clone)] +struct AuditTransaction { + fee: u64, + weight: u32, + sigop_adjusted_weight: u32, + sigop_adjusted_vsize: u32, + sigops: u32, + effective_fee_per_vsize: f64, + parents: Vec, + ancestors: HashSet, + children: HashSet, + ancestor_fee: u64, + ancestor_sigop_adjusted_weight: u32, + ancestor_sigop_adjusted_vsize: u32, + ancestor_sigops: u32, + score: f64, + used: bool, + modified: bool, +} + +impl AuditTransaction { + fn from_gbt_tx(tx: &GbtTransaction) -> Self { + let sigop_adjusted_vsize = tx.sigop_adjusted_vsize(); + let sigop_adjusted_weight = tx.sigop_adjusted_weight(); + let fee_per_vsize = tx.fee_rate(); + + Self { + fee: tx.fee, + weight: tx.weight, + sigop_adjusted_weight, + sigop_adjusted_vsize, + sigops: tx.sigops, + effective_fee_per_vsize: fee_per_vsize, + parents: tx.parents.clone(), + ancestors: HashSet::new(), + children: HashSet::new(), + ancestor_fee: tx.fee, + ancestor_sigop_adjusted_weight: sigop_adjusted_weight, + ancestor_sigop_adjusted_vsize: sigop_adjusted_vsize, + ancestor_sigops: tx.sigops, + score: fee_per_vsize, + used: false, + modified: false, + } + } + + #[inline] + fn ancestor_score(&self) -> f64 { + if self.ancestor_sigop_adjusted_vsize == 0 { + 0.0 + } else { + self.ancestor_fee as f64 / self.ancestor_sigop_adjusted_vsize as f64 + } + } + + fn update_score(&mut self) { + self.score = self.ancestor_score(); + } +} + +/// Priority entry for the modified transactions queue +#[derive(Debug, Clone)] +struct TxPriority { + txid: Txid, + score: f64, +} + +impl PartialEq for TxPriority { + fn eq(&self, other: &Self) -> bool { + self.txid == other.txid + } +} + +impl Eq for TxPriority {} + +impl PartialOrd for TxPriority { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for TxPriority { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Higher score = higher priority (reverse order for BinaryHeap) + self.score + .partial_cmp(&other.score) + .unwrap_or(std::cmp::Ordering::Equal) + } +} + +/// Result of the GBT algorithm +#[derive(Debug)] +pub struct GbtResult { + /// Projected blocks, each containing transaction IDs + pub blocks: Vec>, + /// Statistics for each projected block + pub block_stats: Vec, +} + +/// Build projected mempool blocks using an approximation of Bitcoin Core's transaction selection. +/// +/// Returns up to `max_blocks` projected blocks with their fee statistics. +pub fn build_projected_blocks( + transactions: &[GbtTransaction], + max_block_weight: u32, + max_blocks: usize, +) -> GbtResult { + if transactions.is_empty() || max_blocks == 0 { + return GbtResult { + blocks: vec![], + block_stats: vec![], + }; + } + + // Build audit pool indexed by txid + let mut audit_pool: HashMap = transactions + .iter() + .map(|tx| (tx.txid, AuditTransaction::from_gbt_tx(tx))) + .collect(); + + // Set up ancestor/descendant relationships + let txids: Vec = audit_pool.keys().cloned().collect(); + for txid in &txids { + set_relatives(txid, &mut audit_pool); + } + + // Sort by descending ancestor score + let mut mempool_stack: Vec = txids; + mempool_stack.sort_by(|a, b| { + let score_a = audit_pool.get(a).map(|tx| tx.score).unwrap_or(0.0); + let score_b = audit_pool.get(b).map(|tx| tx.score).unwrap_or(0.0); + score_b + .partial_cmp(&score_a) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + // Build blocks + let mut blocks: Vec> = Vec::new(); + let mut block_fee_rates: Vec> = Vec::new(); + let mut current_block: Vec = Vec::new(); + let mut current_fee_rates: Vec = Vec::new(); + let mut block_weight: u32 = BLOCK_RESERVED_WEIGHT; + let mut block_sigops: u32 = BLOCK_RESERVED_SIGOPS; + #[allow(unused_variables)] + let mut block_fees: u64 = 0; + + let mut modified: BinaryHeap = BinaryHeap::new(); + let mut overflow: Vec = Vec::new(); + let mut failures = 0; + + while (!mempool_stack.is_empty() || !modified.is_empty()) && blocks.len() < max_blocks { + // Get next best transaction from either stack or modified queue + let next_txid = get_next_tx(&mut mempool_stack, &mut modified, &audit_pool); + + let next_txid = match next_txid { + Some(txid) => txid, + None => break, + }; + + let (ancestor_weight, ancestor_sigops, _ancestor_fee, ancestor_score) = { + let tx = match audit_pool.get(&next_txid) { + Some(tx) if !tx.used => tx, + _ => continue, + }; + ( + tx.ancestor_sigop_adjusted_weight, + tx.ancestor_sigops, + tx.ancestor_fee, + tx.score, + ) + }; + + // Check if this package fits in the current block + if blocks.len() < max_blocks - 1 + && (block_weight + ancestor_weight >= max_block_weight - BLOCK_RESERVED_WEIGHT + || block_sigops + ancestor_sigops > BLOCK_SIGOPS) + { + overflow.push(next_txid); + failures += 1; + } else { + // Add the package (ancestors + this transaction) to the block + let package = get_package(&next_txid, &audit_pool); + + for pkg_txid in &package { + if let Some(tx) = audit_pool.get_mut(pkg_txid) { + if !tx.used { + tx.used = true; + current_block.push(*pkg_txid); + current_fee_rates.push(tx.effective_fee_per_vsize); + block_weight += tx.sigop_adjusted_weight; + block_sigops += tx.sigops; + block_fees += tx.fee; + } + } + } + + // Update descendants + update_descendants(&next_txid, &mut audit_pool, &mut modified, ancestor_score); + failures = 0; + } + + // Check if block is full + let exceeded_tries = + failures > 1000 && block_weight > (max_block_weight - BLOCK_RESERVED_WEIGHT - 4_000); + let queues_empty = mempool_stack.is_empty() && modified.is_empty(); + + if (exceeded_tries || queues_empty) + && blocks.len() < max_blocks - 1 + && !current_block.is_empty() + { + blocks.push(std::mem::take(&mut current_block)); + block_fee_rates.push(std::mem::take(&mut current_fee_rates)); + block_weight = BLOCK_RESERVED_WEIGHT; + block_sigops = BLOCK_RESERVED_SIGOPS; + block_fees = 0; + failures = 0; + + // Move overflow back to processing + overflow.reverse(); + for txid in overflow.drain(..) { + if let Some(tx) = audit_pool.get(&txid) { + if tx.modified { + modified.push(TxPriority { + txid, + score: tx.score, + }); + } else { + mempool_stack.push(txid); + } + } + } + } + } + + // Add final block if not empty + if !current_block.is_empty() { + blocks.push(current_block); + block_fee_rates.push(current_fee_rates); + } + + // Calculate block statistics + let block_stats: Vec = blocks + .iter() + .zip(block_fee_rates.iter()) + .map(|(block_txids, fee_rates)| calculate_block_stats(block_txids, fee_rates, &audit_pool)) + .collect(); + + GbtResult { + blocks, + block_stats, + } +} + +fn get_next_tx( + mempool_stack: &mut Vec, + modified: &mut BinaryHeap, + audit_pool: &HashMap, +) -> Option { + loop { + // Get candidates from both queues + let stack_candidate = mempool_stack.last().and_then(|txid| { + audit_pool + .get(txid) + .filter(|tx| !tx.used && !tx.modified) + .map(|tx| (*txid, tx.score)) + }); + + let modified_candidate = modified.peek().and_then(|priority| { + audit_pool + .get(&priority.txid) + .filter(|tx| !tx.used) + .map(|tx| (priority.txid, tx.score)) + }); + + match (stack_candidate, modified_candidate) { + (Some((stack_txid, stack_score)), Some((mod_txid, mod_score))) => { + if mod_score >= stack_score { + modified.pop(); + return Some(mod_txid); + } else { + mempool_stack.pop(); + return Some(stack_txid); + } + } + (Some((txid, _)), None) => { + mempool_stack.pop(); + return Some(txid); + } + (None, Some((txid, _))) => { + modified.pop(); + return Some(txid); + } + (None, None) => { + // Try to clean up invalid entries + if mempool_stack.pop().is_some() { + continue; + } + if modified.pop().is_some() { + continue; + } + return None; + } + } + } +} + +fn set_relatives(txid: &Txid, audit_pool: &mut HashMap) { + // Get parents for this transaction + let parents: Vec = match audit_pool.get(txid) { + Some(tx) => tx + .parents + .iter() + .filter(|p| audit_pool.contains_key(*p)) + .cloned() + .collect(), + None => return, + }; + + // Recursively set relatives for parents first + for parent_txid in &parents { + if audit_pool + .get(parent_txid) + .map(|tx| tx.ancestors.is_empty() && !tx.parents.is_empty()) + .unwrap_or(false) + { + set_relatives(parent_txid, audit_pool); + } + } + + // Collect ancestor info + let mut ancestors: HashSet = HashSet::new(); + let mut total_fee: u64 = 0; + let mut total_sigop_adjusted_weight: u32 = 0; + let mut total_sigop_adjusted_vsize: u32 = 0; + let mut total_sigops: u32 = 0; + + for parent_txid in &parents { + if let Some(parent) = audit_pool.get(parent_txid) { + ancestors.insert(*parent_txid); + for ancestor in &parent.ancestors { + ancestors.insert(*ancestor); + } + total_fee += parent.fee; + total_sigop_adjusted_weight += parent.sigop_adjusted_weight; + total_sigop_adjusted_vsize += parent.sigop_adjusted_vsize; + total_sigops += parent.sigops; + } + } + + // Add ancestor stats from indirect ancestors + for ancestor_txid in &ancestors { + if !parents.contains(ancestor_txid) { + if let Some(ancestor) = audit_pool.get(ancestor_txid) { + total_fee += ancestor.fee; + total_sigop_adjusted_weight += ancestor.sigop_adjusted_weight; + total_sigop_adjusted_vsize += ancestor.sigop_adjusted_vsize; + total_sigops += ancestor.sigops; + } + } + } + + // Update the transaction + if let Some(tx) = audit_pool.get_mut(txid) { + tx.ancestors = ancestors; + tx.ancestor_fee += total_fee; + tx.ancestor_sigop_adjusted_weight += total_sigop_adjusted_weight; + tx.ancestor_sigop_adjusted_vsize += total_sigop_adjusted_vsize; + tx.ancestor_sigops += total_sigops; + tx.update_score(); + } + + // Update children of parents + for parent_txid in parents { + if let Some(parent) = audit_pool.get_mut(&parent_txid) { + parent.children.insert(*txid); + } + } +} + +fn get_package(txid: &Txid, audit_pool: &HashMap) -> Vec { + let mut package: Vec<(Txid, usize)> = Vec::new(); + + if let Some(tx) = audit_pool.get(txid) { + // Add ancestors first, sorted by ancestor count (so parents come before children) + for ancestor_txid in &tx.ancestors { + if let Some(ancestor) = audit_pool.get(ancestor_txid) { + if !ancestor.used { + package.push((*ancestor_txid, ancestor.ancestors.len())); + } + } + } + package.sort_by_key(|(_, count)| *count); + + // Add the transaction itself + package.push((*txid, tx.ancestors.len())); + } + + package.into_iter().map(|(txid, _)| txid).collect() +} + +fn update_descendants( + root_txid: &Txid, + audit_pool: &mut HashMap, + modified: &mut BinaryHeap, + cluster_rate: f64, +) { + let (root_fee, root_sigop_adjusted_weight, root_sigop_adjusted_vsize, root_sigops, children) = { + match audit_pool.get(root_txid) { + Some(tx) => ( + tx.fee, + tx.sigop_adjusted_weight, + tx.sigop_adjusted_vsize, + tx.sigops, + tx.children.clone(), + ), + None => return, + } + }; + + let mut visited: HashSet = HashSet::new(); + let mut stack: Vec = children.into_iter().collect(); + + while let Some(desc_txid) = stack.pop() { + if visited.contains(&desc_txid) { + continue; + } + visited.insert(desc_txid); + + let children_to_add: Vec; + let old_score: f64; + let new_score: f64; + + { + let descendant = match audit_pool.get_mut(&desc_txid) { + Some(tx) => tx, + None => continue, + }; + + old_score = descendant.score; + + // Remove root from ancestors + descendant.ancestors.remove(root_txid); + descendant.ancestor_fee = descendant.ancestor_fee.saturating_sub(root_fee); + descendant.ancestor_sigop_adjusted_weight = descendant + .ancestor_sigop_adjusted_weight + .saturating_sub(root_sigop_adjusted_weight); + descendant.ancestor_sigop_adjusted_vsize = descendant + .ancestor_sigop_adjusted_vsize + .saturating_sub(root_sigop_adjusted_vsize); + descendant.ancestor_sigops = descendant.ancestor_sigops.saturating_sub(root_sigops); + + // Update effective fee rate based on cluster rate + if cluster_rate < descendant.effective_fee_per_vsize { + descendant.effective_fee_per_vsize = cluster_rate; + } + + descendant.update_score(); + new_score = descendant.score; + + children_to_add = descendant.children.iter().cloned().collect(); + } + + // Add to modified queue if score changed + if (new_score - old_score).abs() > f64::EPSILON { + if let Some(tx) = audit_pool.get_mut(&desc_txid) { + tx.modified = true; + } + modified.push(TxPriority { + txid: desc_txid, + score: new_score, + }); + } + + // Add children to stack + for child in children_to_add { + if !visited.contains(&child) { + stack.push(child); + } + } + } +} + +fn calculate_block_stats( + txids: &[Txid], + fee_rates: &[f64], + audit_pool: &HashMap, +) -> MempoolBlock { + if txids.is_empty() { + return MempoolBlock::default(); + } + + let mut total_size: u64 = 0; + let mut total_weight: u64 = 0; + let mut total_fees: u64 = 0; + + for txid in txids { + if let Some(tx) = audit_pool.get(txid) { + total_weight += tx.weight as u64; + total_size += tx.weight as u64 / 4; // Approximate size + total_fees += tx.fee; + } + } + + let mut sorted_rates = fee_rates.to_vec(); + sorted_rates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let n = sorted_rates.len(); + let median_fee = if n == 0 { + 0.0 + } else if n % 2 == 0 { + (sorted_rates[n / 2 - 1] + sorted_rates[n / 2]) / 2.0 + } else { + sorted_rates[n / 2] + }; + + // Calculate percentiles for fee range: [min, 10th, 25th, 50th, 75th, 90th, max] + let fee_range = if n == 0 { + vec![0.0; 7] + } else { + vec![ + sorted_rates[0], + sorted_rates[(n as f64 * 0.1) as usize], + sorted_rates[(n as f64 * 0.25) as usize], + sorted_rates[(n as f64 * 0.5) as usize], + sorted_rates[((n as f64 * 0.75) as usize).min(n - 1)], + sorted_rates[((n as f64 * 0.9) as usize).min(n - 1)], + sorted_rates[n - 1], + ] + }; + + MempoolBlock { + block_size: total_size, + block_vsize: total_weight as f64 / 4.0, + n_tx: txids.len(), + total_fees, + median_fee, + fee_range, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bitcoin::hashes::Hash; + + fn make_txid(n: u8) -> Txid { + let mut bytes = [0u8; 32]; + bytes[0] = n; + Txid::from_slice(&bytes).unwrap() + } + + #[test] + fn test_empty_mempool() { + let result = build_projected_blocks(&[], DEFAULT_BLOCK_WEIGHT, 8); + assert!(result.blocks.is_empty()); + assert!(result.block_stats.is_empty()); + } + + #[test] + fn test_single_transaction() { + let txid = make_txid(1); + let tx = GbtTransaction { + txid, + fee: 1000, + weight: 400, + sigops: 1, + parents: vec![], + }; + + let result = build_projected_blocks(&[tx], DEFAULT_BLOCK_WEIGHT, 8); + assert_eq!(result.blocks.len(), 1); + assert_eq!(result.blocks[0].len(), 1); + assert_eq!(result.blocks[0][0], txid); + } + + #[test] + fn test_parent_child_relationship() { + let parent_txid = make_txid(1); + let child_txid = make_txid(2); + + let parent = GbtTransaction { + txid: parent_txid, + fee: 500, + weight: 400, + sigops: 1, + parents: vec![], + }; + + let child = GbtTransaction { + txid: child_txid, + fee: 1000, + weight: 400, + sigops: 1, + parents: vec![parent_txid], + }; + + let result = build_projected_blocks(&[parent, child], DEFAULT_BLOCK_WEIGHT, 8); + assert_eq!(result.blocks.len(), 1); + assert_eq!(result.blocks[0].len(), 2); + // Parent should come before child + let parent_pos = result.blocks[0].iter().position(|&t| t == parent_txid); + let child_pos = result.blocks[0].iter().position(|&t| t == child_txid); + assert!(parent_pos < child_pos); + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index def76a2..93b48e3 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -4,7 +4,9 @@ mod transaction; pub mod bincode_util; pub mod electrum_merkle; +pub mod fee_estimation; pub mod fees; +pub mod gbt; pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList}; pub use self::fees::get_tx_fee;