seedhammer/dcrd
seedhammer/dcrd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mempool: Implement orphan expiration.

Browse Source 
This implements orphan expiration in the mempool such that any orphans
that have not had their ancestors materialize within 15 minutes of their
initial arrival time will be evicted which in turn will remove any other
orphans that attempted to redeem it.

In order to perform the evictions with reasonable efficiency, an
opportunistic scan interval of 5 minutes is used.  That is to say that
there is not a hard deadline on the scan interval and instead it runs
when a new orphan is added to the pool if enough time has passed.

The following is an example of running this code against the main
network for around an hour while intentionally generating orphans to
test:

2019-10-28 01:44:36.272 [DBG] TXMP: Expired 2 orphans (remaining: 46)
2019-10-28 01:49:39.915 [DBG] TXMP: Expired 17 orphans (remaining: 49)
2019-10-28 01:54:47.704 [DBG] TXMP: Expired 15 orphans (remaining: 43)
2019-10-28 01:59:53.414 [DBG] TXMP: Expired 16 orphans (remaining: 56)
2019-10-28 02:05:09.858 [DBG] TXMP: Expired 16 orphans (remaining: 68)
2019-10-28 02:10:10.567 [DBG] TXMP: Expired 16 orphans (remaining: 78)
2019-10-28 02:15:11.736 [DBG] TXMP: Expired 26 orphans (remaining: 82)
2019-10-28 02:48:34.352 [DBG] TXMP: Expired 85 orphans (remaining: 0)

As can be seen from the above, without orphan expiration on this data
set, the orphan pool would have grown an additional 193 entries.
This commit is contained in:
Dave Collins 2016-08-23 23:27:41 -05:00
parent a195642ec7
commit 93efe6bfe3
No known key found for this signature in database
GPG Key ID: B8904D9D9C93D1F2
2 changed files with 84 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
mempool/log.go
View File

@ -29,3 +29,12 @@ func DisableLog() {
func UseLogger(logger slog.Logger) {
log = logger
}
// pickNoun returns the singular or plural form of a noun depending
// on the count n.
func pickNoun(n int, singular, plural string) string {
if n == 1 {
return singular
}
return plural
}

94
mempool/mempool.go
View File

@ -56,6 +56,15 @@ const (
// maxNullDataOutputs is the maximum number of OP_RETURN null data
// pushes in a transaction, after which it is considered non-standard.
maxNullDataOutputs = 4
// orphanTTL is the maximum amount of time an orphan is allowed to
// stay in the orphan pool before it expires and is evicted during the
// next scan.
orphanTTL = time.Minute * 15
// orphanExpireScanInterval is the minimum amount of time in between
// scans of the orphan pool to evict expired transactions.
orphanExpireScanInterval = time.Minute * 5
)
// Config is a descriptor containing the memory pool configuration.
@ -217,6 +226,14 @@ type VerboseTxDesc struct {
Depends []*TxDesc
}
// orphanTx is a normal transaction that references an ancestor transaction
// that is not yet available. It also contains additional information related
// to it such as an expiration time to help prevent caching the orphan forever.
type orphanTx struct {
tx *dcrutil.Tx
expiration time.Time
}
// TxPool is used as a source of transactions that need to be mined into blocks
// and relayed to other peers. It is safe for concurrent access from multiple
// peers.
@ -224,10 +241,11 @@ type TxPool struct {
// The following variables must only be used atomically.
lastUpdated int64 // last time pool was updated.
mtx sync.RWMutex
cfg Config
pool map[chainhash.Hash]*TxDesc
orphans map[chainhash.Hash]*dcrutil.Tx
mtx sync.RWMutex
cfg Config
pool map[chainhash.Hash]*TxDesc
orphans map[chainhash.Hash]*orphanTx
orphansByPrev map[wire.OutPoint]map[chainhash.Hash]*dcrutil.Tx
outpoints map[wire.OutPoint]*dcrutil.Tx
@ -237,6 +255,12 @@ type TxPool struct {
pennyTotal float64 // exponentially decaying total for penny spends.
lastPennyUnix int64 // unix time of last ``penny spend''
// nextExpireScan is the time after which the orphan pool will be
// scanned in order to evict orphans. This is NOT a hard deadline as
// the scan will only run when an orphan is added to the pool as opposed
// to on an unconditional timer.
nextExpireScan time.Time
}
// insertVote inserts a vote into the map of block votes.
@ -329,10 +353,9 @@ var _ mining.TxSource = (*TxPool)(nil)
//
// This function MUST be called with the mempool lock held (for writes).
func (mp *TxPool) removeOrphan(tx *dcrutil.Tx, removeRedeemers bool) {
txHash := tx.Hash()
// Nothing to do if passed tx is not an orphan.
tx, exists := mp.orphans[*txHash]
txHash := tx.Hash()
otx, exists := mp.orphans[*txHash]
if !exists {
return
}
@ -340,7 +363,7 @@ func (mp *TxPool) removeOrphan(tx *dcrutil.Tx, removeRedeemers bool) {
log.Tracef("Removing orphan transaction %v", txHash)
// Remove the reference from the previous orphan index.
for _, txIn := range tx.MsgTx().TxIn {
for _, txIn := range otx.tx.MsgTx().TxIn {
orphans, exists := mp.orphansByPrev[txIn.PreviousOutPoint]
if exists {
delete(orphans, *txHash)
@ -389,6 +412,32 @@ func (mp *TxPool) RemoveOrphan(tx *dcrutil.Tx) {
//
// This function MUST be called with the mempool lock held (for writes).
func (mp *TxPool) limitNumOrphans() {
// Scan through the orphan pool and remove any expired orphans when it's
// time. This is done for efficiency so the scan only happens periodically
// instead of on every orphan added to the pool.
if now := time.Now(); now.After(mp.nextExpireScan) {
origNumOrphans := len(mp.orphans)
for _, otx := range mp.orphans {
if now.After(otx.expiration) {
// Remove redeemers too because the missing parents are very
// unlikely to ever materialize since the orphan has already
// been around more than long enough for them to be delivered.
mp.removeOrphan(otx.tx, true)
}
}
// Set next expiration scan to occur after the scan interval.
mp.nextExpireScan = now.Add(orphanExpireScanInterval)
numOrphans := len(mp.orphans)
if numExpired := origNumOrphans - numOrphans; numExpired > 0 {
log.Debugf("Expired %d %s (remaining: %d)", numExpired,
pickNoun(numExpired, "orphan", "orphans"), numOrphans)
}
}
// Nothing to do if adding another orphan will not cause the pool to
// exceed the limit.
if len(mp.orphans)+1 <= mp.cfg.Policy.MaxOrphanTxs {
return
}
@ -399,8 +448,10 @@ func (mp *TxPool) limitNumOrphans() {
// is not important here because an adversary would have to be
// able to pull off preimage attacks on the hashing function in
// order to target eviction of specific entries anyways.
for _, tx := range mp.orphans {
mp.removeOrphan(tx, false)
for _, otx := range mp.orphans {
// Don't remove redeemers in the case of a random eviction since
// it is quite possible it might be needed again shortly.
mp.removeOrphan(otx.tx, false)
break
}
}
@ -414,11 +465,15 @@ func (mp *TxPool) addOrphan(tx *dcrutil.Tx) {
return
}
// Limit the number orphan transactions to prevent memory exhaustion. A
// random orphan is evicted to make room if needed.
// Limit the number orphan transactions to prevent memory exhaustion.
// This will periodically remove any expired orphans and evict a random
// orphan if space is still needed.
mp.limitNumOrphans()
mp.orphans[*tx.Hash()] = tx
mp.orphans[*tx.Hash()] = &orphanTx{
tx: tx,
expiration: time.Now().Add(orphanTTL),
}
for _, txIn := range tx.MsgTx().TxIn {
if _, exists := mp.orphansByPrev[txIn.PreviousOutPoint]; !exists {
mp.orphansByPrev[txIn.PreviousOutPoint] =
@ -1695,11 +1750,12 @@ func (mp *TxPool) LastUpdated() time.Time {
// transactions until they are mined into a block.
func New(cfg *Config) *TxPool {
return &TxPool{
cfg: *cfg,
pool: make(map[chainhash.Hash]*TxDesc),
orphans: make(map[chainhash.Hash]*dcrutil.Tx),
orphansByPrev: make(map[wire.OutPoint]map[chainhash.Hash]*dcrutil.Tx),
outpoints: make(map[wire.OutPoint]*dcrutil.Tx),
votes: make(map[chainhash.Hash][]mining.VoteDesc),
cfg: *cfg,
pool: make(map[chainhash.Hash]*TxDesc),
orphans: make(map[chainhash.Hash]*orphanTx),
orphansByPrev: make(map[wire.OutPoint]map[chainhash.Hash]*dcrutil.Tx),
outpoints: make(map[wire.OutPoint]*dcrutil.Tx),
votes: make(map[chainhash.Hash][]mining.VoteDesc),
nextExpireScan: time.Now().Add(orphanExpireScanInterval),
}
}