From 89ab143dcbefcab5d866cab469ca70dc5fd41d9f Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 26 Oct 2019 15:37:13 -0500 Subject: [PATCH] indexers: Use spend journal for index catchup. When an index is enabled for the first time (or after having previously been disabled and re-enabled), the index needs to be updated to account for the current state of the main chain. This poses a problem for indexes that require access to the previous scripts, such as the address index, since the outputs are no longer in the utxo set due to pruning. Currently, in order to handle this, the code has to load all of the referenced transactions in order to get access to the output scripts which is extremely inefficient. The address index is already notoriously time consuming to generate due to the sheer amount of information, but it is only exacerbated by this inefficiency. However, the necessary information exists in a much more efficient form in the spend journal and the recently introduced interfaces which decouple the script lookup from the concrete type provide a path to significantly optimize the index catch up process. Consequently, this modifies the indexer.ChainQueryer interface to provide access to a source of previous transaction scripts and their associated versions spent by the given block, updates the index initialization to make use of it, and updates the blockchain implementation of the interface to provide the required information via the spend journal. As an example of the speedup obtained by this, the following shows how long it took to catch up the address index from block 0 to block 391500 on a 7200 RPM HDD: before: 55 minutes, 25 seconds after: 34 minutes, 4 seconds That amounts to about a 38.5% reduction. It is also worth noting that the speedup is not linear, so as the number of transactions continues to the grow, the speedup will be increasingly more significant. --- blockchain/chain.go | 18 +++++++ blockchain/indexers/common.go | 4 ++ blockchain/indexers/manager.go | 98 +--------------------------------- 3 files changed, 24 insertions(+), 96 deletions(-) diff --git a/blockchain/chain.go b/blockchain/chain.go index 0ccce654..6432a8ad 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -2039,6 +2039,24 @@ func (q *chainQueryerAdapter) BestHeight() int64 { return q.BestSnapshot().Height } +// PrevScripts returns a source of previous transaction scripts and their +// associated versions spent by the given block by using the spend journal. +// +// It is defined via a separate internal struct to avoid polluting the public +// API of the BlockChain type itself. +// +// This is part of the indexers.ChainQueryer interface. +func (q *chainQueryerAdapter) PrevScripts(dbTx database.Tx, block *dcrutil.Block) (indexers.PrevScripter, error) { + // Load all of the spent transaction output data from the database. + stxos, err := dbFetchSpendJournalEntry(dbTx, block) + if err != nil { + return nil, err + } + + prevScripts := stxosToScriptSource(block, stxos, currentCompressionVersion) + return prevScripts, nil +} + // Config is a descriptor which specifies the blockchain instance configuration. type Config struct { // DB defines the database which houses the blocks and will be used to diff --git a/blockchain/indexers/common.go b/blockchain/indexers/common.go index c3c36bec..22a204bf 100644 --- a/blockchain/indexers/common.go +++ b/blockchain/indexers/common.go @@ -89,6 +89,10 @@ type ChainQueryer interface { // BlockHashByHeight returns the hash of the block at the given height in // the main chain. BlockHashByHeight(int64) (*chainhash.Hash, error) + + // PrevScripts returns a source of previous transaction scripts and their + // associated versions spent by the given block. + PrevScripts(database.Tx, *dcrutil.Block) (PrevScripter, error) } // IndexManager provides a generic interface that is called when blocks are diff --git a/blockchain/indexers/manager.go b/blockchain/indexers/manager.go index 9c6cfc88..5466b27d 100644 --- a/blockchain/indexers/manager.go +++ b/blockchain/indexers/manager.go @@ -9,7 +9,6 @@ import ( "bytes" "fmt" - "github.com/decred/dcrd/blockchain/stake/v3" "github.com/decred/dcrd/blockchain/v3/internal/progresslog" "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrd/chaincfg/v2" @@ -468,8 +467,7 @@ func (m *Manager) Init(chain ChainQueryer, interrupt <-chan struct{}) error { var prevScripts PrevScripter if indexNeedsInputs(indexer) { var err error - prevScripts, err = makeScriptSource(dbTx, block, - interrupt) + prevScripts, err = chain.PrevScripts(dbTx, block) if err != nil { return err } @@ -608,8 +606,7 @@ func (m *Manager) Init(chain ChainQueryer, interrupt <-chan struct{}) error { // index. if prevScripts == nil && indexNeedsInputs(indexer) { var errMakeView error - prevScripts, errMakeView = makeScriptSource(dbTx, block, - interrupt) + prevScripts, errMakeView = chain.PrevScripts(dbTx, block) if errMakeView != nil { return errMakeView } @@ -673,97 +670,6 @@ func dbFetchTx(dbTx database.Tx, hash *chainhash.Hash) (*wire.MsgTx, error) { return &msgTx, nil } -// scriptSourceEntry houses a script and its associated version. -type scriptSourceEntry struct { - version uint16 - script []byte -} - -// scriptSource provides a source of transaction output scripts and their -// associated script version for given outpoints and implements the PrevScripter -// interface so it may be used in cases that require access to said scripts. -type scriptSource map[wire.OutPoint]scriptSourceEntry - -// PrevScript returns the script and script version associated with the provided -// previous outpoint along with a bool that indicates whether or not the -// requested entry exists. This ensures the caller is able to distinguish -// between missing entries and empty v0 scripts. -func (s scriptSource) PrevScript(prevOut *wire.OutPoint) (uint16, []byte, bool) { - entry, ok := s[*prevOut] - if !ok { - return 0, nil, false - } - return entry.version, entry.script, true -} - -// makeScriptSource creates a source of previous transaction scripts and their -// associated versions for the given block by using the transaction index in -// order to look up all inputs referenced by the transactions in the block. -// This is sometimes needed when catching indexes up because many of the txouts -// could actually already be spent however the associated scripts are still -// required to index them. -func makeScriptSource(dbTx database.Tx, block *dcrutil.Block, interrupt <-chan struct{}) (scriptSource, error) { - source := make(scriptSource) - processTxns := func(txns []*dcrutil.Tx, regularTree bool) error { - for txIdx, tx := range txns { - // Coinbases do not reference any inputs. Since the block is - // required to have already gone through full validation, it has - // already been proven on the first transaction in the block is a - // coinbase. - if regularTree && txIdx == 0 { - continue - } - msgTx := tx.MsgTx() - isVote := !regularTree && stake.IsSSGen(msgTx) - - // Use the transaction index to load all of the referenced inputs - // and add their outputs to the view. - for txInIdx, txIn := range msgTx.TxIn { - // Ignore stakebase since it has no input. - if isVote && txInIdx == 0 { - continue - } - - // Skip already fetched outputs. - prevOut := &txIn.PreviousOutPoint - if _, ok := source[*prevOut]; ok { - continue - } - - originTx, err := dbFetchTx(dbTx, &prevOut.Hash) - if err != nil { - return err - } - - if prevOut.Index >= uint32(len(originTx.TxOut)) { - continue - } - - prevTxOut := originTx.TxOut[prevOut.Index] - source[*prevOut] = scriptSourceEntry{ - version: prevTxOut.Version, - script: prevTxOut.PkScript, - } - } - - if interruptRequested(interrupt) { - return errInterruptRequested - } - } - - return nil - } - - if err := processTxns(block.STransactions(), false); err != nil { - return nil, err - } - if err := processTxns(block.Transactions(), true); err != nil { - return nil, err - } - - return source, nil -} - // ConnectBlock must be invoked when a block is extending the main chain. It // keeps track of the state of each index it is managing, performs some sanity // checks, and invokes each indexer.