seedhammer/dcrd
seedhammer/dcrd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

indexers: Use spend journal for index catchup.

Browse Source 
When an index is enabled for the first time (or after having previously
been disabled and re-enabled), the index needs to be updated to account
for the current state of the main chain.

This poses a problem for indexes that require access to the previous
scripts, such as the address index, since the outputs are no longer in
the utxo set due to pruning.  Currently, in order to handle this, the
code has to load all of the referenced transactions in order to get
access to the output scripts which is extremely inefficient.  The
address index is already notoriously time consuming to generate due to
the sheer amount of information, but it is only exacerbated by this
inefficiency.

However, the necessary information exists in a much more efficient form
in the spend journal and the recently introduced interfaces which
decouple the script lookup from the concrete type provide a path to
significantly optimize the index catch up process.

Consequently, this modifies the indexer.ChainQueryer interface to
provide access to a source of previous transaction scripts and their
associated versions spent by the given block, updates the index
initialization to make use of it, and updates the blockchain
implementation of the interface to provide the required information via
the spend journal.

As an example of the speedup obtained by this, the following shows how
long it took to catch up the address index from block 0 to block 391500
on a 7200 RPM HDD:

before: 55 minutes, 25 seconds
 after: 34 minutes, 4 seconds

That amounts to about a 38.5% reduction.  It is also worth noting that
the speedup is not linear, so as the number of transactions continues to
the grow, the speedup will be increasingly more significant.
This commit is contained in:
Dave Collins 2019-10-26 15:37:13 -05:00
parent f13191ac48
commit 89ab143dcb
No known key found for this signature in database
GPG Key ID: B8904D9D9C93D1F2
3 changed files with 24 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
blockchain/chain.go
View File

@ -2039,6 +2039,24 @@ func (q *chainQueryerAdapter) BestHeight() int64 {
return q.BestSnapshot().Height
}
// PrevScripts returns a source of previous transaction scripts and their
// associated versions spent by the given block by using the spend journal.
//
// It is defined via a separate internal struct to avoid polluting the public
// API of the BlockChain type itself.
//
// This is part of the indexers.ChainQueryer interface.
func (q *chainQueryerAdapter) PrevScripts(dbTx database.Tx, block *dcrutil.Block) (indexers.PrevScripter, error) {
// Load all of the spent transaction output data from the database.
stxos, err := dbFetchSpendJournalEntry(dbTx, block)
if err != nil {
return nil, err
}
prevScripts := stxosToScriptSource(block, stxos, currentCompressionVersion)
return prevScripts, nil
}
// Config is a descriptor which specifies the blockchain instance configuration.
type Config struct {
// DB defines the database which houses the blocks and will be used to

4
blockchain/indexers/common.go
View File

@ -89,6 +89,10 @@ type ChainQueryer interface {
// BlockHashByHeight returns the hash of the block at the given height in
// the main chain.
BlockHashByHeight(int64) (*chainhash.Hash, error)
// PrevScripts returns a source of previous transaction scripts and their
// associated versions spent by the given block.
PrevScripts(database.Tx, *dcrutil.Block) (PrevScripter, error)
}
// IndexManager provides a generic interface that is called when blocks are

98
blockchain/indexers/manager.go
View File

@ -9,7 +9,6 @@ import (
"bytes"
"fmt"
"github.com/decred/dcrd/blockchain/stake/v3"
"github.com/decred/dcrd/blockchain/v3/internal/progresslog"
"github.com/decred/dcrd/chaincfg/chainhash"
"github.com/decred/dcrd/chaincfg/v2"
@ -468,8 +467,7 @@ func (m *Manager) Init(chain ChainQueryer, interrupt <-chan struct{}) error {
var prevScripts PrevScripter
if indexNeedsInputs(indexer) {
var err error
prevScripts, err = makeScriptSource(dbTx, block,
interrupt)
prevScripts, err = chain.PrevScripts(dbTx, block)
if err != nil {
return err
}
@ -608,8 +606,7 @@ func (m *Manager) Init(chain ChainQueryer, interrupt <-chan struct{}) error {
// index.
if prevScripts == nil && indexNeedsInputs(indexer) {
var errMakeView error
prevScripts, errMakeView = makeScriptSource(dbTx, block,
interrupt)
prevScripts, errMakeView = chain.PrevScripts(dbTx, block)
if errMakeView != nil {
return errMakeView
}
@ -673,97 +670,6 @@ func dbFetchTx(dbTx database.Tx, hash *chainhash.Hash) (*wire.MsgTx, error) {
return &msgTx, nil
}
// scriptSourceEntry houses a script and its associated version.
type scriptSourceEntry struct {
version uint16
script []byte
}
// scriptSource provides a source of transaction output scripts and their
// associated script version for given outpoints and implements the PrevScripter
// interface so it may be used in cases that require access to said scripts.
type scriptSource map[wire.OutPoint]scriptSourceEntry
// PrevScript returns the script and script version associated with the provided
// previous outpoint along with a bool that indicates whether or not the
// requested entry exists. This ensures the caller is able to distinguish
// between missing entries and empty v0 scripts.
func (s scriptSource) PrevScript(prevOut *wire.OutPoint) (uint16, []byte, bool) {
entry, ok := s[*prevOut]
if !ok {
return 0, nil, false
}
return entry.version, entry.script, true
}
// makeScriptSource creates a source of previous transaction scripts and their
// associated versions for the given block by using the transaction index in
// order to look up all inputs referenced by the transactions in the block.
// This is sometimes needed when catching indexes up because many of the txouts
// could actually already be spent however the associated scripts are still
// required to index them.
func makeScriptSource(dbTx database.Tx, block *dcrutil.Block, interrupt <-chan struct{}) (scriptSource, error) {
source := make(scriptSource)
processTxns := func(txns []*dcrutil.Tx, regularTree bool) error {
for txIdx, tx := range txns {
// Coinbases do not reference any inputs. Since the block is
// required to have already gone through full validation, it has
// already been proven on the first transaction in the block is a
// coinbase.
if regularTree && txIdx == 0 {
continue
}
msgTx := tx.MsgTx()
isVote := !regularTree && stake.IsSSGen(msgTx)
// Use the transaction index to load all of the referenced inputs
// and add their outputs to the view.
for txInIdx, txIn := range msgTx.TxIn {
// Ignore stakebase since it has no input.
if isVote && txInIdx == 0 {
continue
}
// Skip already fetched outputs.
prevOut := &txIn.PreviousOutPoint
if _, ok := source[*prevOut]; ok {
continue
}
originTx, err := dbFetchTx(dbTx, &prevOut.Hash)
if err != nil {
return err
}
if prevOut.Index >= uint32(len(originTx.TxOut)) {
continue
}
prevTxOut := originTx.TxOut[prevOut.Index]
source[*prevOut] = scriptSourceEntry{
version: prevTxOut.Version,
script: prevTxOut.PkScript,
}
}
if interruptRequested(interrupt) {
return errInterruptRequested
}
}
return nil
}
if err := processTxns(block.STransactions(), false); err != nil {
return nil, err
}
if err := processTxns(block.Transactions(), true); err != nil {
return nil, err
}
return source, nil
}
// ConnectBlock must be invoked when a block is extending the main chain. It
// keeps track of the state of each index it is managing, performs some sanity
// checks, and invokes each indexer.