From 952bd7bba34c8aeab86f63f9c9f69fc74ff1a7e1 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 21 Aug 2019 01:27:46 -0500 Subject: [PATCH] gcs: Support independent fp rate and bin size. This modifies the code to support an independent false positive rate and Golomb coding bin size. Among other things, this permits more optimal parameters for minimizing the filter size to be specified. This capability will be used in the upcoming version 2 filters that will ultimately be included in header commitments. For a concrete example, the current version 1 filter for block 89341 on mainnet contains 2470 items resulting in a full serialized size of 6,669 bytes. In contrast, if the optimal parameters were specified as described by the comments in this commit, with no other changes to the items included in the filter, that same filter would be 6,505 bytes, which is a size reduction of about 2.46%. This might not seem like a significant amount, but consider that there is a filter for every block, so it really adds up. Since the internal filter no longer directly has a P parameter, this moves the method to obtain it to the FilterV1 type and adds a new test to ensure it is returned properly. Additionally, all of the tests are converted to use the parameters while retaining the same effective parameters to help prove correctness of the new code. Finally, it also significantly reduces the number of allocations required to construct a filter resulting in faster filter construction and reduced pressure on the GC and does some other minor consistency cleanup while here. In terms of the reduction in allocations, the following is a before and after comparison of building filters with 50k and 100k elements: benchmark old ns/op new ns/op delta -------------------------------------------------------------- BenchmarkFilterBuild50000 18095111 15680001 -13.35% BenchmarkFilterBuild100000 31980156 31389892 -1.85% benchmark old allocs new allocs delta -------------------------------------------------------------- BenchmarkFilterBuild50000 31 6 -80.65% BenchmarkFilterBuild100000 34 6 -82.35% benchmark old bytes new bytes delta -------------------------------------------------------------- BenchmarkFilterBuild50000 1202343 688271 -42.76% BenchmarkFilterBuild100000 2488472 1360000 -45.35% --- gcs/gcs.go | 180 ++++++++++++++++++++++++++++++------------------ gcs/gcs_test.go | 90 +++++++++++++++++++----- 2 files changed, 185 insertions(+), 85 deletions(-) diff --git a/gcs/gcs.go b/gcs/gcs.go index 48569fcb..a73a5e0c 100644 --- a/gcs/gcs.go +++ b/gcs/gcs.go @@ -18,8 +18,6 @@ import ( "github.com/decred/dcrd/crypto/blake256" ) -// Inspired by https://github.com/rasky/gcs - // KeySize is the size of the byte array required for key material for the // SipHash keyed hash function. const KeySize = 16 @@ -40,34 +38,62 @@ func (s *uint64s) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] } type filter struct { version uint16 n uint32 - p uint8 - modulusNP uint64 + b uint8 + modulusNM uint64 filterNData []byte filterData []byte // Slice into filterNData with raw filter bytes. } -// newFilter builds a new GCS filter of the specified version with the collision -// probability of `1/(2**P)`, key `key`, and including every `[]byte` in `data` -// as a member of the set. -func newFilter(version uint16, P uint8, key [KeySize]byte, data [][]byte) (*filter, error) { - if len(data) > math.MaxInt32 { +// newFilter builds a new GCS filter with the specified version and provided +// tunable parameters that contains every item of the passed data as a member of +// the set. +// +// B is the tunable bits parameter for constructing the filter that is used as +// the bin size in the underlying Golomb coding with a value of 2^B. The +// optimal value of B to minimize the size of the filter for a given false +// positive rate 1/M is floor(log_2(M) - 0.055256). The maximum allowed value +// for B is 32. +// +// M is the inverse of the target false positive rate for the filter. The +// optimal value of M to minimize the size of the filter for a given B is +// ceil(1.497137 * 2^B). +// +// key is a key used in the SipHash function used to hash each data element +// prior to inclusion in the filter. This helps thwart would be attackers +// attempting to choose elements that intentionally cause false positives. +// +// The general process for determining optimal parameters for B and M to +// minimize the size of the filter is to start with the desired false positive +// rate and calculate B per the aforementioned formula accordingly. Then, if +// the application permits the false positive rate to be varied, calculate the +// optimal value of M via the formula provided under the description of M. +// +// NOTE: Since this function must only be used internally, it will panic if +// called with a value of B greater than 32. +func newFilter(version uint16, B uint8, M uint64, key [KeySize]byte, data [][]byte) (*filter, error) { + if B > 32 { + panic(fmt.Sprintf("B value of %d is greater than max allowed 32", B)) + } + switch version { + case 1: + default: + panic(fmt.Sprintf("version %d filters are not supported", version)) + } + + numEntries := uint64(len(data)) + if numEntries > math.MaxInt32 { str := fmt.Sprintf("unable to create filter with %d entries greater "+ "than max allowed %d", len(data), math.MaxInt32) return nil, makeError(ErrNTooBig, str) } - if P > 32 { - str := fmt.Sprintf("P value of %d is greater than max allowed 32", P) - return nil, makeError(ErrPTooBig, str) - } // Create the filter object and insert metadata. - modP := uint64(1 << P) - modPMask := modP - 1 + modBMask := uint64(1<>1) >> 3 + b.bytes = make([]byte, 0, sizeHint) - // Write the sorted list of values into the filter bitstream, - // compressing it using Golomb coding. - var value, lastValue, remainder uint64 + // Write the sorted list of values into the filter bitstream using Golomb + // coding. + var quotient, prevValue, remainder uint64 for _, v := range values { - // Calculate the difference between this value and the last, - // modulo P. - remainder = (v - lastValue) & modPMask + delta := v - prevValue + prevValue = v - // Calculate the difference between this value and the last, - // divided by P. - value = (v - lastValue - remainder) >> f.p - lastValue = v + // Calculate the remainder of the difference between this value and the + // previous when dividing by 2^B. + // + // r = d % 2^B + remainder = delta & modBMask - // Write the P multiple into the bitstream in unary; the - // average should be around 1 (2 bits - 0b10). - for value > 0 { + // Calculate the quotient of the difference between this value and the + // previous when dividing by 2^B. + // + // q = floor(d / 2^B) + quotient = (delta - remainder) >> f.b + + // Write the quotient into the bitstream in unary. The average value + // will be around 1 for reasonably optimal parameters (which is encoded + // as 2 bits - 0b10). + for quotient > 0 { b.writeOne() - value-- + quotient-- } b.writeZero() - // Write the remainder as a big-endian integer with enough bits - // to represent the appropriate collision probability. - b.writeNBits(remainder, uint(f.p)) + // Write the remainder into the bitstream as a big-endian integer with + // B bits. Note that Golomb coding typically uses truncated binary + // encoding in order to support arbitrary bin sizes, however, since 2^B + // is necessarily fixed to a power of 2, it is equivalent to a regular + // binary code. + b.writeNBits(remainder, uint(f.b)) } // Save the filter data internally as n + filter bytes @@ -127,38 +170,31 @@ func newFilter(version uint16, P uint8, key [KeySize]byte, data [][]byte) (*filt } // Bytes returns the serialized format of the GCS filter which includes N, but -// does not include P (returned by a separate method) or the key used by -// SipHash. +// does not include other parameters such as the false positive rate or the key. func (f *filter) Bytes() []byte { return f.filterNData } -// P returns the filter's collision probability as a negative power of 2 (that -// is, a collision probability of `1/2**20` is represented as 20). -func (f *filter) P() uint8 { - return f.p -} - // N returns the size of the data set used to build the filter. func (f *filter) N() uint32 { return f.n } // readFullUint64 reads a value represented by the sum of a unary multiple of -// the filter's P modulus (`2**P`) and a big-endian P-bit remainder. +// the Golomb coding bin size (2^B) and a big-endian B-bit remainder. func (f *filter) readFullUint64(b *bitReader) (uint64, error) { v, err := b.readUnary() if err != nil { return 0, err } - rem, err := b.readNBits(uint(f.p)) + rem, err := b.readNBits(uint(f.b)) if err != nil { return 0, err } // Add the multiple and the remainder. - return v< 32 { + str := fmt.Sprintf("P value of %d is greater than max allowed 32", P) + return nil, makeError(ErrPTooBig, str) + } + + filter, err := newFilter(1, P, 1<