|
|
@ -27,43 +27,43 @@ return true
|
|
|
|
`
|
|
|
|
`
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// ErrTooLargeOffset indicates the offset is too large in bitset.
|
|
|
|
var ErrTooLargeOffset = errors.New("too large offset")
|
|
|
|
var ErrTooLargeOffset = errors.New("too large offset")
|
|
|
|
|
|
|
|
|
|
|
|
type (
|
|
|
|
type (
|
|
|
|
BitSetProvider interface {
|
|
|
|
// A Filter is a bloom filter.
|
|
|
|
check([]uint) (bool, error)
|
|
|
|
Filter struct {
|
|
|
|
set([]uint) error
|
|
|
|
bits uint
|
|
|
|
|
|
|
|
bitSet bitSetProvider
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
BloomFilter struct {
|
|
|
|
bitSetProvider interface {
|
|
|
|
bits uint
|
|
|
|
check([]uint) (bool, error)
|
|
|
|
bitSet BitSetProvider
|
|
|
|
set([]uint) error
|
|
|
|
}
|
|
|
|
}
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
// New create a BloomFilter, store is the backed redis, key is the key for the bloom filter,
|
|
|
|
// New create a Filter, store is the backed redis, key is the key for the bloom filter,
|
|
|
|
// bits is how many bits will be used, maps is how many hashes for each addition.
|
|
|
|
// bits is how many bits will be used, maps is how many hashes for each addition.
|
|
|
|
// best practices:
|
|
|
|
// best practices:
|
|
|
|
// elements - means how many actual elements
|
|
|
|
// elements - means how many actual elements
|
|
|
|
// when maps = 14, formula: 0.7*(bits/maps), bits = 20*elements, the error rate is 0.000067 < 1e-4
|
|
|
|
// when maps = 14, formula: 0.7*(bits/maps), bits = 20*elements, the error rate is 0.000067 < 1e-4
|
|
|
|
// for detailed error rate table, see http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
|
|
|
|
// for detailed error rate table, see http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
|
|
|
|
func New(store *redis.Redis, key string, bits uint) *BloomFilter {
|
|
|
|
func New(store *redis.Redis, key string, bits uint) *Filter {
|
|
|
|
return &BloomFilter{
|
|
|
|
return &Filter{
|
|
|
|
bits: bits,
|
|
|
|
bits: bits,
|
|
|
|
bitSet: newRedisBitSet(store, key, bits),
|
|
|
|
bitSet: newRedisBitSet(store, key, bits),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (f *BloomFilter) Add(data []byte) error {
|
|
|
|
// Add adds data into f.
|
|
|
|
|
|
|
|
func (f *Filter) Add(data []byte) error {
|
|
|
|
locations := f.getLocations(data)
|
|
|
|
locations := f.getLocations(data)
|
|
|
|
err := f.bitSet.set(locations)
|
|
|
|
return f.bitSet.set(locations)
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (f *BloomFilter) Exists(data []byte) (bool, error) {
|
|
|
|
// Exists checks if data is in f.
|
|
|
|
|
|
|
|
func (f *Filter) Exists(data []byte) (bool, error) {
|
|
|
|
locations := f.getLocations(data)
|
|
|
|
locations := f.getLocations(data)
|
|
|
|
isSet, err := f.bitSet.check(locations)
|
|
|
|
isSet, err := f.bitSet.check(locations)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
@ -76,7 +76,7 @@ func (f *BloomFilter) Exists(data []byte) (bool, error) {
|
|
|
|
return true, nil
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (f *BloomFilter) getLocations(data []byte) []uint {
|
|
|
|
func (f *Filter) getLocations(data []byte) []uint {
|
|
|
|
locations := make([]uint, maps)
|
|
|
|
locations := make([]uint, maps)
|
|
|
|
for i := uint(0); i < maps; i++ {
|
|
|
|
for i := uint(0); i < maps; i++ {
|
|
|
|
hashValue := hash.Hash(append(data, byte(i)))
|
|
|
|
hashValue := hash.Hash(append(data, byte(i)))
|
|
|
|