languages/go
languages/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-09-05 16:04:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

runtime,hash/maphash: reuse hashSets to save memory pressure

Browse source 
Might help with OOMs on 32-bit platforms

Change-Id: Idd5129c61ecdfeedd5a9a18fce85dbba27cab946
Reviewed-on: https://go-review.googlesource.com/c/go/+/574475
Reviewed-by: Emmanuel Odeke <emmanuel@orijtech.com>
Reviewed-by: Keith Randall <khr@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Than McIntosh <thanm@google.com>
This commit is contained in:
Keith Randall 2024-03-26 09:23:16 -07:00 committed by Keith Randall
parent d3e5e9fdf6
commit b47f2febea
2 changed files with 60 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
src/hash/maphash/smhasher_test.go
View file

@ -112,6 +112,8 @@ func (s *hashSet) check(t *testing.T) {
if float64(collisions) > expected+SLOP*(3*stddev+1) {
t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f", collisions, expected, stddev)
}
// Reset for reuse
s.list = s.list[:0]
}
// a string plus adding zeros must make distinct hashes
@ -212,8 +214,8 @@ func TestSmhasherCyclic(t *testing.T) {
r := rand.New(rand.NewSource(1234))
const REPEAT = 8
const N = 1000000
h := newHashSet()
for n := 4; n <= 12; n++ {
h := newHashSet()
b := make([]byte, REPEAT*n)
for i := 0; i < N; i++ {
b[0] = byte(i * 79 % 97)
@ -238,18 +240,18 @@ func TestSmhasherSparse(t *testing.T) {
if testing.Short() {
t.Skip("Skipping in short mode")
}
sparse(t, 32, 6)
sparse(t, 40, 6)
sparse(t, 48, 5)
sparse(t, 56, 5)
sparse(t, 64, 5)
sparse(t, 96, 4)
sparse(t, 256, 3)
sparse(t, 2048, 2)
}
func sparse(t *testing.T, n int, k int) {
b := make([]byte, n/8)
h := newHashSet()
sparse(t, h, 32, 6)
sparse(t, h, 40, 6)
sparse(t, h, 48, 5)
sparse(t, h, 56, 5)
sparse(t, h, 64, 5)
sparse(t, h, 96, 4)
sparse(t, h, 256, 3)
sparse(t, h, 2048, 2)
}
func sparse(t *testing.T, h *hashSet, n int, k int) {
b := make([]byte, n/8)
setbits(h, b, 0, k)
h.check(t)
}
@ -276,15 +278,15 @@ func TestSmhasherPermutation(t *testing.T) {
if testing.Short() {
t.Skip("Skipping in short mode")
}
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, []uint32{0, 1}, 20)
permutation(t, []uint32{0, 1 << 31}, 20)
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, s []uint32, n int) {
b := make([]byte, n*4)
h := newHashSet()
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, h, []uint32{0, 1}, 20)
permutation(t, h, []uint32{0, 1 << 31}, 20)
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, h *hashSet, s []uint32, n int) {
b := make([]byte, n*4)
genPerm(h, b, s, 0)
h.check(t)
}
@ -418,8 +420,8 @@ func windowed(t *testing.T, k key) {
}
const BITS = 16
h := newHashSet()
for r := 0; r < k.bits(); r++ {
h := newHashSet()
for i := 0; i < 1<<BITS; i++ {
k.clear()
for j := 0; j < BITS; j++ {
@ -438,18 +440,18 @@ func TestSmhasherText(t *testing.T) {
if testing.Short() {
t.Skip("Skipping in short mode")
}
text(t, "Foo", "Bar")
text(t, "FooBar", "")
text(t, "", "FooBar")
h := newHashSet()
text(t, h, "Foo", "Bar")
text(t, h, "FooBar", "")
text(t, h, "", "FooBar")
}
func text(t *testing.T, prefix, suffix string) {
func text(t *testing.T, h *hashSet, prefix, suffix string) {
const N = 4
const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
const L = len(S)
b := make([]byte, len(prefix)+N+len(suffix))
copy(b, prefix)
copy(b[len(prefix)+N:], suffix)
h := newHashSet()
c := b[len(prefix):]
for i := 0; i < L; i++ {
c[0] = S[i]

62
src/runtime/hash_test.go
View file

@ -121,6 +121,8 @@ func (s *HashSet) check(t *testing.T) {
if float64(collisions) > expected+SLOP*(3*stddev+1) {
t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f threshold=%f", collisions, expected, stddev, expected+SLOP*(3*stddev+1))
}
// Reset for reuse
s.list = s.list[:0]
}
// a string plus adding zeros must make distinct hashes
@ -230,8 +232,8 @@ func TestSmhasherCyclic(t *testing.T) {
r := rand.New(rand.NewSource(1234))
const REPEAT = 8
const N = 1000000
h := newHashSet()
for n := 4; n <= 12; n++ {
h := newHashSet()
b := make([]byte, REPEAT*n)
for i := 0; i < N; i++ {
b[0] = byte(i * 79 % 97)
@ -256,18 +258,18 @@ func TestSmhasherSparse(t *testing.T) {
if testing.Short() {
t.Skip("Skipping in short mode")
}
sparse(t, 32, 6)
sparse(t, 40, 6)
sparse(t, 48, 5)
sparse(t, 56, 5)
sparse(t, 64, 5)
sparse(t, 96, 4)
sparse(t, 256, 3)
sparse(t, 2048, 2)
}
func sparse(t *testing.T, n int, k int) {
b := make([]byte, n/8)
h := newHashSet()
sparse(t, h, 32, 6)
sparse(t, h, 40, 6)
sparse(t, h, 48, 5)
sparse(t, h, 56, 5)
sparse(t, h, 64, 5)
sparse(t, h, 96, 4)
sparse(t, h, 256, 3)
sparse(t, h, 2048, 2)
}
func sparse(t *testing.T, h *HashSet, n int, k int) {
b := make([]byte, n/8)
setbits(h, b, 0, k)
h.check(t)
}
@ -297,15 +299,15 @@ func TestSmhasherPermutation(t *testing.T) {
if race.Enabled {
t.Skip("Too long for race mode")
}
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, []uint32{0, 1}, 20)
permutation(t, []uint32{0, 1 << 31}, 20)
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, s []uint32, n int) {
b := make([]byte, n*4)
h := newHashSet()
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, h, []uint32{0, 1}, 20)
permutation(t, h, []uint32{0, 1 << 31}, 20)
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, h *HashSet, s []uint32, n int) {
b := make([]byte, n*4)
genPerm(h, b, s, 0)
h.check(t)
}
@ -542,14 +544,15 @@ func TestSmhasherWindowed(t *testing.T) {
if race.Enabled {
t.Skip("Too long for race mode")
}
h := newHashSet()
t.Logf("32 bit keys")
windowed(t, &Int32Key{})
windowed(t, h, &Int32Key{})
t.Logf("64 bit keys")
windowed(t, &Int64Key{})
windowed(t, h, &Int64Key{})
t.Logf("string keys")
windowed(t, &BytesKey{make([]byte, 128)})
windowed(t, h, &BytesKey{make([]byte, 128)})
}
func windowed(t *testing.T, k Key) {
func windowed(t *testing.T, h *HashSet, k Key) {
if GOARCH == "wasm" {
t.Skip("Too slow on wasm")
}
@ -566,7 +569,6 @@ func windowed(t *testing.T, k Key) {
const BITS = 16
for r := 0; r < k.bits(); r++ {
h := newHashSet()
for i := 0; i < 1<<BITS; i++ {
k.clear()
for j := 0; j < BITS; j++ {
@ -585,18 +587,18 @@ func TestSmhasherText(t *testing.T) {
if testing.Short() {
t.Skip("Skipping in short mode")
}
text(t, "Foo", "Bar")
text(t, "FooBar", "")
text(t, "", "FooBar")
h := newHashSet()
text(t, h, "Foo", "Bar")
text(t, h, "FooBar", "")
text(t, h, "", "FooBar")
}
func text(t *testing.T, prefix, suffix string) {
func text(t *testing.T, h *HashSet, prefix, suffix string) {
const N = 4
const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
const L = len(S)
b := make([]byte, len(prefix)+N+len(suffix))
copy(b, prefix)
copy(b[len(prefix)+N:], suffix)
h := newHashSet()
c := b[len(prefix):]
for i := 0; i < L; i++ {
c[0] = S[i]