From 5d6b7fcaa1444f6c17d519c9ce7bc0771bfd96ec Mon Sep 17 00:00:00 2001 From: Hugues Bruant Date: Tue, 14 Mar 2017 11:11:28 -0700 Subject: [PATCH] runtime: add mapdelete_fast* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add benchmarks for map delete with int32/int64/string key Benchmark results on darwin/amd64 name old time/op new time/op delta MapDelete/Int32/1-8 151ns ± 8% 99ns ± 3% -34.39% (p=0.008 n=5+5) MapDelete/Int32/2-8 128ns ± 2% 111ns ±15% -13.40% (p=0.040 n=5+5) MapDelete/Int32/4-8 128ns ± 5% 114ns ± 2% -10.82% (p=0.008 n=5+5) MapDelete/Int64/1-8 144ns ± 0% 104ns ± 3% -27.53% (p=0.016 n=4+5) MapDelete/Int64/2-8 153ns ± 1% 126ns ± 3% -17.17% (p=0.008 n=5+5) MapDelete/Int64/4-8 178ns ± 3% 136ns ± 2% -23.60% (p=0.008 n=5+5) MapDelete/Str/1-8 187ns ± 3% 171ns ± 3% -8.54% (p=0.008 n=5+5) MapDelete/Str/2-8 221ns ± 3% 206ns ± 4% -7.18% (p=0.016 n=5+4) MapDelete/Str/4-8 256ns ± 5% 232ns ± 2% -9.36% (p=0.016 n=4+5) name old time/op new time/op delta BinaryTree17-8 2.78s ± 7% 2.70s ± 1% ~ (p=0.151 n=5+5) Fannkuch11-8 3.21s ± 2% 3.19s ± 1% ~ (p=0.310 n=5+5) FmtFprintfEmpty-8 49.1ns ± 3% 50.2ns ± 2% ~ (p=0.095 n=5+5) FmtFprintfString-8 78.6ns ± 4% 80.2ns ± 5% ~ (p=0.460 n=5+5) FmtFprintfInt-8 79.7ns ± 1% 81.0ns ± 3% ~ (p=0.103 n=5+5) FmtFprintfIntInt-8 117ns ± 2% 119ns ± 0% ~ (p=0.079 n=5+4) FmtFprintfPrefixedInt-8 153ns ± 1% 146ns ± 3% -4.19% (p=0.024 n=5+5) FmtFprintfFloat-8 239ns ± 1% 237ns ± 1% ~ (p=0.246 n=5+5) FmtManyArgs-8 506ns ± 2% 509ns ± 2% ~ (p=0.238 n=5+5) GobDecode-8 7.06ms ± 4% 6.86ms ± 1% ~ (p=0.222 n=5+5) GobEncode-8 6.01ms ± 5% 5.87ms ± 2% ~ (p=0.222 n=5+5) Gzip-8 246ms ± 4% 236ms ± 1% -4.12% (p=0.008 n=5+5) Gunzip-8 37.7ms ± 4% 37.3ms ± 1% ~ (p=0.841 n=5+5) HTTPClientServer-8 64.9µs ± 1% 64.4µs ± 0% -0.80% (p=0.032 n=5+4) JSONEncode-8 16.0ms ± 2% 16.2ms ±11% ~ (p=0.548 n=5+5) JSONDecode-8 53.2ms ± 2% 53.1ms ± 4% ~ (p=1.000 n=5+5) Mandelbrot200-8 4.33ms ± 2% 4.32ms ± 2% ~ (p=0.841 n=5+5) GoParse-8 3.24ms ± 2% 3.27ms ± 4% ~ (p=0.690 n=5+5) RegexpMatchEasy0_32-8 86.2ns ± 1% 85.2ns ± 3% ~ (p=0.286 n=5+5) RegexpMatchEasy0_1K-8 198ns ± 2% 199ns ± 1% ~ (p=0.310 n=5+5) RegexpMatchEasy1_32-8 82.6ns ± 2% 81.8ns ± 1% ~ (p=0.294 n=5+5) RegexpMatchEasy1_1K-8 359ns ± 2% 354ns ± 1% -1.39% (p=0.048 n=5+5) RegexpMatchMedium_32-8 123ns ± 2% 123ns ± 1% ~ (p=0.905 n=5+5) RegexpMatchMedium_1K-8 38.2µs ± 2% 38.6µs ± 8% ~ (p=0.690 n=5+5) RegexpMatchHard_32-8 1.92µs ± 2% 1.91µs ± 5% ~ (p=0.460 n=5+5) RegexpMatchHard_1K-8 57.6µs ± 1% 57.0µs ± 2% ~ (p=0.310 n=5+5) Revcomp-8 483ms ± 7% 441ms ± 1% -8.79% (p=0.016 n=5+4) Template-8 58.0ms ± 1% 58.2ms ± 7% ~ (p=0.310 n=5+5) TimeParse-8 324ns ± 6% 312ns ± 2% ~ (p=0.087 n=5+5) TimeFormat-8 330ns ± 1% 329ns ± 1% ~ (p=0.968 n=5+5) name old speed new speed delta GobDecode-8 109MB/s ± 4% 112MB/s ± 1% ~ (p=0.222 n=5+5) GobEncode-8 128MB/s ± 5% 131MB/s ± 2% ~ (p=0.222 n=5+5) Gzip-8 78.9MB/s ± 4% 82.3MB/s ± 1% +4.25% (p=0.008 n=5+5) Gunzip-8 514MB/s ± 4% 521MB/s ± 1% ~ (p=0.841 n=5+5) JSONEncode-8 121MB/s ± 2% 120MB/s ±10% ~ (p=0.548 n=5+5) JSONDecode-8 36.5MB/s ± 2% 36.6MB/s ± 4% ~ (p=1.000 n=5+5) GoParse-8 17.9MB/s ± 2% 17.7MB/s ± 4% ~ (p=0.730 n=5+5) RegexpMatchEasy0_32-8 371MB/s ± 1% 375MB/s ± 3% ~ (p=0.310 n=5+5) RegexpMatchEasy0_1K-8 5.15GB/s ± 1% 5.13GB/s ± 1% ~ (p=0.548 n=5+5) RegexpMatchEasy1_32-8 387MB/s ± 2% 391MB/s ± 1% ~ (p=0.310 n=5+5) RegexpMatchEasy1_1K-8 2.85GB/s ± 2% 2.89GB/s ± 1% ~ (p=0.056 n=5+5) RegexpMatchMedium_32-8 8.07MB/s ± 2% 8.06MB/s ± 1% ~ (p=0.730 n=5+5) RegexpMatchMedium_1K-8 26.8MB/s ± 2% 26.6MB/s ± 7% ~ (p=0.690 n=5+5) RegexpMatchHard_32-8 16.7MB/s ± 2% 16.7MB/s ± 5% ~ (p=0.421 n=5+5) RegexpMatchHard_1K-8 17.8MB/s ± 1% 18.0MB/s ± 2% ~ (p=0.310 n=5+5) Revcomp-8 527MB/s ± 6% 577MB/s ± 1% +9.44% (p=0.016 n=5+4) Template-8 33.5MB/s ± 1% 33.4MB/s ± 7% ~ (p=0.310 n=5+5) Updates #19495 Change-Id: Ib9ece1690813d9b4788455db43d30891e2138df5 Reviewed-on: https://go-review.googlesource.com/38172 Reviewed-by: Hugues Bruant Reviewed-by: Keith Randall Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/builtin.go | 184 +++++++++--------- .../compile/internal/gc/builtin/runtime.go | 3 + src/cmd/compile/internal/gc/order.go | 38 ++-- src/cmd/compile/internal/gc/walk.go | 79 ++++---- src/runtime/hashmap_fast.go | 169 ++++++++++++++++ src/runtime/map_test.go | 76 ++++++-- test/live.go | 10 +- 7 files changed, 390 insertions(+), 169 deletions(-) diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index 5f65d8135a..675de836ce 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -88,61 +88,64 @@ var runtimeDecls = [...]struct { {"mapassign_faststr", funcTag, 61}, {"mapiterinit", funcTag, 66}, {"mapdelete", funcTag, 66}, - {"mapiternext", funcTag, 67}, - {"makechan", funcTag, 69}, - {"chanrecv1", funcTag, 71}, - {"chanrecv2", funcTag, 72}, - {"chansend1", funcTag, 74}, + {"mapdelete_fast32", funcTag, 67}, + {"mapdelete_fast64", funcTag, 67}, + {"mapdelete_faststr", funcTag, 67}, + {"mapiternext", funcTag, 68}, + {"makechan", funcTag, 70}, + {"chanrecv1", funcTag, 72}, + {"chanrecv2", funcTag, 73}, + {"chansend1", funcTag, 75}, {"closechan", funcTag, 23}, - {"writeBarrier", varTag, 76}, - {"writebarrierptr", funcTag, 77}, - {"typedmemmove", funcTag, 78}, - {"typedmemclr", funcTag, 79}, - {"typedslicecopy", funcTag, 80}, - {"selectnbsend", funcTag, 81}, - {"selectnbrecv", funcTag, 82}, - {"selectnbrecv2", funcTag, 84}, - {"newselect", funcTag, 85}, - {"selectsend", funcTag, 74}, - {"selectrecv", funcTag, 86}, + {"writeBarrier", varTag, 77}, + {"writebarrierptr", funcTag, 78}, + {"typedmemmove", funcTag, 79}, + {"typedmemclr", funcTag, 80}, + {"typedslicecopy", funcTag, 81}, + {"selectnbsend", funcTag, 82}, + {"selectnbrecv", funcTag, 83}, + {"selectnbrecv2", funcTag, 85}, + {"newselect", funcTag, 86}, + {"selectsend", funcTag, 75}, + {"selectrecv", funcTag, 87}, {"selectdefault", funcTag, 56}, - {"selectgo", funcTag, 87}, + {"selectgo", funcTag, 88}, {"block", funcTag, 5}, - {"makeslice", funcTag, 89}, - {"makeslice64", funcTag, 90}, - {"growslice", funcTag, 91}, - {"memmove", funcTag, 92}, - {"memclrNoHeapPointers", funcTag, 94}, - {"memclrHasPointers", funcTag, 94}, - {"memequal", funcTag, 95}, - {"memequal8", funcTag, 96}, - {"memequal16", funcTag, 96}, - {"memequal32", funcTag, 96}, - {"memequal64", funcTag, 96}, - {"memequal128", funcTag, 96}, - {"int64div", funcTag, 97}, - {"uint64div", funcTag, 98}, - {"int64mod", funcTag, 97}, - {"uint64mod", funcTag, 98}, - {"float64toint64", funcTag, 99}, - {"float64touint64", funcTag, 100}, - {"float64touint32", funcTag, 102}, - {"int64tofloat64", funcTag, 103}, - {"uint64tofloat64", funcTag, 104}, - {"uint32tofloat64", funcTag, 105}, - {"complex128div", funcTag, 106}, - {"racefuncenter", funcTag, 107}, + {"makeslice", funcTag, 90}, + {"makeslice64", funcTag, 91}, + {"growslice", funcTag, 92}, + {"memmove", funcTag, 93}, + {"memclrNoHeapPointers", funcTag, 95}, + {"memclrHasPointers", funcTag, 95}, + {"memequal", funcTag, 96}, + {"memequal8", funcTag, 97}, + {"memequal16", funcTag, 97}, + {"memequal32", funcTag, 97}, + {"memequal64", funcTag, 97}, + {"memequal128", funcTag, 97}, + {"int64div", funcTag, 98}, + {"uint64div", funcTag, 99}, + {"int64mod", funcTag, 98}, + {"uint64mod", funcTag, 99}, + {"float64toint64", funcTag, 100}, + {"float64touint64", funcTag, 101}, + {"float64touint32", funcTag, 103}, + {"int64tofloat64", funcTag, 104}, + {"uint64tofloat64", funcTag, 105}, + {"uint32tofloat64", funcTag, 106}, + {"complex128div", funcTag, 107}, + {"racefuncenter", funcTag, 108}, {"racefuncexit", funcTag, 5}, - {"raceread", funcTag, 107}, - {"racewrite", funcTag, 107}, - {"racereadrange", funcTag, 108}, - {"racewriterange", funcTag, 108}, - {"msanread", funcTag, 108}, - {"msanwrite", funcTag, 108}, + {"raceread", funcTag, 108}, + {"racewrite", funcTag, 108}, + {"racereadrange", funcTag, 109}, + {"racewriterange", funcTag, 109}, + {"msanread", funcTag, 109}, + {"msanwrite", funcTag, 109}, } func runtimeTypes() []*Type { - var typs [109]*Type + var typs [110]*Type typs[0] = bytetype typs[1] = typPtr(typs[0]) typs[2] = Types[TANY] @@ -210,47 +213,48 @@ func runtimeTypes() []*Type { typs[64] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[58]), anonfield(typs[2])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) typs[65] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[58]), anonfield(typs[3]), anonfield(typs[1])}, []*Node{anonfield(typs[3]), anonfield(typs[11])}) typs[66] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[58]), anonfield(typs[3])}, nil) - typs[67] = functype(nil, []*Node{anonfield(typs[3])}, nil) - typs[68] = typChan(typs[2], Cboth) - typs[69] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[68])}) - typs[70] = typChan(typs[2], Crecv) - typs[71] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[70]), anonfield(typs[3])}, nil) - typs[72] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[70]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) - typs[73] = typChan(typs[2], Csend) - typs[74] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[73]), anonfield(typs[3])}, nil) - typs[75] = typArray(typs[0], 3) - typs[76] = tostruct([]*Node{namedfield("enabled", typs[11]), namedfield("pad", typs[75]), namedfield("needed", typs[11]), namedfield("cgo", typs[11]), namedfield("alignme", typs[17])}) - typs[77] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[2])}, nil) - typs[78] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[3])}, nil) - typs[79] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3])}, nil) - typs[80] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2]), anonfield(typs[2])}, []*Node{anonfield(typs[32])}) - typs[81] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[73]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) - typs[82] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[70])}, []*Node{anonfield(typs[11])}) - typs[83] = typPtr(typs[11]) - typs[84] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[83]), anonfield(typs[70])}, []*Node{anonfield(typs[11])}) - typs[85] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[8])}, nil) - typs[86] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[70]), anonfield(typs[3]), anonfield(typs[83])}, nil) - typs[87] = functype(nil, []*Node{anonfield(typs[1])}, []*Node{anonfield(typs[32])}) - typs[88] = typSlice(typs[2]) - typs[89] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32]), anonfield(typs[32])}, []*Node{anonfield(typs[88])}) - typs[90] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[88])}) - typs[91] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[88]), anonfield(typs[32])}, []*Node{anonfield(typs[88])}) - typs[92] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[49])}, nil) - typs[93] = Types[TUNSAFEPTR] - typs[94] = functype(nil, []*Node{anonfield(typs[93]), anonfield(typs[49])}, nil) - typs[95] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[49])}, []*Node{anonfield(typs[11])}) - typs[96] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) - typs[97] = functype(nil, []*Node{anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[15])}) - typs[98] = functype(nil, []*Node{anonfield(typs[17]), anonfield(typs[17])}, []*Node{anonfield(typs[17])}) - typs[99] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[15])}) - typs[100] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[17])}) - typs[101] = Types[TUINT32] - typs[102] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[101])}) - typs[103] = functype(nil, []*Node{anonfield(typs[15])}, []*Node{anonfield(typs[13])}) - typs[104] = functype(nil, []*Node{anonfield(typs[17])}, []*Node{anonfield(typs[13])}) - typs[105] = functype(nil, []*Node{anonfield(typs[101])}, []*Node{anonfield(typs[13])}) - typs[106] = functype(nil, []*Node{anonfield(typs[19]), anonfield(typs[19])}, []*Node{anonfield(typs[19])}) - typs[107] = functype(nil, []*Node{anonfield(typs[49])}, nil) - typs[108] = functype(nil, []*Node{anonfield(typs[49]), anonfield(typs[49])}, nil) + typs[67] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[58]), anonfield(typs[2])}, nil) + typs[68] = functype(nil, []*Node{anonfield(typs[3])}, nil) + typs[69] = typChan(typs[2], Cboth) + typs[70] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[69])}) + typs[71] = typChan(typs[2], Crecv) + typs[72] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[71]), anonfield(typs[3])}, nil) + typs[73] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[71]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) + typs[74] = typChan(typs[2], Csend) + typs[75] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[74]), anonfield(typs[3])}, nil) + typs[76] = typArray(typs[0], 3) + typs[77] = tostruct([]*Node{namedfield("enabled", typs[11]), namedfield("pad", typs[76]), namedfield("needed", typs[11]), namedfield("cgo", typs[11]), namedfield("alignme", typs[17])}) + typs[78] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[2])}, nil) + typs[79] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[3])}, nil) + typs[80] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3])}, nil) + typs[81] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[2]), anonfield(typs[2])}, []*Node{anonfield(typs[32])}) + typs[82] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[74]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) + typs[83] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[71])}, []*Node{anonfield(typs[11])}) + typs[84] = typPtr(typs[11]) + typs[85] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[3]), anonfield(typs[84]), anonfield(typs[71])}, []*Node{anonfield(typs[11])}) + typs[86] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[8])}, nil) + typs[87] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[71]), anonfield(typs[3]), anonfield(typs[84])}, nil) + typs[88] = functype(nil, []*Node{anonfield(typs[1])}, []*Node{anonfield(typs[32])}) + typs[89] = typSlice(typs[2]) + typs[90] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[32]), anonfield(typs[32])}, []*Node{anonfield(typs[89])}) + typs[91] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[89])}) + typs[92] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[89]), anonfield(typs[32])}, []*Node{anonfield(typs[89])}) + typs[93] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[49])}, nil) + typs[94] = Types[TUNSAFEPTR] + typs[95] = functype(nil, []*Node{anonfield(typs[94]), anonfield(typs[49])}, nil) + typs[96] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[49])}, []*Node{anonfield(typs[11])}) + typs[97] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[11])}) + typs[98] = functype(nil, []*Node{anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[15])}) + typs[99] = functype(nil, []*Node{anonfield(typs[17]), anonfield(typs[17])}, []*Node{anonfield(typs[17])}) + typs[100] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[15])}) + typs[101] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[17])}) + typs[102] = Types[TUINT32] + typs[103] = functype(nil, []*Node{anonfield(typs[13])}, []*Node{anonfield(typs[102])}) + typs[104] = functype(nil, []*Node{anonfield(typs[15])}, []*Node{anonfield(typs[13])}) + typs[105] = functype(nil, []*Node{anonfield(typs[17])}, []*Node{anonfield(typs[13])}) + typs[106] = functype(nil, []*Node{anonfield(typs[102])}, []*Node{anonfield(typs[13])}) + typs[107] = functype(nil, []*Node{anonfield(typs[19]), anonfield(typs[19])}, []*Node{anonfield(typs[19])}) + typs[108] = functype(nil, []*Node{anonfield(typs[49])}, nil) + typs[109] = functype(nil, []*Node{anonfield(typs[49]), anonfield(typs[49])}, nil) return typs[:] } diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index cec0425947..168aaaf6f4 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -108,6 +108,9 @@ func mapassign_fast64(mapType *byte, hmap map[any]any, key any) (val *any) func mapassign_faststr(mapType *byte, hmap map[any]any, key any) (val *any) func mapiterinit(mapType *byte, hmap map[any]any, hiter *any) func mapdelete(mapType *byte, hmap map[any]any, key *any) +func mapdelete_fast32(mapType *byte, hmap map[any]any, key any) +func mapdelete_fast64(mapType *byte, hmap map[any]any, key any) +func mapdelete_faststr(mapType *byte, hmap map[any]any, key any) func mapiternext(hiter *any) // *byte is really *runtime.Type diff --git a/src/cmd/compile/internal/gc/order.go b/src/cmd/compile/internal/gc/order.go index c15e9084e3..e6032c33d0 100644 --- a/src/cmd/compile/internal/gc/order.go +++ b/src/cmd/compile/internal/gc/order.go @@ -206,23 +206,15 @@ func orderaddrtemp(n *Node, order *Order) *Node { return ordercopyexpr(n, n.Type, order, 0) } -// ordermapkeytemp prepares n.Right to be a key in a map runtime call. -func ordermapkeytemp(n *Node, order *Order) { +// ordermapkeytemp prepares n to be a key in a map runtime call and returns n. +// It should only be used for map runtime calls which have *_fast* versions. +func ordermapkeytemp(t *Type, n *Node, order *Order) *Node { // Most map calls need to take the address of the key. - // Exception: map(accessN|assign)_fast* calls. See golang.org/issue/19015. - var p string - switch n.Etype { - case 0: // n is an rvalue - p, _ = mapaccessfast(n.Left.Type) - case 1: // n is an lvalue - p = mapassignfast(n.Left.Type) - default: - Fatalf("unexpected node type: %+v", n) + // Exception: map*_fast* calls. See golang.org/issue/19015. + if mapfast(t) == mapslow { + return orderaddrtemp(n, order) } - if p != "" { - return - } - n.Right = orderaddrtemp(n.Right, order) + return n } type ordermarker int @@ -560,7 +552,7 @@ func orderstmt(n *Node, order *Order) { if r.Right.Op == OARRAYBYTESTR { r.Right.Op = OARRAYBYTESTRTMP } - ordermapkeytemp(r, order) + r.Right = ordermapkeytemp(r.Left.Type, r.Right, order) orderokas2(n, order) cleantemp(t, order) @@ -640,10 +632,12 @@ func orderstmt(n *Node, order *Order) { case ODELETE: orderexprlist(n.Left.List, order) - t1 := marktemp(order) - np := n.Left.List.Addr(1) // map key - *np = ordercopyexpr(*np, (*np).Type, order, 0) - poptemp(t1, order) + if mapfast(n.Left.List.First().Type) == mapslow { + t1 := marktemp(order) + np := n.Left.List.Addr(1) // map key + *np = ordercopyexpr(*np, (*np).Type, order, 0) + poptemp(t1, order) + } default: ordercall(n.Left, order) @@ -656,7 +650,7 @@ func orderstmt(n *Node, order *Order) { t := marktemp(order) n.List.SetFirst(orderexpr(n.List.First(), order, nil)) n.List.SetSecond(orderexpr(n.List.Second(), order, nil)) - n.List.SetSecond(orderaddrtemp(n.List.Second(), order)) // map key + n.List.SetSecond(ordermapkeytemp(n.List.First().Type, n.List.Second(), order)) order.out = append(order.out, n) cleantemp(t, order) @@ -1069,7 +1063,7 @@ func orderexpr(n *Node, order *Order, lhs *Node) *Node { needCopy = true } - ordermapkeytemp(n, order) + n.Right = ordermapkeytemp(n.Left.Type, n.Right, order) if needCopy { n = ordercopyexpr(n, n.Type, order, 0) } diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 96b564df7c..e21816653b 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -804,16 +804,15 @@ opswitch: r.Right = walkexpr(r.Right, init) t := r.Left.Type - _, p := mapaccessfast(t) + fast := mapfast(t) var key *Node - if p != "" { + if fast != mapslow { // fast versions take key by value key = r.Right } else { // standard version takes key by reference // orderexpr made sure key is addressable. key = nod(OADDR, r.Right, nil) - p = "mapaccess2" } // from: @@ -824,7 +823,7 @@ opswitch: a := n.List.First() if w := t.Val().Width; w <= 1024 { // 1024 must match ../../../../runtime/hashmap.go:maxZero - fn := mapfn(p, t) + fn := mapfn(mapaccess2[fast], t) r = mkcall1(fn, fn.Type.Results(), init, typename(t), r.Left, key) } else { fn := mapfn("mapaccess2_fat", t) @@ -862,11 +861,13 @@ opswitch: map_ = walkexpr(map_, init) key = walkexpr(key, init) - // orderstmt made sure key is addressable. - key = nod(OADDR, key, nil) - t := map_.Type - n = mkcall1(mapfndel("mapdelete", t), nil, init, typename(t), map_, key) + fast := mapfast(t) + if fast == mapslow { + // orderstmt made sure key is addressable. + key = nod(OADDR, key, nil) + } + n = mkcall1(mapfndel(mapdelete[fast], t), nil, init, typename(t), map_, key) case OAS2DOTTYPE: walkexprlistsafe(n.List.Slice(), init) @@ -1184,30 +1185,27 @@ opswitch: t := map_.Type if n.Etype == 1 { // This m[k] expression is on the left-hand side of an assignment. - p := mapassignfast(t) - if p == "" { + fast := mapfast(t) + if fast == mapslow { // standard version takes key by reference. // orderexpr made sure key is addressable. key = nod(OADDR, key, nil) - p = "mapassign" } - n = mkcall1(mapfn(p, t), nil, init, typename(t), map_, key) + n = mkcall1(mapfn(mapassign[fast], t), nil, init, typename(t), map_, key) } else { // m[k] is not the target of an assignment. - p, _ := mapaccessfast(t) - if p == "" { + fast := mapfast(t) + if fast == mapslow { // standard version takes key by reference. // orderexpr made sure key is addressable. key = nod(OADDR, key, nil) - p = "mapaccess1" } if w := t.Val().Width; w <= 1024 { // 1024 must match ../../../../runtime/hashmap.go:maxZero - n = mkcall1(mapfn(p, t), typPtr(t.Val()), init, typename(t), map_, key) + n = mkcall1(mapfn(mapaccess1[fast], t), typPtr(t.Val()), init, typename(t), map_, key) } else { - p = "mapaccess1_fat" z := zeroaddr(w) - n = mkcall1(mapfn(p, t), typPtr(t.Val()), init, typename(t), map_, key, z) + n = mkcall1(mapfn("mapaccess1_fat", t), typPtr(t.Val()), init, typename(t), map_, key, z) } } n.Type = typPtr(t.Val()) @@ -2633,38 +2631,39 @@ func mapfndel(name string, t *Type) *Node { return fn } -// mapaccessfast returns the name of the fast map access runtime routine for t. -func mapaccessfast(t *Type) (access1, access2 string) { - // Check ../../runtime/hashmap.go:maxValueSize before changing. - if t.Val().Width > 128 { - return "", "" - } - switch algtype(t.Key()) { - case AMEM32: - return "mapaccess1_fast32", "mapaccess2_fast32" - case AMEM64: - return "mapaccess1_fast64", "mapaccess2_fast64" - case ASTRING: - return "mapaccess1_faststr", "mapaccess2_faststr" - } - return "", "" +const ( + mapslow = iota + mapfast32 + mapfast64 + mapfaststr + nmapfast +) + +type mapnames [nmapfast]string + +func mkmapnames(base string) mapnames { + return mapnames{base, base + "_fast32", base + "_fast64", base + "_faststr"} } -// mapassignfast returns the name of the fast map assign runtime routine for t. -func mapassignfast(t *Type) (assign string) { +var mapaccess1 mapnames = mkmapnames("mapaccess1") +var mapaccess2 mapnames = mkmapnames("mapaccess2") +var mapassign mapnames = mkmapnames("mapassign") +var mapdelete mapnames = mkmapnames("mapdelete") + +func mapfast(t *Type) int { // Check ../../runtime/hashmap.go:maxValueSize before changing. if t.Val().Width > 128 { - return "" + return mapslow } switch algtype(t.Key()) { case AMEM32: - return "mapassign_fast32" + return mapfast32 case AMEM64: - return "mapassign_fast64" + return mapfast64 case ASTRING: - return "mapassign_faststr" + return mapfaststr } - return "" + return mapslow } func writebarrierfn(name string, l *Type, r *Type) *Node { diff --git a/src/runtime/hashmap_fast.go b/src/runtime/hashmap_fast.go index f1a5bf3fc3..0a625cca56 100644 --- a/src/runtime/hashmap_fast.go +++ b/src/runtime/hashmap_fast.go @@ -692,3 +692,172 @@ done: h.flags &^= hashWriting return val } + +func mapdelete_fast32(t *maptype, h *hmap, key uint32) { + if raceenabled && h != nil { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapdelete_fast32)) + } + if h == nil || h.count == 0 { + return + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + + hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapdelete + h.flags |= hashWriting + + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + continue + } + k := (*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)) + if key != *k { + continue + } + *k = 0 + v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*4 + i*uintptr(t.valuesize)) + typedmemclr(t.elem, v) + b.tophash[i] = empty + h.count-- + goto done + } + b = b.overflow(t) + if b == nil { + goto done + } + } + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting +} + +func mapdelete_fast64(t *maptype, h *hmap, key uint64) { + if raceenabled && h != nil { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapdelete_fast64)) + } + if h == nil || h.count == 0 { + return + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + + hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapdelete + h.flags |= hashWriting + + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + continue + } + k := (*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)) + if key != *k { + continue + } + *k = 0 + v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*8 + i*uintptr(t.valuesize)) + typedmemclr(t.elem, v) + b.tophash[i] = empty + h.count-- + goto done + } + b = b.overflow(t) + if b == nil { + goto done + } + } + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting +} + +func mapdelete_faststr(t *maptype, h *hmap, ky string) { + if raceenabled && h != nil { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapdelete_faststr)) + } + if h == nil || h.count == 0 { + return + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + + key := stringStructOf(&ky) + hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapdelete + h.flags |= hashWriting + + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + continue + } + k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize)) + if k.len != key.len { + continue + } + if k.str != key.str && !memequal(k.str, key.str, uintptr(key.len)) { + continue + } + typedmemclr(t.key, unsafe.Pointer(k)) + v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*2*sys.PtrSize + i*uintptr(t.valuesize)) + typedmemclr(t.elem, v) + b.tophash[i] = empty + h.count-- + goto done + } + b = b.overflow(t) + if b == nil { + goto done + } + } + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting +} diff --git a/src/runtime/map_test.go b/src/runtime/map_test.go index 8ec67d5ab0..45d14126c2 100644 --- a/src/runtime/map_test.go +++ b/src/runtime/map_test.go @@ -619,35 +619,85 @@ func TestNonEscapingMap(t *testing.T) { } } -func benchmarkMapAssignInt32(b *testing.B, pow uint) { +func benchmarkMapAssignInt32(b *testing.B, n int) { a := make(map[int32]int) for i := 0; i < b.N; i++ { - a[int32(i&((1<