mirror of
https://github.com/golang/go
synced 2024-10-14 03:43:28 +00:00
runtime: move whenHeap into heap slice
This CL changes the timer heap from a slice of timers to a slice
of {timer, when} pairs, moving timer.whenHeap into the heap
backing store itself. This makes the ownership clearer (t.whenHeap
was always protected by the heap lock, not the timer's lock)
and also avoids an indirection when doing heap operations,
which look at the when field quite a lot.
goos: linux
goarch: amd64
pkg: time
cpu: AMD Ryzen 9 7950X 16-Core Processor
│ s7base.txt │ s7.txt │
│ sec/op │ sec/op vs base │
AdjustTimers10000-32 244.1µ ± 5% 195.9µ ± 13% -19.76% (p=0.001 n=10)
AdjustTimers10000SingleThread-32 1.674m ± 6% 1.573m ± 13% -6.03% (p=0.001 n=10)
AdjustTimers10000NoReset-32 194.0µ ± 2% 170.6µ ± 1% -12.06% (p=0.000 n=10)
AdjustTimers10000NoSleep-32 223.0µ ± 7% 183.9µ ± 2% -17.54% (p=0.000 n=10)
AdjustTimers10000NoResetNoSleep-32 181.1µ ± 1% 151.3µ ± 1% -16.45% (p=0.000 n=10)
goos: darwin
goarch: arm64
pkg: time
cpu: Apple M3 Pro
│ m3base.txt │ m3.txt │
│ sec/op │ sec/op vs base │
AdjustTimers10000-12 266.6µ ± 2% 234.2µ ± 1% -12.16% (p=0.000 n=10)
AdjustTimers10000SingleThread-12 1.194m ± 0% 1.191m ± 1% -0.33% (p=0.029 n=10)
AdjustTimers10000NoReset-12 260.9µ ± 2% 239.6µ ± 2% -8.15% (p=0.000 n=10)
AdjustTimers10000NoSleep-12 247.8µ ± 2% 223.3µ ± 2% -9.90% (p=0.000 n=10)
AdjustTimers10000NoResetNoSleep-12 231.7µ ± 2% 209.2µ ± 2% -9.68% (p=0.000 n=10)
Change-Id: I546f077068476d7a7b855889419b292525fb2bc5
Reviewed-on: https://go-review.googlesource.com/c/go/+/574896
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>
This commit is contained in:
Russ Cox
parent
bb523c9542
commit
bb7a29991c
|
|
@ -65,18 +65,6 @@ type timer struct {
|
|||
// If non-nil, the timers containing t.
|
||||
ts *timers
|
||||
|
||||
// whenHeap is a (perhaps outdated) copy of t.when for use
|
||||
// ordering t within t.ts.heap.
|
||||
// When t is in a heap but t.whenHeap is outdated,
|
||||
// the timerModified state bit is set.
|
||||
// The actual update t.whenHeap = t.when must be
|
||||
// delayed until the heap can be reordered at the same time
|
||||
// (meaning t's lock must be held for whenHeap,
|
||||
// and t.ts's lock must be held for the heap reordering).
|
||||
// Since writes to whenHeap are protected by two locks (t.mu and t.ts.mu),
|
||||
// it is permitted to read whenHeap when holding either one.
|
||||
whenHeap int64
|
||||
|
||||
// sendLock protects sends on the timer's channel.
|
||||
// Not used for async (pre-Go 1.23) behavior when debug.asynctimerchan.Load() != 0.
|
||||
sendLock mutex
|
||||
|
|
@ -98,9 +86,9 @@ type timers struct {
|
|||
// access the timers of another P, so we have to lock.
|
||||
mu mutex
|
||||
|
||||
// heap is the set of timers, ordered by t.whenHeap.
|
||||
// heap is the set of timers, ordered by heap[i].when.
|
||||
// Must hold lock to access.
|
||||
heap []*timer
|
||||
heap []timerWhen
|
||||
|
||||
// len is an atomic copy of len(heap).
|
||||
len atomic.Uint32
|
||||
|
|
@ -112,7 +100,7 @@ type timers struct {
|
|||
// raceCtx is the race context used while executing timer functions.
|
||||
raceCtx uintptr
|
||||
|
||||
// minWhenHeap is the minimum heap[i].whenHeap value (= heap[0].whenHeap).
|
||||
// minWhenHeap is the minimum heap[i].when value (= heap[0].when).
|
||||
// The wakeTime method uses minWhenHeap and minWhenModified
|
||||
// to determine the next wake time.
|
||||
// If minWhenHeap = 0, it means there are no timers in the heap.
|
||||
|
|
@ -124,6 +112,11 @@ type timers struct {
|
|||
minWhenModified atomic.Int64
|
||||
}
|
||||
|
||||
type timerWhen struct {
|
||||
timer *timer
|
||||
when int64
|
||||
}
|
||||
|
||||
func (ts *timers) lock() {
|
||||
lock(&ts.mu)
|
||||
}
|
||||
|
|
@ -146,9 +139,9 @@ const (
|
|||
// timerHeaped is set when the timer is stored in some P's heap.
|
||||
timerHeaped uint8 = 1 << iota
|
||||
|
||||
// timerModified is set when t.when has been modified but
|
||||
// t.whenHeap still needs to be updated as well.
|
||||
// The change to t.whenHeap waits until the heap in which
|
||||
// timerModified is set when t.when has been modified
|
||||
// but the heap's heap[i].when entry still needs to be updated.
|
||||
// That change waits until the heap in which
|
||||
// the timer appears can be locked and rearranged.
|
||||
// timerModified is only set when timerHeaped is also set.
|
||||
timerModified
|
||||
|
|
@ -204,7 +197,7 @@ func (t *timer) lock() {
|
|||
// unlock updates t.astate and unlocks the timer.
|
||||
func (t *timer) unlock() {
|
||||
t.trace("unlock")
|
||||
// Let heap fast paths know whether t.whenHeap is accurate.
|
||||
// Let heap fast paths know whether heap[i].when is accurate.
|
||||
// Also let maybeRunChan know whether channel is in heap.
|
||||
t.astate.Store(t.state)
|
||||
unlock(&t.mu)
|
||||
|
|
@ -222,41 +215,35 @@ func (t *timer) hchan() *hchan {
|
|||
return (*hchan)(efaceOf(&t.arg).data)
|
||||
}
|
||||
|
||||
// updateHeap updates t.whenHeap as directed by t.state, updating t.state
|
||||
// and returning a bool indicating whether the state (and t.whenHeap) changed.
|
||||
// updateHeap updates t as directed by t.state, updating t.state
|
||||
// and returning a bool indicating whether the state (and ts.heap[0].when) changed.
|
||||
// The caller must hold t's lock, or the world can be stopped instead.
|
||||
// If ts != nil, then ts must be locked, t must be ts.heap[0], and updateHeap
|
||||
// The timer set t.ts must be non-nil and locked, t must be t.ts.heap[0], and updateHeap
|
||||
// takes care of moving t within the timers heap to preserve the heap invariants.
|
||||
// If ts == nil, then t must not be in a heap (or is in a heap that is
|
||||
// temporarily not maintaining its invariant, such as during timers.adjust).
|
||||
func (t *timer) updateHeap(ts *timers) (updated bool) {
|
||||
func (t *timer) updateHeap() (updated bool) {
|
||||
assertWorldStoppedOrLockHeld(&t.mu)
|
||||
t.trace("updateHeap")
|
||||
if ts != nil {
|
||||
if t.ts != ts || t != ts.heap[0] {
|
||||
badTimer()
|
||||
}
|
||||
assertLockHeld(&ts.mu)
|
||||
ts := t.ts
|
||||
if ts == nil || t != ts.heap[0].timer {
|
||||
badTimer()
|
||||
}
|
||||
assertLockHeld(&ts.mu)
|
||||
if t.state&timerZombie != 0 {
|
||||
// Take timer out of heap.
|
||||
t.state &^= timerHeaped | timerZombie | timerModified
|
||||
if ts != nil {
|
||||
ts.zombies.Add(-1)
|
||||
ts.deleteMin()
|
||||
}
|
||||
ts.zombies.Add(-1)
|
||||
ts.deleteMin()
|
||||
return true
|
||||
}
|
||||
|
||||
if t.state&timerModified != 0 {
|
||||
// Apply t.whenHeap update and move within heap.
|
||||
// Update ts.heap[0].when and move within heap.
|
||||
t.state &^= timerModified
|
||||
t.whenHeap = t.when
|
||||
// Move t to the right position.
|
||||
if ts != nil {
|
||||
ts.siftDown(0)
|
||||
ts.updateMinWhenHeap()
|
||||
}
|
||||
ts.heap[0].when = t.when
|
||||
ts.siftDown(0)
|
||||
ts.updateMinWhenHeap()
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -384,10 +371,9 @@ func (ts *timers) addHeap(t *timer) {
|
|||
throw("ts set in timer")
|
||||
}
|
||||
t.ts = ts
|
||||
t.whenHeap = t.when
|
||||
ts.heap = append(ts.heap, t)
|
||||
ts.heap = append(ts.heap, timerWhen{t, t.when})
|
||||
ts.siftUp(len(ts.heap) - 1)
|
||||
if t == ts.heap[0] {
|
||||
if t == ts.heap[0].timer {
|
||||
ts.updateMinWhenHeap()
|
||||
}
|
||||
}
|
||||
|
|
@ -461,7 +447,7 @@ func (t *timer) stop() bool {
|
|||
// ts must be locked.
|
||||
func (ts *timers) deleteMin() {
|
||||
assertLockHeld(&ts.mu)
|
||||
t := ts.heap[0]
|
||||
t := ts.heap[0].timer
|
||||
if t.ts != ts {
|
||||
throw("wrong timers")
|
||||
}
|
||||
|
|
@ -470,7 +456,7 @@ func (ts *timers) deleteMin() {
|
|||
if last > 0 {
|
||||
ts.heap[0] = ts.heap[last]
|
||||
}
|
||||
ts.heap[last] = nil
|
||||
ts.heap[last] = timerWhen{}
|
||||
ts.heap = ts.heap[:last]
|
||||
if last > 0 {
|
||||
ts.siftDown(0)
|
||||
|
|
@ -522,9 +508,9 @@ func (t *timer) modify(when, period int64, f func(arg any, seq uintptr, delay in
|
|||
t.ts.zombies.Add(-1)
|
||||
t.state &^= timerZombie
|
||||
}
|
||||
// Cannot modify t.whenHeap until t.ts is locked.
|
||||
// The corresponding heap[i].when is updated later.
|
||||
// See comment in type timer above and in timers.adjust below.
|
||||
if when < t.whenHeap {
|
||||
if min := t.ts.minWhenModified.Load(); min == 0 || when < min {
|
||||
wake = true
|
||||
// Force timerModified bit out to t.astate before updating t.minWhenModified,
|
||||
// to synchronize with t.ts.adjust. See comment in adjust.
|
||||
|
|
@ -655,20 +641,20 @@ func (ts *timers) cleanHead() {
|
|||
// in heap[0] for the tail of the heap, we'll get a non-zombie timer,
|
||||
// shortening this loop.
|
||||
n := len(ts.heap)
|
||||
if t := ts.heap[n-1]; t.astate.Load()&timerZombie != 0 {
|
||||
if t := ts.heap[n-1].timer; t.astate.Load()&timerZombie != 0 {
|
||||
t.lock()
|
||||
if t.state&timerZombie != 0 {
|
||||
t.state &^= timerHeaped | timerZombie | timerModified
|
||||
t.ts = nil
|
||||
ts.zombies.Add(-1)
|
||||
ts.heap[n-1] = nil
|
||||
ts.heap[n-1] = timerWhen{}
|
||||
ts.heap = ts.heap[:n-1]
|
||||
}
|
||||
t.unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
t := ts.heap[0]
|
||||
t := ts.heap[0].timer
|
||||
if t.ts != ts {
|
||||
throw("bad ts")
|
||||
}
|
||||
|
|
@ -679,7 +665,7 @@ func (ts *timers) cleanHead() {
|
|||
}
|
||||
|
||||
t.lock()
|
||||
updated := t.updateHeap(ts)
|
||||
updated := t.updateHeap()
|
||||
t.unlock()
|
||||
if !updated {
|
||||
// Head of timers does not need adjustment.
|
||||
|
|
@ -700,7 +686,16 @@ func (ts *timers) take(src *timers) {
|
|||
// The world is stopped, so we ignore the locking of ts and src here.
|
||||
// That would introduce a sched < timers lock ordering,
|
||||
// which we'd rather avoid in the static ranking.
|
||||
ts.move(src.heap)
|
||||
for _, tw := range src.heap {
|
||||
t := tw.timer
|
||||
t.ts = nil
|
||||
if t.state&timerZombie != 0 {
|
||||
t.state &^= timerHeaped | timerZombie | timerModified
|
||||
} else {
|
||||
t.state &^= timerModified
|
||||
ts.addHeap(t)
|
||||
}
|
||||
}
|
||||
src.heap = nil
|
||||
src.zombies.Store(0)
|
||||
src.minWhenHeap.Store(0)
|
||||
|
|
@ -710,20 +705,6 @@ func (ts *timers) take(src *timers) {
|
|||
}
|
||||
}
|
||||
|
||||
// moveTimers moves a slice of timers to pp. The slice has been taken
|
||||
// from a different P.
|
||||
// The world must be stopped so that ts is safe to modify.
|
||||
func (ts *timers) move(timers []*timer) {
|
||||
assertWorldStopped()
|
||||
for _, t := range timers {
|
||||
t.ts = nil
|
||||
t.updateHeap(nil)
|
||||
if t.state&timerHeaped != 0 {
|
||||
ts.addHeap(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// adjust looks through the timers in ts.heap for
|
||||
// any timers that have been modified to run earlier, and puts them in
|
||||
// the correct place in the heap. While looking for those timers,
|
||||
|
|
@ -776,7 +757,7 @@ func (ts *timers) adjust(now int64, force bool) {
|
|||
// 2. Set minWhenModified = 0
|
||||
// (Other goroutines may modify timers and update minWhenModified now.)
|
||||
// 3. Scan timers
|
||||
// 4. Set minWhenHeap = heap[0].whenHeap
|
||||
// 4. Set minWhenHeap = heap[0].when
|
||||
//
|
||||
// That order preserves a correct value of wakeTime throughout the entire
|
||||
// operation:
|
||||
|
|
@ -802,7 +783,8 @@ func (ts *timers) adjust(now int64, force bool) {
|
|||
|
||||
changed := false
|
||||
for i := 0; i < len(ts.heap); i++ {
|
||||
t := ts.heap[i]
|
||||
tw := &ts.heap[i]
|
||||
t := tw.timer
|
||||
if t.ts != ts {
|
||||
throw("bad ts")
|
||||
}
|
||||
|
|
@ -813,22 +795,25 @@ func (ts *timers) adjust(now int64, force bool) {
|
|||
}
|
||||
|
||||
t.lock()
|
||||
if t.state&timerHeaped == 0 {
|
||||
switch {
|
||||
case t.state&timerHeaped == 0:
|
||||
badTimer()
|
||||
}
|
||||
if t.state&timerZombie != 0 {
|
||||
ts.zombies.Add(-1) // updateHeap will return updated=true and we will delete t
|
||||
}
|
||||
if t.updateHeap(nil) {
|
||||
|
||||
case t.state&timerZombie != 0:
|
||||
ts.zombies.Add(-1)
|
||||
t.state &^= timerHeaped | timerZombie | timerModified
|
||||
n := len(ts.heap)
|
||||
ts.heap[i] = ts.heap[n-1]
|
||||
ts.heap[n-1] = timerWhen{}
|
||||
ts.heap = ts.heap[:n-1]
|
||||
t.ts = nil
|
||||
i--
|
||||
changed = true
|
||||
|
||||
case t.state&timerModified != 0:
|
||||
tw.when = t.when
|
||||
t.state &^= timerModified
|
||||
changed = true
|
||||
if t.state&timerHeaped == 0 {
|
||||
n := len(ts.heap)
|
||||
ts.heap[i] = ts.heap[n-1]
|
||||
ts.heap[n-1] = nil
|
||||
ts.heap = ts.heap[:n-1]
|
||||
t.ts = nil
|
||||
i--
|
||||
}
|
||||
}
|
||||
t.unlock()
|
||||
}
|
||||
|
|
@ -945,20 +930,19 @@ Redo:
|
|||
if len(ts.heap) == 0 {
|
||||
return -1
|
||||
}
|
||||
t := ts.heap[0]
|
||||
tw := ts.heap[0]
|
||||
t := tw.timer
|
||||
if t.ts != ts {
|
||||
throw("bad ts")
|
||||
}
|
||||
|
||||
if t.astate.Load()&(timerModified|timerZombie) == 0 && t.whenHeap > now {
|
||||
if t.astate.Load()&(timerModified|timerZombie) == 0 && tw.when > now {
|
||||
// Fast path: not ready to run.
|
||||
// The access of t.whenHeap is protected by the caller holding
|
||||
// ts.lock, even though t itself is unlocked.
|
||||
return t.whenHeap
|
||||
return tw.when
|
||||
}
|
||||
|
||||
t.lock()
|
||||
if t.updateHeap(ts) {
|
||||
if t.updateHeap() {
|
||||
t.unlock()
|
||||
goto Redo
|
||||
}
|
||||
|
|
@ -1019,18 +1003,16 @@ func (t *timer) unlockAndRun(now int64) {
|
|||
} else {
|
||||
next = 0
|
||||
}
|
||||
ts := t.ts
|
||||
t.when = next
|
||||
if t.state&timerHeaped != 0 {
|
||||
t.when = next
|
||||
t.state |= timerModified
|
||||
if next == 0 {
|
||||
t.state |= timerZombie
|
||||
t.ts.zombies.Add(1)
|
||||
}
|
||||
} else {
|
||||
t.when = next
|
||||
t.updateHeap()
|
||||
}
|
||||
ts := t.ts
|
||||
t.updateHeap(ts)
|
||||
t.unlock()
|
||||
|
||||
if raceenabled {
|
||||
|
|
@ -1089,7 +1071,7 @@ func (t *timer) unlockAndRun(now int64) {
|
|||
// The caller must have locked ts.
|
||||
func (ts *timers) verify() {
|
||||
assertLockHeld(&ts.mu)
|
||||
for i, t := range ts.heap {
|
||||
for i, tw := range ts.heap {
|
||||
if i == 0 {
|
||||
// First timer has no parent.
|
||||
continue
|
||||
|
|
@ -1097,8 +1079,8 @@ func (ts *timers) verify() {
|
|||
|
||||
// The heap is 4-ary. See siftupTimer and siftdownTimer.
|
||||
p := (i - 1) / 4
|
||||
if t.whenHeap < ts.heap[p].whenHeap {
|
||||
print("bad timer heap at ", i, ": ", p, ": ", ts.heap[p].whenHeap, ", ", i, ": ", t.whenHeap, "\n")
|
||||
if tw.when < ts.heap[p].when {
|
||||
print("bad timer heap at ", i, ": ", p, ": ", ts.heap[p].when, ", ", i, ": ", tw.when, "\n")
|
||||
throw("bad timer heap")
|
||||
}
|
||||
}
|
||||
|
|
@ -1108,14 +1090,14 @@ func (ts *timers) verify() {
|
|||
}
|
||||
}
|
||||
|
||||
// updateMinWhenHeap sets ts.minWhenHeap to ts.heap[0].whenHeap.
|
||||
// updateMinWhenHeap sets ts.minWhenHeap to ts.heap[0].when.
|
||||
// The caller must have locked ts or the world must be stopped.
|
||||
func (ts *timers) updateMinWhenHeap() {
|
||||
assertWorldStoppedOrLockHeld(&ts.mu)
|
||||
if len(ts.heap) == 0 {
|
||||
ts.minWhenHeap.Store(0)
|
||||
} else {
|
||||
ts.minWhenHeap.Store(ts.heap[0].whenHeap)
|
||||
ts.minWhenHeap.Store(ts.heap[0].when)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1168,56 +1150,56 @@ func timeSleepUntil() int64 {
|
|||
// siftUp puts the timer at position i in the right place
|
||||
// in the heap by moving it up toward the top of the heap.
|
||||
func (ts *timers) siftUp(i int) {
|
||||
t := ts.heap
|
||||
if i >= len(t) {
|
||||
heap := ts.heap
|
||||
if i >= len(heap) {
|
||||
badTimer()
|
||||
}
|
||||
when := t[i].whenHeap
|
||||
tw := heap[i]
|
||||
when := tw.when
|
||||
if when <= 0 {
|
||||
badTimer()
|
||||
}
|
||||
tmp := t[i]
|
||||
for i > 0 {
|
||||
p := (i - 1) / 4 // parent
|
||||
if when >= t[p].whenHeap {
|
||||
if when >= heap[p].when {
|
||||
break
|
||||
}
|
||||
t[i] = t[p]
|
||||
heap[i] = heap[p]
|
||||
i = p
|
||||
}
|
||||
if tmp != t[i] {
|
||||
t[i] = tmp
|
||||
if heap[i].timer != tw.timer {
|
||||
heap[i] = tw
|
||||
}
|
||||
}
|
||||
|
||||
// siftDown puts the timer at position i in the right place
|
||||
// in the heap by moving it down toward the bottom of the heap.
|
||||
func (ts *timers) siftDown(i int) {
|
||||
t := ts.heap
|
||||
n := len(t)
|
||||
heap := ts.heap
|
||||
n := len(heap)
|
||||
if i >= n {
|
||||
badTimer()
|
||||
}
|
||||
when := t[i].whenHeap
|
||||
tw := heap[i]
|
||||
when := tw.when
|
||||
if when <= 0 {
|
||||
badTimer()
|
||||
}
|
||||
tmp := t[i]
|
||||
for {
|
||||
c := i*4 + 1 // left child
|
||||
c3 := c + 2 // mid child
|
||||
if c >= n {
|
||||
break
|
||||
}
|
||||
w := t[c].whenHeap
|
||||
if c+1 < n && t[c+1].whenHeap < w {
|
||||
w = t[c+1].whenHeap
|
||||
w := heap[c].when
|
||||
if c+1 < n && heap[c+1].when < w {
|
||||
w = heap[c+1].when
|
||||
c++
|
||||
}
|
||||
if c3 < n {
|
||||
w3 := t[c3].whenHeap
|
||||
if c3+1 < n && t[c3+1].whenHeap < w3 {
|
||||
w3 = t[c3+1].whenHeap
|
||||
w3 := heap[c3].when
|
||||
if c3+1 < n && heap[c3+1].when < w3 {
|
||||
w3 = heap[c3+1].when
|
||||
c3++
|
||||
}
|
||||
if w3 < w {
|
||||
|
|
@ -1228,11 +1210,11 @@ func (ts *timers) siftDown(i int) {
|
|||
if w >= when {
|
||||
break
|
||||
}
|
||||
t[i] = t[c]
|
||||
heap[i] = heap[c]
|
||||
i = c
|
||||
}
|
||||
if tmp != t[i] {
|
||||
t[i] = tmp
|
||||
if heap[i].timer != tw.timer {
|
||||
heap[i] = tw
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue