system/freebsd-src
system/freebsd-src
1
0
Fork 0
mirror of https://github.com/freebsd/freebsd-src synced 2024-10-07 00:50:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

nvme: Use adaptive spinning when polling for completion or state change

Browse source 
We only use nvme_completion_poll in the initialization path. The
commands they queue and wait for finish quickly as they involve no I/O
to the drive's media. These command take about 20-200 microsecnds
each. Set the wait time to 1us and then increase it by 1.5 each
successive iteration (max 1ms). This reduces initialization time by
80ms in cpervica's tests.

Use this same technique waiting for RDY state transitions. This saves
another 20ms. In total we're down from ~330ms to ~2ms.

Tested by:		cperciva
Sponsored by:		Netflix
Reviewed by:		mav
Differential Review:	https://reviews.freebsd.org/D32259
This commit is contained in:
Warner Losh 2021-10-01 11:32:48 -06:00
parent ef7d2c1fc1
commit 83581511d9
2 changed files with 30 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
sys/dev/nvme/nvme_ctrlr.c
View file

@ -260,10 +260,17 @@ nvme_ctrlr_fail_req_task(void *arg, int pending)
mtx_unlock(&ctrlr->lock);
}
/*
* Wait for RDY to change.
*
* Starts sleeping for 1us and geometrically increases it the longer we wait,
* capped at 1ms.
*/
static int
nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
{
int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);
sbintime_t delta_t = SBT_1US;
uint32_t csts;
while (1) {
@ -278,7 +285,9 @@ nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
"within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
return (ENXIO);
}
pause("nvmerdy", 1);
pause_sbt("nvmerdy", delta_t, 0, C_PREL(1));
delta_t = min(SBT_1MS, delta_t * 3 / 2);
}
return (0);

33
sys/dev/nvme/nvme_private.h
View file

@ -455,25 +455,32 @@ int nvme_shutdown(device_t dev);
int nvme_detach(device_t dev);
/*
* Wait for a command to complete using the nvme_completion_poll_cb.
* Used in limited contexts where the caller knows it's OK to block
* briefly while the command runs. The ISR will run the callback which
* will set status->done to true, usually within microseconds. If not,
* then after one second timeout handler should reset the controller
* and abort all outstanding requests including this polled one. If
* still not after ten seconds, then something is wrong with the driver,
* and panic is the only way to recover.
* Wait for a command to complete using the nvme_completion_poll_cb. Used in
* limited contexts where the caller knows it's OK to block briefly while the
* command runs. The ISR will run the callback which will set status->done to
* true, usually within microseconds. If not, then after one second timeout
* handler should reset the controller and abort all outstanding requests
* including this polled one. If still not after ten seconds, then something is
* wrong with the driver, and panic is the only way to recover.
*
* Most commands using this interface aren't actual I/O to the drive's media so
* complete within a few microseconds. Adaptively spin for one tick to catch the
* vast majority of these without waiting for a tick plus scheduling delays. Since
* these are on startup, this drastically reduces startup time.
*/
static __inline
void
nvme_completion_poll(struct nvme_completion_poll_status *status)
{
int sanity = hz * 10;
int timeout = ticks + 10 * hz;
sbintime_t delta_t = SBT_1US;
while (!atomic_load_acq_int(&status->done) && --sanity > 0)
pause("nvme", 1);
if (sanity <= 0)
panic("NVME polled command failed to complete within 10s.");
while (!atomic_load_acq_int(&status->done)) {
if (timeout - ticks < 0)
panic("NVME polled command failed to complete within 10s.");
pause_sbt("nvme", delta_t, 0, C_PREL(1));
delta_t = min(SBT_1MS, delta_t * 3 / 2);
}
}
static __inline void