From aacaeeee8ecd7084a33b2d8e140aad37b3b0eddc Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Wed, 5 Jun 2024 12:54:15 -0700 Subject: [PATCH] nvmf: Permit failing I/O requests while disconnected Add a kern.nvmf.fail_on_disconnection sysctl similar to the kern.iscsi.fail_on_disconnection sysctl. This causes pending I/O requests to fail with an error if an association is disconnected instead of requeueing to be retried once the association is reconnected. As with iSCSI, the default is to queue and retry operations. Reviewed by: imp Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D45308 --- share/man/man4/nvmf.4 | 24 +++++++++++++++++++++++- sys/dev/nvmf/host/nvmf.c | 5 +++++ sys/dev/nvmf/host/nvmf_ns.c | 25 ++++++++++++++++++++----- sys/dev/nvmf/host/nvmf_sim.c | 10 ++++++++-- sys/dev/nvmf/host/nvmf_var.h | 3 +++ 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/share/man/man4/nvmf.4 b/share/man/man4/nvmf.4 index 8afbb4d9daaf..298365acefa9 100644 --- a/share/man/man4/nvmf.4 +++ b/share/man/man4/nvmf.4 @@ -3,7 +3,7 @@ .\" .\" Copyright (c) 2024 Chelsio Communications, Inc. .\" -.Dd May 2, 2024 +.Dd June 5, 2024 .Dt NVMF 4 .Os .Sh NAME @@ -65,6 +65,28 @@ disk driver. Associations require a supported transport such as .Xr nvmf_tcp 4 for associations using TCP/IP. +.Sh SYSCTL VARIABLES +The following variables are available as both +.Xr sysctl 8 +variables and +.Xr loader 8 +tunables: +.Bl -tag -width indent +.It Va kern.nvmf.fail_on_disconnection +Determines the behavior when an association's connection is interrupted. +By default, input/output operations are suspended while a host is disconnected. +This includes operations pending at the time the association's connection was +interrupted as well as new requests submitted while the host is disconnected. +Once a new association is established, suspended I/O requests are retried. +When set to 1, input/output operations fail with +.Er EIO +while a host is disconnected and +.Xr nda 4 +peripherals are destroyed after the first failed I/O request. +Note that any destroyed +.Xr nda 4 +peripherals will be recreated after a new association is established. +.El .Sh SEE ALSO .Xr nda 4 , .Xr nvme 4 , diff --git a/sys/dev/nvmf/host/nvmf.c b/sys/dev/nvmf/host/nvmf.c index 9684170c1de9..c309836ed8a8 100644 --- a/sys/dev/nvmf/host/nvmf.c +++ b/sys/dev/nvmf/host/nvmf.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,10 @@ static struct cdevsw nvmf_cdevsw; +bool nvmf_fail_disconnect = false; +SYSCTL_BOOL(_kern_nvmf, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN, + &nvmf_fail_disconnect, 0, "Fail I/O requests on connection failure"); + MALLOC_DEFINE(M_NVMF, "nvmf", "NVMe over Fabrics host"); static void nvmf_disconnect_task(void *arg, int pending); diff --git a/sys/dev/nvmf/host/nvmf_ns.c b/sys/dev/nvmf/host/nvmf_ns.c index 0727ca960a57..8381cc4aec54 100644 --- a/sys/dev/nvmf/host/nvmf_ns.c +++ b/sys/dev/nvmf/host/nvmf_ns.c @@ -84,13 +84,22 @@ nvmf_ns_biodone(struct bio *bio) ns = bio->bio_dev->si_drv1; /* If a request is aborted, resubmit or queue it for resubmission. */ - if (bio->bio_error == ECONNABORTED) { + if (bio->bio_error == ECONNABORTED && !nvmf_fail_disconnect) { bio->bio_error = 0; bio->bio_driver2 = 0; mtx_lock(&ns->lock); if (ns->disconnected) { - TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue); - mtx_unlock(&ns->lock); + if (nvmf_fail_disconnect) { + mtx_unlock(&ns->lock); + bio->bio_error = ECONNABORTED; + bio->bio_flags |= BIO_ERROR; + bio->bio_resid = bio->bio_bcount; + biodone(bio); + } else { + TAILQ_INSERT_TAIL(&ns->pending_bios, bio, + bio_queue); + mtx_unlock(&ns->lock); + } } else { mtx_unlock(&ns->lock); nvmf_ns_strategy(bio); @@ -163,6 +172,7 @@ nvmf_ns_submit_bio(struct nvmf_namespace *ns, struct bio *bio) struct nvme_dsm_range *dsm_range; struct memdesc mem; uint64_t lba, lba_count; + int error; dsm_range = NULL; memset(&cmd, 0, sizeof(cmd)); @@ -201,10 +211,15 @@ nvmf_ns_submit_bio(struct nvmf_namespace *ns, struct bio *bio) mtx_lock(&ns->lock); if (ns->disconnected) { - TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue); + if (nvmf_fail_disconnect) { + error = ECONNABORTED; + } else { + TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue); + error = 0; + } mtx_unlock(&ns->lock); free(dsm_range, M_NVMF); - return (0); + return (error); } req = nvmf_allocate_request(nvmf_select_io_queue(ns->sc), &cmd, diff --git a/sys/dev/nvmf/host/nvmf_sim.c b/sys/dev/nvmf/host/nvmf_sim.c index 00dad07889d1..71bb71dd4063 100644 --- a/sys/dev/nvmf/host/nvmf_sim.c +++ b/sys/dev/nvmf/host/nvmf_sim.c @@ -40,7 +40,10 @@ nvmf_ccb_done(union ccb *ccb) return; if (nvmf_cqe_aborted(&ccb->nvmeio.cpl)) { - ccb->ccb_h.status = CAM_REQUEUE_REQ; + if (nvmf_fail_disconnect) + ccb->ccb_h.status = CAM_DEV_NOT_THERE; + else + ccb->ccb_h.status = CAM_REQUEUE_REQ; xpt_done(ccb); } else if (ccb->nvmeio.cpl.status != 0) { ccb->ccb_h.status = CAM_NVME_STATUS_ERROR; @@ -106,7 +109,10 @@ nvmf_sim_io(struct nvmf_softc *sc, union ccb *ccb) mtx_lock(&sc->sim_mtx); if (sc->sim_disconnected) { mtx_unlock(&sc->sim_mtx); - nvmeio->ccb_h.status = CAM_REQUEUE_REQ; + if (nvmf_fail_disconnect) + nvmeio->ccb_h.status = CAM_DEV_NOT_THERE; + else + nvmeio->ccb_h.status = CAM_REQUEUE_REQ; xpt_done(ccb); return; } diff --git a/sys/dev/nvmf/host/nvmf_var.h b/sys/dev/nvmf/host/nvmf_var.h index 2fa0216baab8..cf88d2f7b01e 100644 --- a/sys/dev/nvmf/host/nvmf_var.h +++ b/sys/dev/nvmf/host/nvmf_var.h @@ -140,6 +140,9 @@ extern driver_t nvme_nvmf_driver; MALLOC_DECLARE(M_NVMF); #endif +/* If true, I/O requests will fail while the host is disconnected. */ +extern bool nvmf_fail_disconnect; + /* nvmf.c */ void nvmf_complete(void *arg, const struct nvme_completion *cqe); void nvmf_io_complete(void *arg, size_t xfered, int error);