From bb53f071e85a2ebb5b405e7fec4661a725b7caf5 Mon Sep 17 00:00:00 2001
From: Rick Macklem <rmacklem@FreeBSD.org>
Date: Wed, 12 Jun 2024 16:41:12 -0700
Subject: [PATCH] nfscl: Add support for read delegations and atomic upgrade

For NFSv4.1/4.2, an atomic upgrade of a delegation from a
read delegation to a write delegation is allowed and can
result in significantly improved performance.
This patch adds this upgrade to the NFSv4.1/4.2 client and
enables use of read delegations.

For a test case of building a FreeBSD kernel (sources and
output objects) over a NFSv4.2 mount, these changes reduced
the elapsed time by 30% and included a reduction of 80% for
RPC counts when delegations were enabled.  As such, with this
patch there are at least certain cases where enabling
delegations seems to be worth the increased complexity they
bring.

This patch should only affect the NFSv4.1/4.2 behaviour
when delegations are enabled, which is not the default.

MFC after:	1 month
---
 sys/fs/nfsclient/nfs_clstate.c | 38 ++++++++++++++++++++--------------
 sys/fs/nfsclient/nfs_clsubs.c  |  6 +++---
 sys/fs/nfsclient/nfs_clvnops.c |  2 +-
 3 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c
index aef5d71592af..0a1eb51e279f 100644
--- a/sys/fs/nfsclient/nfs_clstate.c
+++ b/sys/fs/nfsclient/nfs_clstate.c
@@ -439,18 +439,6 @@ nfscl_deleg(mount_t mp, struct nfsclclient *clp, u_int8_t *nfhp,
 
 	KASSERT(mp != NULL, ("nfscl_deleg: mp NULL"));
 	nmp = VFSTONFS(mp);
-	/*
-	 * First, if we have received a Read delegation for a file on a
-	 * read/write file system, just return it, because they aren't
-	 * useful, imho.
-	 */
-	if (dp != NULL && !NFSMNT_RDONLY(mp) &&
-	    (dp->nfsdl_flags & NFSCLDL_READ)) {
-		nfscl_trydelegreturn(dp, cred, nmp, p);
-		free(dp, M_NFSCLDELEG);
-		*dpp = NULL;
-		return (0);
-	}
 
 	/*
 	 * Since a delegation might be added to the mount,
@@ -478,17 +466,35 @@ nfscl_deleg(mount_t mp, struct nfsclclient *clp, u_int8_t *nfhp,
 		nfscl_delegcnt++;
 	} else {
 		/*
-		 * Delegation already exists, what do we do if a new one??
+		 * A delegation already exists.  If the new one is a Write
+		 * delegation and the old one a Read delegation, return the
+		 * Read delegation.  Otherwise, return the new delegation.
 		 */
 		if (dp != NULL) {
-			printf("Deleg already exists!\n");
-			free(dp, M_NFSCLDELEG);
-			*dpp = NULL;
+			if ((dp->nfsdl_flags & NFSCLDL_WRITE) != 0 &&
+			    (tdp->nfsdl_flags & NFSCLDL_READ) != 0) {
+				TAILQ_REMOVE(&clp->nfsc_deleg, tdp, nfsdl_list);
+				LIST_REMOVE(tdp, nfsdl_hash);
+				*dpp = NULL;
+				TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp,
+				    nfsdl_list);
+				LIST_INSERT_HEAD(NFSCLDELEGHASH(clp, nfhp,
+				    fhlen), dp, nfsdl_hash);
+				dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
+			} else {
+				*dpp = NULL;
+				tdp = dp;	/* Return this one. */
+			}
 		} else {
 			*dpp = tdp;
+			tdp = NULL;
 		}
 	}
 	NFSUNLOCKCLSTATE();
+	if (tdp != NULL) {
+		nfscl_trydelegreturn(tdp, cred, nmp, p);
+		free(tdp, M_NFSCLDELEG);
+	}
 	return (0);
 }
 
diff --git a/sys/fs/nfsclient/nfs_clsubs.c b/sys/fs/nfsclient/nfs_clsubs.c
index 80ab979d22d7..8bb51e29e1d1 100644
--- a/sys/fs/nfsclient/nfs_clsubs.c
+++ b/sys/fs/nfsclient/nfs_clsubs.c
@@ -188,7 +188,7 @@ ncl_getattrcache(struct vnode *vp, struct vattr *vaper)
 	np = VTONFS(vp);
 	vap = &np->n_vattr.na_vattr;
 	nmp = VFSTONFS(vp->v_mount);
-	mustflush = nfscl_mustflush(vp);	/* must be before mtx_lock() */
+	mustflush = nfscl_nodeleg(vp, 0);	/* must be before mtx_lock() */
 	NFSLOCKNODE(np);
 	/* XXX n_mtime doesn't seem to be updated on a miss-and-reload */
 	timeo = (time_second - np->n_mtime.tv_sec) / 10;
@@ -221,8 +221,8 @@ ncl_getattrcache(struct vnode *vp, struct vattr *vaper)
 		    (time_second - np->n_attrstamp), timeo);
 #endif
 
-	if ((time_second - np->n_attrstamp) >= timeo &&
-	    (mustflush != 0 || np->n_attrstamp == 0)) {
+	if (mustflush != 0 && (np->n_attrstamp == 0 ||
+	    time_second - np->n_attrstamp >= timeo)) {
 		nfsstatsv1.attrcache_misses++;
 		NFSUNLOCKNODE(np);
 		KDTRACE_NFS_ATTRCACHE_GET_MISS(vp);
diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c
index 76a3cdf9281e..13341dfc26e0 100644
--- a/sys/fs/nfsclient/nfs_clvnops.c
+++ b/sys/fs/nfsclient/nfs_clvnops.c
@@ -940,7 +940,7 @@ nfs_close(struct vop_close_args *ap)
 		/*
 		 * Get attributes so "change" is up to date.
 		 */
-		if (error == 0 && nfscl_mustflush(vp) != 0 &&
+		if (error == 0 && nfscl_nodeleg(vp, 0) != 0 &&
 		    vp->v_type == VREG &&
 		    (VFSTONFS(vp->v_mount)->nm_flag & NFSMNT_NOCTO) == 0) {
 			ret = nfsrpc_getattr(vp, cred, ap->a_td, &nfsva);