iscsi: per-session timeouts and rapid teardown of session on reconnect

Add per-Session configurable ping (SCSI NOP) and login timeouts.

Remove the torn down, old iSCSI session quickly, when performing a reconnect.

Reviewed By: trasz
Sponsored by:        NetApp, Inc.
Differential Revision: https://reviews.freebsd.org/D34198

lib/libiscsiutil/libiscsiutil.h

@@ -54,6 +54,8 @@ struct connection {
 	int		conn_max_send_data_segment_length;
 	int		conn_max_burst_length;
 	int		conn_first_burst_length;
+	int		conn_ping_timeout;
+	int		conn_login_timeout;
 };
 
 struct pdu {

sys/dev/iscsi/iscsi.c

@@ -42,9 +42,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/kthread.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
+#include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/socket.h>
+#include <sys/sockopt.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/sx.h>
@@ -86,6 +88,7 @@ SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 static int debug = 1;
 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
     &debug, 0, "Enable debug messages");
+
 static int ping_timeout = 5;
 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
     0, "Timeout for ping (NOP-Out) requests, in seconds");
@@ -380,6 +383,26 @@ iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 static void
 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 {
+	/*
+	 * As we will be reconnecting shortly,
+	 * discard outstanding data immediately on
+	 * close(), also notify peer via RST if
+	 * any packets come in.
+	 */
+	struct socket *so;
+	so = is->is_conn->ic_socket;
+	if (so != NULL) {
+		struct sockopt sopt;
+		struct linger sl;
+		sopt.sopt_dir     = SOPT_SET;
+		sopt.sopt_level   = SOL_SOCKET;
+		sopt.sopt_name    = SO_LINGER;
+		sopt.sopt_val     = &sl;
+		sopt.sopt_valsize = sizeof(sl);
+		sl.l_onoff        = 1;	/* non-zero value enables linger option in kernel */
+		sl.l_linger       = 0;	/* timeout interval in seconds */
+		sosetopt(is->is_conn->ic_socket, &sopt);
+	}
 
 	icl_conn_close(is->is_conn);
 
@@ -576,7 +599,7 @@ iscsi_callout(void *context)
 	}
 
 	if (is->is_login_phase) {
-		if (login_timeout > 0 && is->is_timeout > login_timeout) {
+		if (is->is_login_timeout > 0 && is->is_timeout > is->is_login_timeout) {
 			ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 			    "reconnecting", is->is_timeout);
 			reconnect_needed = true;
@@ -584,7 +607,7 @@ iscsi_callout(void *context)
 		goto out;
 	}
 
-	if (ping_timeout <= 0) {
+	if (is->is_ping_timeout <= 0) {
 		/*
 		 * Pings are disabled.  Don't send NOP-Out in this case.
 		 * Reset the timeout, to avoid triggering reconnection,
@@ -594,9 +617,9 @@ iscsi_callout(void *context)
 		goto out;
 	}
 
-	if (is->is_timeout >= ping_timeout) {
+	if (is->is_timeout >= is->is_ping_timeout) {
 		ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
-		    "reconnecting", ping_timeout);
+		    "reconnecting", is->is_ping_timeout);
 		reconnect_needed = true;
 		goto out;
 	}
@@ -1509,6 +1532,12 @@ iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
 	is->is_waiting_for_iscsid = false;
 	is->is_login_phase = false;
 	is->is_timeout = 0;
+	is->is_ping_timeout = is->is_conf.isc_ping_timeout;
+	if (is->is_ping_timeout < 0)
+		is->is_ping_timeout = ping_timeout;
+	is->is_login_timeout = is->is_conf.isc_login_timeout;
+	if (is->is_login_timeout < 0)
+		is->is_login_timeout = login_timeout;
 	is->is_connected = true;
 	is->is_reason[0] = '\0';
 
@@ -1915,6 +1944,12 @@ iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
 		sx_xunlock(&sc->sc_lock);
 		return (error);
 	}
+	is->is_ping_timeout = is->is_conf.isc_ping_timeout;
+	if (is->is_ping_timeout < 0)
+		is->is_ping_timeout = ping_timeout;
+	is->is_login_timeout = is->is_conf.isc_login_timeout;
+	if (is->is_login_timeout < 0)
+		is->is_login_timeout = login_timeout;
 
 	sbt = mstosbt(995);
 	pr = mstosbt(10);

sys/dev/iscsi/iscsi.h

@@ -75,6 +75,8 @@ struct iscsi_session {
 
 	struct callout			is_callout;
 	unsigned int			is_timeout;
+	int				is_ping_timeout;
+	int				is_login_timeout;
 
 	/*
 	 * XXX: This could be rewritten using a single variable,

sys/dev/iscsi/iscsi_ioctl.h

@@ -71,7 +71,8 @@ struct iscsi_session_conf {
 	int		isc_enable;
 	int		isc_dscp;
 	int		isc_pcp;
-	int		isc_spare[2];
+	int		isc_ping_timeout;
+	int		isc_login_timeout;
 };
 
 /*

usr.bin/iscsictl/iscsi.conf.5

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 6, 2016
+.Dd February 25, 2022
 .Dt ISCSI.CONF 5
 .Os
 .Sh NAME
@@ -160,6 +160,30 @@ The PCP can be set to a value in the range between
 to
 .Qq Ar 7 .
 When omitted, the default for the outgoing interface is used.
+.It Cm PingTimeout
+Specify the time in seconds to wait between pings (SCSI NOP), and
+for a ping response before declaring the session as dead and
+attempting a re-establishment.
+If this entry is not present in the conf file, the default value
+configured using
+.Qq Ar kern.iscsi.ping_timeout
+(default at
+.Qq Ar 5
+seconds) is taken by the driver.
+If present, the PingTimeout can be set to any positive value
+starting with
+.Qq Ar 1 .
+.It Cm LoginTimeout
+Specify the time in seconds to wait for a login PDU to be sent or
+received after trying to establish a new session.
+When no login PDU is received within this time, the login on a
+particular connection fails and a new reconnection attempt is made.
+If this entry is not present in the conf file, the default value of
+.Qq Ar 60
+seconds is used, as configured by
+.Qq Ar kern.iscsi.login_timeout .
+The LoginTimeout can be set to any positive value starting with
+.Qq Ar 1 .
 .El
 .Sh FILES
 .Bl -tag -width indent

usr.bin/iscsictl/iscsictl.c

@@ -88,6 +88,8 @@ target_new(struct conf *conf)
 	targ->t_conf = conf;
 	targ->t_dscp = -1;
 	targ->t_pcp = -1;
+	targ->t_pingtimeout = -1;
+	targ->t_logintimeout = -1;
 	TAILQ_INSERT_TAIL(&conf->conf_targets, targ, t_next);
 
 	return (targ);
@@ -361,6 +363,8 @@ conf_from_target(struct iscsi_session_conf *conf,
 		conf->isc_data_digest = ISCSI_DIGEST_NONE;
 	conf->isc_dscp = targ->t_dscp;
 	conf->isc_pcp = targ->t_pcp;
+	conf->isc_ping_timeout = targ->t_pingtimeout;
+	conf->isc_login_timeout = targ->t_logintimeout;
 }
 
 static int
@@ -544,6 +548,14 @@ kernel_list(int iscsi_fd, const struct target *targ __unused,
 			if (conf->isc_pcp != -1)
 				xo_emit("{L:/%-26s}{V:pcp/0x%02x}\n",
 				    "Target PCP:", conf->isc_pcp);
+			if (conf->isc_ping_timeout != -1)
+				xo_emit("{L:/%-26s}{V:PingTimeout/%d}\n",
+				    "Target PingTimeout:",
+				    conf->isc_ping_timeout);
+			if (conf->isc_login_timeout != -1)
+				xo_emit("{L:/%-26s}{V:LoginTimeout/%d}\n",
+				    "Target LoginTimeout:",
+				    conf->isc_login_timeout);
 			xo_close_container("target");
 
 			xo_open_container("auth");

usr.bin/iscsictl/iscsictl.h

@@ -79,6 +79,8 @@ struct target {
 	int			t_protocol;
 	int			t_dscp;
 	int			t_pcp;
+	int			t_pingtimeout;
+	int			t_logintimeout;
 	char			*t_offload;
 	char			*t_user;
 	char			*t_secret;

usr.bin/iscsictl/parse.y

@@ -61,7 +61,7 @@ extern void	yyrestart(FILE *);
 %token AUTH_METHOD ENABLE HEADER_DIGEST DATA_DIGEST TARGET_NAME TARGET_ADDRESS
 %token INITIATOR_NAME INITIATOR_ADDRESS INITIATOR_ALIAS USER SECRET
 %token MUTUAL_USER MUTUAL_SECRET SEMICOLON SESSION_TYPE PROTOCOL OFFLOAD
-%token IGNORED EQUALS OPENING_BRACKET CLOSING_BRACKET DSCP
+%token IGNORED EQUALS OPENING_BRACKET CLOSING_BRACKET DSCP PINGTIMEOUT LOGINTIMEOUT
 %token AF11 AF12 AF13 AF21 AF22 AF23 AF31 AF32 AF33 AF41 AF42 AF43
 %token BE EF CS0 CS1 CS2 CS3 CS4 CS5 CS6 CS7
 
@@ -133,6 +133,10 @@ target_entry:
 	dscp
 	|
 	pcp
+	|
+	ping_timeout
+	|
+	login_timeout
 	;
 
 target_name:	TARGET_NAME EQUALS STR
@@ -367,6 +371,38 @@ pcp:	PCP EQUALS STR
 	}
 	;
 
+ping_timeout:	PINGTIMEOUT EQUALS STR
+	{
+		uint64_t tmp;
+
+		if (target->t_pingtimeout != -1)
+			xo_errx(1, "duplicated PingTimeout at line %d", lineno);
+
+		if (expand_number($3, &tmp) != 0) {
+			yyerror("invalid numeric value");
+			free($3);
+			return(1);
+		}
+		target->t_pingtimeout = tmp;
+	}
+	;
+
+login_timeout:	LOGINTIMEOUT EQUALS STR
+	{
+		uint64_t tmp;
+
+		if (target->t_logintimeout != -1)
+			xo_errx(1, "duplicated LoginTimeout at line %d", lineno);
+
+		if (expand_number($3, &tmp) != 0) {
+			yyerror("invalid numeric value");
+			free($3);
+			return(1);
+		}
+		target->t_logintimeout = tmp;
+	}
+	;
+
 %%
 
 void

usr.bin/iscsictl/token.l

@@ -69,6 +69,8 @@ offload			{ return OFFLOAD; }
 port			{ return IGNORED; }
 dscp			{ return DSCP; }
 pcp			{ return PCP; }
+PingTimeout		{ return PINGTIMEOUT; }
+LoginTimeout		{ return LOGINTIMEOUT; }
 MaxConnections		{ return IGNORED; }
 TargetAlias		{ return IGNORED; }
 TargetPortalGroupTag	{ return IGNORED; }

usr.sbin/iscsid/iscsid.c

@@ -38,9 +38,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/linker.h>
 #include <sys/socket.h>
+#include <sys/sysctl.h>
 #include <sys/capsicum.h>
 #include <sys/wait.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <assert.h>
 #include <capsicum_helpers.h>
 #include <errno.h>
@@ -383,6 +385,32 @@ connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
 				    from_addr);
 		}
 	}
+	/*
+	 * Reduce TCP SYN_SENT timeout while
+	 * no connectivity exists, to allow
+	 * rapid reuse of the available slots.
+	 */
+	int keepinit = 0;
+	if (conn->conn_conf.isc_login_timeout > 0) {
+		keepinit = conn->conn_conf.isc_login_timeout;
+		log_debugx("session specific LoginTimeout at %d sec",
+			keepinit);
+	}
+	if (conn->conn_conf.isc_login_timeout == -1) {
+		char value[8];
+		size_t size = sizeof(value);
+		sysctlbyname("kern.iscsi.login_timeout", &value, &size,
+			NULL, 0);
+		keepinit = strtol(value, NULL, 10);
+		log_debugx("global login_timeout at %d sec", keepinit);
+	}
+	if (keepinit > 0) {
+		if (setsockopt(conn->conn.conn_socket,
+		    IPPROTO_TCP, TCP_KEEPINIT,
+		    &keepinit, sizeof(keepinit)) == -1)
+			log_warnx("setsockopt(TCP_KEEPINIT) "
+			    "failed for %s", to_addr);
+	}
 	if (from_ai != NULL) {
 		error = bind(conn->conn.conn_socket, from_ai->ai_addr,
 		    from_ai->ai_addrlen);