gmultipath: add ATF tests

Add ATF tests for most gmultipath operations. Add some dtrace probes too,
primarily for configuration changes that happen in response to provider
errors.

PR:		178473
MFC after:	2 weeks
Sponsored by:	Axcient
Differential Revision:	https://reviews.freebsd.org/D22235

etc/mtree/BSD.tests.dist

@@ -754,6 +754,8 @@
                 ..
                 mirror
                 ..
+                multipath
+                ..
                 nop
                 ..
                 part

sys/geom/geom_subr.c

@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/errno.h>
 #include <sys/sbuf.h>
+#include <sys/sdt.h>
 #include <geom/geom.h>
 #include <geom/geom_dbg.h>
 #include <geom/geom_int.h>
@@ -66,6 +67,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/kdb.h>
 #endif
 
+SDT_PROVIDER_DEFINE(geom);
+
 struct class_list_head g_classes = LIST_HEAD_INITIALIZER(g_classes);
 static struct g_tailq_head geoms = TAILQ_HEAD_INITIALIZER(geoms);
 char *g_wait_event, *g_wait_up, *g_wait_down, *g_wait_sim;

sys/geom/multipath/g_multipath.c

@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/bio.h>
 #include <sys/sbuf.h>
+#include <sys/sdt.h>
 #include <sys/sysctl.h>
 #include <sys/kthread.h>
 #include <sys/malloc.h>
@@ -61,6 +62,14 @@ static u_int g_multipath_exclusive = 1;
 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW,
     &g_multipath_exclusive, 0, "Exclusively open providers");
 
+SDT_PROVIDER_DECLARE(geom);
+SDT_PROBE_DEFINE2(geom, multipath, config, restore, "char*", "char*");
+SDT_PROBE_DEFINE2(geom, multipath, config, remove, "char*", "char*");
+SDT_PROBE_DEFINE2(geom, multipath, config, disconnect, "char*", "char*");
+SDT_PROBE_DEFINE3(geom, multipath, config, fail, "char*", "char*", "int");
+SDT_PROBE_DEFINE2(geom, multipath, config, taste, "char*", "char*");
+SDT_PROBE_DEFINE2(geom, multipath, io, restart, "struct bio*", "struct bio*");
+
 static enum {
 	GKT_NIL,
 	GKT_RUN,
@@ -146,6 +155,8 @@ g_multipath_fault(struct g_consumer *cp, int cause)
 			printf("GEOM_MULTIPATH: "
 			    "all paths in %s were marked FAIL, restore %s\n",
 			    sc->sc_name, lcp->provider->name);
+			SDT_PROBE2(geom, multipath, config, restore,
+			    sc->sc_name, lcp->provider->name);
 			lcp->index &= ~MP_FAIL;
 		}
 	}
@@ -217,6 +228,8 @@ g_mpd(void *arg, int flags __unused)
 	if (cp->provider) {
 		printf("GEOM_MULTIPATH: %s removed from %s\n",
 		    cp->provider->name, gp->name);
+		SDT_PROBE2(geom, multipath, config, remove,
+		    gp->name, cp->provider->name);
 		g_detach(cp);
 	}
 	g_destroy_consumer(cp);
@@ -234,6 +247,8 @@ g_multipath_orphan(struct g_consumer *cp)
 	g_topology_assert();
 	printf("GEOM_MULTIPATH: %s in %s was disconnected\n",
 	    cp->provider->name, cp->geom->name);
+	SDT_PROBE2(geom, multipath, config, disconnect,
+	    cp->geom->name, cp->provider->name);
 	sc = cp->geom->softc;
 	cnt = (uintptr_t *)&cp->private;
 	mtx_lock(&sc->sc_mtx);
@@ -411,6 +426,8 @@ g_multipath_done_error(struct bio *bp)
 	if ((cp->index & MP_FAIL) == 0) {
 		printf("GEOM_MULTIPATH: Error %d, %s in %s marked FAIL\n",
 		    bp->bio_error, pp->name, sc->sc_name);
+		SDT_PROBE3(geom, multipath, config, fail,
+		    sc->sc_name, pp->name, bp->bio_error);
 		g_multipath_fault(cp, MP_FAIL);
 	}
 	(*cnt)--;
@@ -426,6 +443,7 @@ g_multipath_done_error(struct bio *bp)
 	 */
 	if (pbp->bio_children < (uintptr_t)pbp->bio_driver1) {
 		pbp->bio_inbed++;
+		SDT_PROBE2(geom, multipath, io, restart, bp, pbp);
 		g_destroy_bio(bp);
 		g_multipath_start(pbp);
 	} else {
@@ -831,6 +849,7 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		return (NULL);
 	if (g_multipath_debug)
 		printf("MULTIPATH: %s/%s\n", md.md_name, md.md_uuid);
+	SDT_PROBE2(geom, multipath, config, taste, md.md_name, md.md_uuid);
 
 	/*
 	 * Let's check if such a device already is present. We check against
@@ -1230,8 +1249,12 @@ g_multipath_ctl_fail(struct gctl_req *req, struct g_class *mp, int fail)
 				name, sc->sc_name, fail ? "FAIL" : "OK");
 			if (fail) {
 				g_multipath_fault(cp, MP_FAIL);
+				SDT_PROBE3(geom, multipath, config, fail,
+				    sc->sc_name, cp->provider->name, 0);
 			} else {
 				cp->index &= ~MP_FAIL;
+				SDT_PROBE2(geom, multipath, config, restore,
+				    sc->sc_name, cp->provider->name);
 			}
 		}
 	}
@@ -1277,6 +1300,8 @@ g_multipath_ctl_remove(struct gctl_req *req, struct g_class *mp)
 			found = 1;
 			printf("GEOM_MULTIPATH: removing %s from %s\n",
 			    cp->provider->name, cp->geom->name);
+			SDT_PROBE2(geom, multipath, config, remove,
+			    cp->geom->name, cp->provider->name);
 			sc->sc_ndisks--;
 			g_multipath_fault(cp, MP_LOST);
 			cnt = (uintptr_t *)&cp->private;

tests/sys/geom/class/Makefile

@@ -8,6 +8,7 @@ TESTS_SUBDIRS+=	concat
 TESTS_SUBDIRS+=	eli
 TESTS_SUBDIRS+=	gate
 TESTS_SUBDIRS+=	mirror
+TESTS_SUBDIRS+=	multipath
 TESTS_SUBDIRS+=	nop
 TESTS_SUBDIRS+=	part
 TESTS_SUBDIRS+=	raid3

tests/sys/geom/class/multipath/Makefile

@@ -0,0 +1,13 @@
+# $FreeBSD$
+
+PACKAGE=	tests
+
+TESTSDIR=	${TESTSBASE}/sys/geom/class/${.CURDIR:T}
+
+ATF_TESTS_SH+=	failloop
+ATF_TESTS_SH+=	misc
+TEST_METADATA.failloop+= is_exclusive=true
+
+${PACKAGE}FILES+=		conf.sh
+
+.include <bsd.test.mk>

tests/sys/geom/class/multipath/conf.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+# Copyright (c) 2019 Axcient
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+MD_DEVS="md.devs"
+MULTIPATH_DEVS="multipath.devs"
+
+alloc_md()
+{
+	local md
+
+	md=$(mdconfig -a -t swap -s 1M) || atf_fail "mdconfig -a failed"
+	echo ${md} >> $MD_DEVS
+	echo ${md}
+}
+
+# Verify expected state.
+# check_multipath_state <active_path> <geom_state> <prov0_state> <prov1_state> [prov2_state]
+check_multipath_state()
+{
+	local want_active_path=$1
+	local want_geom_state=$2
+	local want_prov0_state=$3
+	local want_prov1_state=$4
+	local want_prov2_state=$5
+	local geom_state
+	local prov0_state
+	local prov1_state
+	local prov2_state
+
+	geom_state=`gmultipath list "$name" | awk '/^State:/ {print $2}'`
+	atf_check_equal "$want_geom_state" "$geom_state"
+	prov0_state=`gmultipath list "$name" | awk '/1. Name: md[0-9]/ {trigger=1} /State:/ && trigger == 1 {print $2; trigger=0;}'`
+	prov1_state=`gmultipath list "$name" | awk '/2. Name: md[0-9]/ {trigger=1} /State:/ && trigger == 1 {print $2; trigger=0;}'`
+	prov2_state=`gmultipath list "$name" | awk '/3. Name: md[0-9]/ {trigger=1} /State:/ && trigger == 1 {print $2; trigger=0;}'`
+	atf_check_equal "$want_active_path" "`gmultipath getactive "$name"`"
+	atf_check_equal "$want_prov0_state" $prov0_state
+	atf_check_equal "$want_prov1_state" $prov1_state
+	if [ -n "$want_prov2_state" ]; then
+		atf_check_equal "$want_prov2_state" $prov2_state
+	fi
+}
+
+common_cleanup()
+{
+	name=$(cat $MULTIPATH_DEVS)
+	if [ -n "$name" -a -c "/dev/multipath/$name" ]; then
+		gmultipath destroy "$name"
+		rm $MULTIPATH_DEVS
+	fi
+	if [ -f "$MD_DEVS" ]; then
+		while read test_md; do
+			gnop destroy -f ${test_md}.nop 2>/dev/null
+			mdconfig -d -u $test_md 2>/dev/null
+		done < $MD_DEVS
+		rm $MD_DEVS
+	fi
+	true
+}
+
+load_dtrace()
+{
+	if ! kldstat -q -m sdt; then
+		kldload sdt || atf_skip "could not load module for dtrace SDT"
+	fi
+}
+
+load_gmultipath()
+{
+	if ! kldstat -q -m g_multipath; then
+		geom multipath load || atf_skip "could not load module for geom multipath"
+	fi
+}
+
+load_gnop()
+{
+	if ! kldstat -q -m g_nop; then
+		geom nop load || atf_skip "could not load module for geom nop"
+	fi
+}
+
+mkname()
+{
+	mktemp -u mp.XXXXXX | tee $MULTIPATH_DEVS
+}

tests/sys/geom/class/multipath/failloop.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+# Copyright (c) 2019 Axcient
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+. $(atf_get_srcdir)/conf.sh
+
+# See also https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=178473
+atf_test_case failloop cleanup
+failloop_head()
+{
+	atf_set "descr" "A persistent failure in the provider should not cause an infinite loop, nor restore any providers that were faulted by the same bio"
+	atf_set "require.user" "root"
+	atf_set "require.config" "allow_sysctl_side_effects"
+}
+failloop_body()
+{
+	sysctl -n kern.geom.notaste > kern.geom.notaste.txt
+	load_gnop
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check gnop create /dev/${md0}
+	atf_check gnop create /dev/${md1}
+	atf_check -s exit:0 gmultipath create "$name" ${md0}.nop ${md1}.nop
+	sysctl kern.geom.notaste=1
+
+	atf_check gnop configure -r 100 -w 100  ${md0}.nop
+	atf_check gnop configure -r 100 -w 100  ${md1}.nop
+	dd_status=`dtrace \
+		-o restore_count \
+		-i 'geom:multipath:config:restore {@restore = count()}' \
+		-c "dd if=/dev/zero of=/dev/multipath/"$name" bs=4096 count=1" \
+		2>&1 | awk '/exited with status/ {print $NF}'`
+	# The dd command should've failed ...
+	atf_check_equal 1 $dd_status
+	# and triggered 1 or 2 path restores
+	if [ `cat restore_count` -gt 2 ]; then
+		atf_fail "gmultipath restored paths too many times"
+	fi
+}
+failloop_cleanup()
+{
+	if [ -f kern.geom.notaste.txt ]; then
+		sysctl kern.geom.notaste=`cat kern.geom.notaste.txt`
+	fi
+	common_cleanup
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case failloop
+}

tests/sys/geom/class/multipath/misc.sh

@@ -0,0 +1,363 @@
+#!/bin/sh
+# Copyright (c) 2019 Axcient
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+. $(atf_get_srcdir)/conf.sh
+
+atf_test_case add cleanup
+add_head()
+{
+	atf_set "descr" "Add a new path"
+	atf_set "require.user" "root"
+}
+add_body()
+{
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	md2=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create "$name" ${md0} ${md1}
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "PASSIVE" 
+
+	# Add a new path
+	atf_check -s exit:0 gmultipath add "$name" ${md2}
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "PASSIVE" "PASSIVE"
+}
+add_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case create_A cleanup
+create_A_head()
+{
+	atf_set "descr" "Create an Active/Active multipath device"
+	atf_set "require.user" "root"
+}
+create_A_body()
+{
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create -A "$name" ${md0} ${md1}
+	check_multipath_state "${md1} ${md0}" "OPTIMAL" "ACTIVE" "ACTIVE" 
+}
+create_A_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case create_R cleanup
+create_R_head()
+{
+	atf_set "descr" "Create an Active/Read multipath device"
+	atf_set "require.user" "root"
+}
+create_R_body()
+{
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create -R "$name" ${md0} ${md1}
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "READ" 
+}
+create_R_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case depart_and_arrive cleanup
+depart_and_arrive_head()
+{
+	atf_set "descr" "gmultipath should remove devices that disappear, and automatically reattach labeled providers that reappear"
+	atf_set "require.user" "root"
+}
+depart_and_arrive_body()
+{
+	load_gnop
+	load_gmultipath
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	# We need a non-zero offset to gmultipath won't see the label when it
+	# tastes the md device.  We only want the label to be visible on the
+	# gnop device.
+	offset=131072
+	atf_check gnop create -o $offset /dev/${md0}
+	atf_check gnop create -o $offset /dev/${md1}
+	atf_check -s exit:0 gmultipath label "$name" ${md0}.nop
+	# gmultipath is too smart to let us create a gmultipath device by label
+	# when the two providers aren't actually related.  So we create a
+	# device by label with one provider, and then manually add the second.
+	atf_check -s exit:0 gmultipath add "$name" ${md1}.nop
+	NDEVS=`gmultipath list "$name" | grep -c 'md[0-9]*\.nop'`
+	atf_check_equal 2 $NDEVS
+
+	# Now fail the labeled provider
+	atf_check -s exit:0 gnop destroy -f ${md0}.nop
+	# It should be automatically removed from the multipath device
+	NDEVS=`gmultipath list "$name" | grep -c 'md[0-9]*\.nop'`
+	atf_check_equal 1 $NDEVS
+
+	# Now return the labeled provider
+	atf_check gnop create -o $offset /dev/${md0}
+	# It should be automatically restored to the multipath device.  We
+	# don't really care which path is active.
+	NDEVS=`gmultipath list "$name" | grep -c 'md[0-9]*\.nop'`
+	atf_check_equal 2 $NDEVS
+	STATE=`gmultipath list "$name" | awk '/^State:/ {print $2}'`
+	atf_check_equal "OPTIMAL" $STATE
+}
+depart_and_arrive_cleanup()
+{
+	common_cleanup
+}
+
+
+atf_test_case fail cleanup
+fail_head()
+{
+	atf_set "descr" "Manually fail a path"
+	atf_set "require.user" "root"
+}
+fail_body()
+{
+	load_gmultipath
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create "$name" ${md0} ${md1}
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "PASSIVE" 
+	# Manually fail the active path
+	atf_check -s exit:0 gmultipath fail "$name" ${md0}
+	check_multipath_state ${md1} "DEGRADED" "FAIL" "ACTIVE" 
+}
+fail_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case fail_on_error cleanup
+fail_on_error_head()
+{
+	atf_set "descr" "An error in the provider will cause gmultipath to mark it as FAIL"
+	atf_set "require.user" "root"
+}
+fail_on_error_body()
+{
+	load_gnop
+	load_gmultipath
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check gnop create /dev/${md0}
+	atf_check gnop create /dev/${md1}
+	atf_check -s exit:0 gmultipath create "$name" ${md0}.nop ${md1}.nop
+	# The first I/O to the first path should fail, causing gmultipath to
+	# fail over to the second path.
+	atf_check gnop configure -q 100 -r 100 -w 100 -x 100 ${md0}.nop
+	atf_check -s exit:0 -o ignore -e ignore dd if=/dev/zero of=/dev/multipath/"$name" bs=4096 count=1
+	check_multipath_state ${md1}.nop "DEGRADED" "FAIL" "ACTIVE" 
+}
+fail_on_error_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case physpath cleanup
+physpath_head()
+{
+	atf_set "descr" "gmultipath should pass through the underlying providers' physical path"
+	atf_set "require.user" "root"
+}
+physpath_body()
+{
+	load_gnop
+	load_gmultipath
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	physpath="some/physical/path"
+	# Create two providers with the same physical paths, mimicing how
+	# multipathed SAS drives appear.  This is the normal way to use
+	# gmultipath.  If the underlying providers' physical paths differ,
+	# then you're probably using gmultipath wrong.
+	atf_check gnop create -z $physpath /dev/${md0}
+	atf_check gnop create -z $physpath /dev/${md1}
+	atf_check -s exit:0 gmultipath create "$name" ${md0}.nop ${md1}.nop
+	gmultipath_physpath=$(diskinfo -p multipath/"$name") 
+	atf_check_equal "$physpath" "$gmultipath_physpath"
+}
+physpath_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case prefer cleanup
+prefer_head()
+{
+	atf_set "descr" "Manually select the preferred path"
+	atf_set "require.user" "root"
+}
+prefer_body()
+{
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	md2=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create "$name" ${md0} ${md1} ${md2}
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "PASSIVE" "PASSIVE"
+
+	# Explicitly prefer the final path
+	atf_check -s exit:0 gmultipath prefer "$name" ${md2}
+	check_multipath_state ${md2} "OPTIMAL" "PASSIVE" "PASSIVE" "ACTIVE"
+}
+prefer_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case restore cleanup
+restore_head()
+{
+	atf_set "descr" "Manually restore a failed path"
+	atf_set "require.user" "root"
+}
+restore_body()
+{
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create "$name" ${md0} ${md1}
+
+	# Explicitly fail the first path
+	atf_check -s exit:0 gmultipath fail "$name" ${md0}
+	check_multipath_state ${md1} "DEGRADED" "FAIL" "ACTIVE" 
+
+	# Explicitly restore it
+	atf_check -s exit:0 gmultipath restore "$name" ${md0}
+	check_multipath_state ${md1} "OPTIMAL" "PASSIVE" "ACTIVE" 
+}
+restore_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case restore_on_error cleanup
+restore_on_error_head()
+{
+	atf_set "descr" "A failed path should be restored if an I/O error is encountered on all other active paths"
+	atf_set "require.user" "root"
+}
+restore_on_error_body()
+{
+	load_gnop
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	name=$(mkname)
+	atf_check gnop create /dev/${md0}
+	atf_check gnop create /dev/${md1}
+	atf_check -s exit:0 gmultipath create "$name" ${md0}.nop ${md1}.nop
+	# Explicitly fail the first path
+	atf_check -s exit:0 gmultipath fail "$name" ${md0}.nop
+
+	# Setup the second path to fail on the next I/O
+	atf_check gnop configure -r 100 -w 100  ${md1}.nop
+	atf_check -s exit:0 -o ignore -e ignore \
+	    dd if=/dev/zero of=/dev/multipath/"$name" bs=4096 count=1
+
+	# Now the first path should be active, and the second should be failed
+	check_multipath_state ${md0}.nop "DEGRADED" "ACTIVE" "FAIL" 
+}
+restore_on_error_cleanup()
+{
+	common_cleanup
+}
+
+atf_test_case rotate cleanup
+rotate_head()
+{
+	atf_set "descr" "Manually rotate the active path"
+	atf_set "require.user" "root"
+}
+rotate_body()
+{
+	load_gmultipath
+	load_dtrace
+
+	md0=$(alloc_md)
+	md1=$(alloc_md)
+	md2=$(alloc_md)
+	name=$(mkname)
+	atf_check -s exit:0 gmultipath create "$name" ${md0} ${md1} ${md2}
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "PASSIVE" "PASSIVE"
+
+	# Explicitly rotate the paths
+	atf_check -s exit:0 gmultipath rotate "$name"
+	check_multipath_state ${md2} "OPTIMAL" "PASSIVE" "PASSIVE" "ACTIVE"
+	# Again
+	atf_check -s exit:0 gmultipath rotate "$name"
+	check_multipath_state ${md1} "OPTIMAL" "PASSIVE" "ACTIVE" "PASSIVE"
+	# Final rotation should restore original configuration
+	atf_check -s exit:0 gmultipath rotate "$name"
+	check_multipath_state ${md0} "OPTIMAL" "ACTIVE" "PASSIVE" "PASSIVE"
+}
+rotate_cleanup()
+{
+	common_cleanup
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case add
+	atf_add_test_case create_A
+	atf_add_test_case create_R
+	atf_add_test_case depart_and_arrive
+	atf_add_test_case fail
+	atf_add_test_case fail_on_error
+	atf_add_test_case physpath
+	atf_add_test_case prefer
+	atf_add_test_case restore
+	atf_add_test_case restore_on_error
+	atf_add_test_case rotate
+}