Make it possible to build CTL as a module.

Reviewed by:	ken
Sponsored by:	FreeBSD Foundation

share/man/man4/Makefile

@@ -99,6 +99,7 @@ MAN=	aac.4 \
 	${_cpuctl.4} \
 	cpufreq.4 \
 	crypto.4 \
+	ctl.4 \
 	cue.4 \
 	cxgb.4 \
 	cxgbe.4 \

share/man/man4/ctl.4

@@ -0,0 +1,90 @@
+.\" Copyright (c) 2013 Edward Tomasz Napierala
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd March 8, 2013
+.Dt CTL 4
+.Os
+.Sh NAME
+.Nm ctl
+.Nd CAM Target Layer
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following line in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device ctl"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+ctl_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+subsystem provides SCSI disk and processor emulation.
+It supports features such as:
+.Pp
+.Bl -bullet -compact
+.It
+Disk and processor device emulation
+.It
+Tagged queueing
+.It
+SCSI task attribute support (ordered, head of queue, simple tags)
+.It
+SCSI implicit command ordering support.
+.It
+Full task management support (abort, LUN reset, target reset, etc.)
+.It
+Support for multiple ports
+.It
+Support for multiple simultaneous initiators
+.It
+Support for multiple simultaneous backing stores
+.It
+Persistent reservation support
+.It
+Mode sense/select support
+.It
+Error injection support
+.It
+All I/O handled in-kernel, no userland context switch overhead
+.El
+.Sh SEE ALSO
+.Xr ctladm 8 ,
+.Xr ctlstat 8
+.Sh HISTORY
+The
+.Nm
+subsystem first appeared in
+.Fx 9.1 .
+.Sh AUTHORS
+The
+.Nm
+subsystem was written by
+.An Kenneth Merry Aq ken@FreeBSD.org .

sys/cam/ctl/README.ctl.txt

@@ -227,9 +227,6 @@ Revision 1.2 Changes
 To Do List:
 ==========
 
- - Make CTL buildable as a module.  Work needs to be done on initialization,
-   and on freeing resources and LUNs when it is built as a module.
-
  - Use devstat(9) for CTL's statistics collection.  CTL uses a home-grown
    statistics collection system that is similar to devstat(9).  ctlstat
    should be retired in favor of iostat, etc., once aggregation modes are

sys/cam/ctl/ctl.c

@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bio.h>
 #include <sys/fcntl.h>
 #include <sys/lock.h>
+#include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/condvar.h>
 #include <sys/malloc.h>
@@ -338,7 +339,7 @@ TUNABLE_INT("kern.cam.ctl.disable", &ctl_disable);
 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 				  int param);
 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
-static void ctl_init(void);
+static int ctl_init(void);
 void ctl_shutdown(void);
 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
@@ -458,11 +459,16 @@ static struct cdevsw ctl_cdevsw = {
 
 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
 
-/*
- * If we have the CAM SIM, we may or may not have another SIM that will
- * cause CTL to get initialized.  If not, we need to initialize it.
- */
-SYSINIT(ctl_init, SI_SUB_CONFIGURE, SI_ORDER_THIRD, ctl_init, NULL);
+static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
+
+static moduledata_t ctl_moduledata = {
+	"ctl",
+	ctl_module_event_handler,
+	NULL
+};
+
+DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
+MODULE_VERSION(ctl, 1);
 
 static void
 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
@@ -942,7 +948,7 @@ ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
 	dest->io_hdr.status = src->hdr.status;
 }
 
-static void
+static int
 ctl_init(void)
 {
 	struct ctl_softc *softc;
@@ -953,7 +959,7 @@ ctl_init(void)
 #if 0
 	int i;
 #endif
-	int retval;
+	int error, retval;
 	//int isc_retval;
 
 	retval = 0;
@@ -962,7 +968,7 @@ ctl_init(void)
 
 	/* If we're disabled, don't initialize. */
 	if (ctl_disable != 0)
-		return;
+		return (0);
 
 	control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 			       M_WAITOK | M_ZERO);
@@ -991,7 +997,7 @@ ctl_init(void)
 		destroy_dev(softc->dev);
 		free(control_softc, M_DEVBUF);
 		control_softc = NULL;
-		return;
+		return (ENOMEM);
 	}
 
 	SYSCTL_ADD_INT(&softc->sysctl_ctx,
@@ -1053,7 +1059,7 @@ ctl_init(void)
 			    &internal_pool)!= 0){
 		printf("ctl: can't allocate %d entry internal pool, "
 		       "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
-		return;
+		return (ENOMEM);
 	}
 
 	if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
@@ -1061,7 +1067,7 @@ ctl_init(void)
 		printf("ctl: can't allocate %d entry emergency pool, "
 		       "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
 		ctl_pool_free(softc, internal_pool);
-		return;
+		return (ENOMEM);
 	}
 
 	if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
@@ -1071,7 +1077,7 @@ ctl_init(void)
 		       "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
 		ctl_pool_free(softc, internal_pool);
 		ctl_pool_free(softc, emergency_pool);
-		return;
+		return (ENOMEM);
 	}
 
 	softc->internal_pool = internal_pool;
@@ -1092,14 +1098,15 @@ ctl_init(void)
 	mtx_unlock(&softc->ctl_lock);
 #endif
 
-	if (kproc_create(ctl_work_thread, softc, &softc->work_thread, 0, 0,
-			 "ctl_thrd") != 0) {
+	error = kproc_create(ctl_work_thread, softc, &softc->work_thread, 0, 0,
+			 "ctl_thrd");
+	if (error != 0) {
 		printf("error creating CTL work thread!\n");
 		ctl_free_lun(lun);
 		ctl_pool_free(softc, internal_pool);
 		ctl_pool_free(softc, emergency_pool);
 		ctl_pool_free(softc, other_pool);
-		return;
+		return (error);
 	}
 	printf("ctl: CAM Target Layer loaded\n");
 
@@ -1139,10 +1146,11 @@ ctl_init(void)
 	if (sizeof(struct callout) > CTL_TIMER_BYTES) {
 		printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
 		       sizeof(struct callout), CTL_TIMER_BYTES);
-		return;
+		return (EINVAL);
 	}
 #endif /* CTL_IO_DELAY */
 
+	return (0);
 }
 
 void
@@ -1199,6 +1207,20 @@ ctl_shutdown(void)
 	printf("ctl: CAM Target Layer unloaded\n");
 }
 
+static int
+ctl_module_event_handler(module_t mod, int what, void *arg)
+{
+
+	switch (what) {
+	case MOD_LOAD:
+		return (ctl_init());
+	case MOD_UNLOAD:
+		return (EBUSY);
+	default:
+		return (EOPNOTSUPP);
+	}
+}
+
 /*
  * XXX KDM should we do some access checks here?  Bump a reference count to
  * prevent a CTL module from being unloaded while someone has it open?

sys/cam/ctl/ctl_frontend_cam_sim.c

@@ -121,12 +121,23 @@ struct cfcs_softc cfcs_softc;
 static int cfcs_max_sense = sizeof(struct scsi_sense_data);
 extern int ctl_disable;
 
-SYSINIT(cfcs_init, SI_SUB_CONFIGURE, SI_ORDER_FOURTH, cfcs_init, NULL);
 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl2cam, CTLFLAG_RD, 0,
 	    "CAM Target Layer SIM frontend");
 SYSCTL_INT(_kern_cam_ctl2cam, OID_AUTO, max_sense, CTLFLAG_RW,
            &cfcs_max_sense, 0, "Maximum sense data size");
 
+static int cfcs_module_event_handler(module_t, int /*modeventtype_t*/, void *);
+
+static moduledata_t cfcs_moduledata = {
+	"ctlcfcs",
+	cfcs_module_event_handler,
+	NULL
+};
+
+DECLARE_MODULE(ctlcfcs, cfcs_moduledata, SI_SUB_CONFIGURE, SI_ORDER_FOURTH);
+MODULE_VERSION(ctlcfcs, 1);
+MODULE_DEPEND(ctlcfi, ctl, 1, 1, 1);
+MODULE_DEPEND(ctlcfi, cam, 1, 1, 1);
 
 int
 cfcs_init(void)
@@ -176,7 +187,7 @@ cfcs_init(void)
 		printf("%s: ctl_frontend_register() failed with error %d!\n",
 		       __func__, retval);
 		mtx_destroy(&softc->lock);
-		return (1);
+		return (retval);
 	}
 
 	/*
@@ -236,7 +247,7 @@ cfcs_init(void)
 			    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 		printf("%s: error creating path\n", __func__);
 		xpt_bus_deregister(cam_sim_path(softc->sim));
-		retval = 1;
+		retval = EINVAL;
 		goto bailout;
 	}
 
@@ -274,6 +285,20 @@ cfcs_shutdown(void)
 
 }
 
+static int
+cfcs_module_event_handler(module_t mod, int what, void *arg)
+{
+
+	switch (what) {
+	case MOD_LOAD:
+		return (cfcs_init());
+	case MOD_UNLOAD:
+		return (EBUSY);
+	default:
+		return (EOPNOTSUPP);
+	}
+}
+
 static void
 cfcs_onoffline(void *arg, int online)
 {

sys/cam/ctl/ctl_frontend_internal.c

@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/types.h>
 #include <sys/malloc.h>
+#include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/condvar.h>
@@ -189,7 +190,7 @@ MALLOC_DEFINE(M_CTL_CFI, "ctlcfi", "CTL CFI");
 static struct cfi_softc fetd_internal_softc;
 extern int ctl_disable;
 
-void cfi_init(void);
+int cfi_init(void);
 void cfi_shutdown(void) __unused;
 static void cfi_online(void *arg);
 static void cfi_offline(void *arg);
@@ -217,9 +218,19 @@ static void cfi_metatask_io_done(union ctl_io *io);
 static void cfi_err_recovery_done(union ctl_io *io);
 static void cfi_lun_io_done(union ctl_io *io);
 
-SYSINIT(cfi_init, SI_SUB_CONFIGURE, SI_ORDER_FOURTH, cfi_init, NULL);
+static int cfi_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 
-void
+static moduledata_t cfi_moduledata = {
+	"ctlcfi",
+	cfi_module_event_handler,
+	NULL
+};
+
+DECLARE_MODULE(ctlcfi, cfi_moduledata, SI_SUB_CONFIGURE, SI_ORDER_FOURTH);
+MODULE_VERSION(ctlcfi, 1);
+MODULE_DEPEND(ctlcfi, ctl, 1, 1, 1);
+
+int
 cfi_init(void)
 {
 	struct cfi_softc *softc;
@@ -234,7 +245,7 @@ cfi_init(void)
 
 	/* If we're disabled, don't initialize */
 	if (ctl_disable != 0)
-		return;
+		return (0);
 
 	if (sizeof(struct cfi_lun_io) > CTL_PORT_PRIV_SIZE) {
 		printf("%s: size of struct cfi_lun_io %zd > "
@@ -292,7 +303,7 @@ cfi_init(void)
 	}
 bailout:
 
-	return;
+	return (0);
 
 bailout_error:
 
@@ -309,6 +320,8 @@ cfi_init(void)
 	default:
 		break;
 	}
+
+	return (ENOMEM);
 }
 
 void
@@ -331,6 +344,20 @@ cfi_shutdown(void)
 		printf("%s: error shrinking LUN pool\n", __func__);
 }
 
+static int
+cfi_module_event_handler(module_t mod, int what, void *arg)
+{
+
+	switch (what) {
+	case MOD_LOAD:
+		return (cfi_init());
+	case MOD_UNLOAD:
+		return (EBUSY);
+	default:
+		return (EOPNOTSUPP);
+	}
+}
+
 static void
 cfi_online(void *arg)
 {

sys/cam/ctl/scsi_ctl.c

@@ -225,11 +225,28 @@ static struct periph_driver ctlfe_driver =
 	ctlfeinit, "ctl",
 	TAILQ_HEAD_INITIALIZER(ctlfe_driver.units), /*generation*/ 0
 };
-PERIPHDRIVER_DECLARE(ctl, ctlfe_driver);
+
+static int ctlfe_module_event_handler(module_t, int /*modeventtype_t*/, void *);
+
+/*
+ * We're not using PERIPHDRIVER_DECLARE(), because it runs at SI_SUB_DRIVERS,
+ * and that happens before CTL gets initialised.
+ */
+static moduledata_t ctlfe_moduledata = {
+	"ctlfe",
+	ctlfe_module_event_handler,
+	NULL
+};
+
+DECLARE_MODULE(ctlfe, ctlfe_moduledata, SI_SUB_CONFIGURE, SI_ORDER_FOURTH);
+MODULE_VERSION(ctlfe, 1);
+MODULE_DEPEND(ctlfe, ctl, 1, 1, 1);
+MODULE_DEPEND(ctlfe, cam, 1, 1, 1);
 
 extern struct ctl_softc *control_softc;
 extern int ctl_disable;
 
+#ifdef seems_unused
 int
 ctlfeinitialize(void)
 {
@@ -257,6 +274,7 @@ ctlfeinitialize(void)
 
 	return (0);
 }
+#endif
 
 void
 ctlfeshutdown(void)
@@ -288,6 +306,21 @@ ctlfeinit(void)
 	}
 }
 
+static int
+ctlfe_module_event_handler(module_t mod, int what, void *arg)
+{
+
+	switch (what) {
+	case MOD_LOAD:
+		periphdriver_register(&ctlfe_driver);
+		return (0);
+	case MOD_UNLOAD:
+		return (EBUSY);
+	default:
+		return (EOPNOTSUPP);
+	}
+}
+
 static void
 ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
 {
@@ -1971,7 +2004,6 @@ ctlfe_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
 	struct cam_sim *sim;
 	cam_status status;
 
-	
 	bus_softc = (struct ctlfe_softc *)arg;
 	sim = bus_softc->sim;
 

sys/modules/Makefile

@@ -78,6 +78,7 @@ SUBDIR=	\
 	${_cs} \
 	${_ct} \
 	${_ctau} \
+	ctl \
 	${_cxgb} \
 	cxgbe \
 	${_cyclic} \

sys/modules/ctl/Makefile

@@ -0,0 +1,26 @@
+# $FreeBSD$
+
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../cam/ctl
+
+KMOD=	ctl
+
+SRCS=	ctl.c
+SRCS+=	ctl_backend.c
+SRCS+=	ctl_backend_block.c
+SRCS+=	ctl_backend_ramdisk.c
+SRCS+=	ctl_cmd_table.c
+SRCS+=	ctl_frontend.c
+SRCS+=	ctl_frontend_cam_sim.c
+SRCS+=	ctl_frontend_internal.c
+SRCS+=	ctl_mem_pool.c
+SRCS+=	ctl_scsi_all.c
+SRCS+=	ctl_error.c
+SRCS+=	ctl_util.c
+SRCS+=	scsi_ctl.c
+SRCS+=	bus_if.h
+SRCS+=	device_if.h
+SRCS+=	vnode_if.h
+
+.include <bsd.kmod.mk>

usr.bin/ctlstat/ctlstat.8

@@ -34,7 +34,7 @@
 .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
 .\" $FreeBSD$
 .\"
-.Dd June 4, 2010
+.Dd March 6, 2013
 .Dt CTLSTAT 8
 .Os
 .Sh NAME
@@ -113,6 +113,7 @@ every 10 seconds.
 .Sh SEE ALSO
 .Xr cam 3 ,
 .Xr cam 4 ,
+.Xr ctl 4 ,
 .Xr xpt 4 ,
 .Xr camcontrol 8 ,
 .Xr ctladm 8 ,

usr.sbin/ctladm/ctladm.8

@@ -34,7 +34,7 @@
 .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.sbin/ctladm/ctladm.8#3 $
 .\" $FreeBSD$
 .\"
-.Dd March 6, 2012
+.Dd March 6, 2013
 .Dt CTLADM 8
 .Os
 .Sh NAME
@@ -975,6 +975,7 @@ This will result in a sense key of NOT READY (0x02), and an ASC/ASCQ of
 .Xr cam 3 ,
 .Xr cam_cdbparse 3 ,
 .Xr cam 4 ,
+.Xr ctl 4 ,
 .Xr xpt 4 ,
 .Xr camcontrol 8
 .Sh HISTORY