From 9923b511ed09f7e9aff331c1de463c09bf9af55e Mon Sep 17 00:00:00 2001
From: Scott Long <scottl@FreeBSD.org>
Date: Thu, 2 Sep 2004 18:59:15 +0000
Subject: [PATCH] Turn PREEMPTION into a kernel option.  Make sure that it's
 defined if FULL_PREEMPTION is defined.  Add a runtime warning to ULE if
 PREEMPTION is enabled (code inspired by the PREEMPTION warning in
 kern_switch.c).  This is a possible MT5 candidate.

---
 sys/alpha/include/param.h |  5 -----
 sys/amd64/include/param.h |  5 -----
 sys/conf/NOTES            |  7 ++++++-
 sys/conf/options          |  3 ++-
 sys/i386/include/param.h  |  5 -----
 sys/kern/kern_mutex.c     |  1 +
 sys/kern/kern_shutdown.c  |  1 +
 sys/kern/kern_switch.c    |  6 +++++-
 sys/kern/sched_ule.c      | 14 ++++++++++++++
 sys/vm/vm_zeroidle.c      |  2 ++
 10 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/sys/alpha/include/param.h b/sys/alpha/include/param.h
index 6e36e70db9ab..1a9c9af545a5 100644
--- a/sys/alpha/include/param.h
+++ b/sys/alpha/include/param.h
@@ -113,11 +113,6 @@
 #define	SSIZE		1		/* initial stack size/NBPG */
 #define	SINCR		1		/* increment of stack/NBPG */
 
-/* PREEMPTION exposes scheduler bugs that need to be fixed. */
-#if 0
-#define	PREEMPTION
-#endif
-
 #ifndef	KSTACK_PAGES
 #define	KSTACK_PAGES	2		/* pages of kstack (with pcb) */
 #endif
diff --git a/sys/amd64/include/param.h b/sys/amd64/include/param.h
index aa8c29e95c1c..5216c55a28dc 100644
--- a/sys/amd64/include/param.h
+++ b/sys/amd64/include/param.h
@@ -119,11 +119,6 @@
 #define	NBPML4		(1ul<<PML4SHIFT)/* bytes/page map lev4 table */
 #define	PML4MASK	(NBPML4-1)
 
-/* PREEMPTION exposes scheduler bugs that need to be fixed. */
-#if 0
-#define	PREEMPTION
-#endif
-
 #define IOPAGES	2		/* pages of i/o permission bitmap */
 
 #ifndef	KSTACK_PAGES
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index 60300f5ba927..43a870906202 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -196,12 +196,16 @@ options 	MUTEX_WAKE_ALL
 
 # SMP Debugging Options:
 #
+# PREEMPTION allows the threads that are in the kernel to be preempted
+#	  by higher priority threads.  It helps with interactivity and
+#	  allows interrupt threads to run sooner rather than waiting.
+#	  WARNING! Only tested on alpha, amd64, and i386.
 # FULL_PREEMPTION instructs the kernel to preempt non-realtime kernel
 #	  threads.  It sole use is to expose race conditions and other
 #	  bugs during development.  Enabling this option will reduce
 #	  performance and increase the frequency of kernel panics by
 #	  design.  If you aren't sure that you need it then you don't.
-#         DON'T TURN THIS ON.
+#	  Relies on the PREEMPTION option.  DON'T TURN THIS ON.
 # MUTEX_DEBUG enables various extra assertions in the mutex code.
 # SLEEPQUEUE_PROFILING enables rudimentary profiling of the hash table
 #	  used to hold active sleep queues.
@@ -213,6 +217,7 @@ options 	MUTEX_WAKE_ALL
 #	  a lock hierarchy violation occurs or if locks are held when going to
 #	  sleep.
 # WITNESS_SKIPSPIN disables the witness checks on spin mutexes.
+options		PREEMPTION
 options 	FULL_PREEMPTION
 options 	MUTEX_DEBUG
 options 	WITNESS
diff --git a/sys/conf/options b/sys/conf/options
index eee12c97b2ea..f7ac6d25b5a2 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -66,7 +66,8 @@ COMPILING_LINT	opt_global.h
 CONSPEED	opt_comconsole.h
 CY_PCI_FASTINTR
 DIRECTIO	opt_directio.h
-FULL_PREEMPTION
+FULL_PREEMPTION	opt_sched.h
+PREEMPTION	opt_sched.h
 GEOM_AES	opt_geom.h
 GEOM_APPLE	opt_geom.h
 GEOM_BDE	opt_geom.h
diff --git a/sys/i386/include/param.h b/sys/i386/include/param.h
index bb9c7a7df750..a4064e3d9d4c 100644
--- a/sys/i386/include/param.h
+++ b/sys/i386/include/param.h
@@ -97,11 +97,6 @@
 #define NBPDR		(1<<PDRSHIFT)	/* bytes/page dir */
 #define PDRMASK		(NBPDR-1)
 
-/* PREEMPTION exposes scheduler bugs that need to be fixed. */
-#if 0
-#define	PREEMPTION
-#endif
-
 #define IOPAGES	2		/* pages of i/o permission bitmap */
 
 #ifndef KSTACK_PAGES
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index c041886663ea..eb345bff2d3f 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_ddb.h"
 #include "opt_mprof.h"
 #include "opt_mutex_wake_all.h"
+#include "opt_sched.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index a720becb7e13..25e81c4e21fe 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_mac.h"
 #include "opt_panic.h"
 #include "opt_show_busybufs.h"
+#include "opt_sched.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c
index e36128f702c9..69cfb5f5bae1 100644
--- a/sys/kern/kern_switch.c
+++ b/sys/kern/kern_switch.c
@@ -88,7 +88,6 @@ reassigned to keep this true.
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_full_preemption.h"
 #include "opt_sched.h"
 
 #include <sys/param.h>
@@ -109,6 +108,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #endif
 
+#ifdef FULL_PREEMPTION
+#ifndef PREEMPTION
+#error "The FULL_PREEMPTION option requires the PREEMPTION option"
+#endif
+#endif
 
 CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS);
 
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 5582a40f0d90..e235c1f9b752 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -27,6 +27,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <opt_sched.h>
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kdb.h>
@@ -75,6 +77,18 @@ SYSCTL_INT(_kern_sched, OID_AUTO, slice_max, CTLFLAG_RW, &slice_max, 0, "");
 int realstathz;
 int tickincr = 1;
 
+#ifdef PREEMPTION
+static void
+printf_caddr_t(void *data)
+{
+	printf("%s", (char *)data);
+}
+static char preempt_warning[] =
+    "WARNING: Kernel PREEMPTION is unstable under SCHED_ULE.\n"; 
+SYSINIT(preempt_warning, SI_SUB_COPYRIGHT, SI_ORDER_ANY, printf_caddr_t,
+    preempt_warning)
+#endif
+
 /*
  * These datastructures are allocated within their parent datastructure but
  * are scheduler specific.
diff --git a/sys/vm/vm_zeroidle.c b/sys/vm/vm_zeroidle.c
index 6e50b6b581b1..9d779de097f2 100644
--- a/sys/vm/vm_zeroidle.c
+++ b/sys/vm/vm_zeroidle.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <opt_sched.h>
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>