Make it possible to re-evaluate cpu_features.

Add cpuctl(4) ioctl CPUCTL_EVAL_CPU_FEATURES which forces re-read of
cpu_features, cpu_features2, cpu_stdext_features, and
std_stdext_features2.

The intent is to allow the kernel to see the changes in the CPU
features after micocode update.  Of course, the update is not atomic
across variables and not synchronized with readers.  See the man page
warning as well.

Reviewed by:	imp (previous version), jilles
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D13770

sys/amd64/amd64/machdep.c

@@ -1535,7 +1535,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 
 	kmdp = init_ops.parse_preload_data(modulep);
 
-	identify_cpu();
+	identify_cpu1();
 	identify_hypervisor();
 
 	/* Init basic tunables, hz etc */

sys/dev/cpuctl/cpuctl.c

@@ -73,6 +73,7 @@ static int cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data,
     struct thread *td);
 static int cpuctl_do_cpuid_count(int cpu, cpuctl_cpuid_count_args_t *data,
     struct thread *td);
+static int cpuctl_do_eval_cpu_features(int cpu, struct thread *td);
 static int cpuctl_do_update(int cpu, cpuctl_update_args_t *data,
     struct thread *td);
 static int update_intel(int cpu, cpuctl_update_args_t *args,
@@ -159,7 +160,8 @@ cpuctl_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
 	}
 	/* Require write flag for "write" requests. */
 	if ((cmd == CPUCTL_MSRCBIT || cmd == CPUCTL_MSRSBIT ||
-	    cmd == CPUCTL_UPDATE || cmd == CPUCTL_WRMSR) &&
+	    cmd == CPUCTL_UPDATE || cmd == CPUCTL_WRMSR ||
+	    cmd == CPUCTL_EVAL_CPU_FEATURES) &&
 	    (flags & FWRITE) == 0)
 		return (EPERM);
 	switch (cmd) {
@@ -187,6 +189,9 @@ cpuctl_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
 		ret = cpuctl_do_cpuid_count(cpu,
 		    (cpuctl_cpuid_count_args_t *)data, td);
 		break;
+	case CPUCTL_EVAL_CPU_FEATURES:
+		ret = cpuctl_do_eval_cpu_features(cpu, td);
+		break;
 	default:
 		ret = EINVAL;
 		break;
@@ -504,6 +509,29 @@ update_via(int cpu, cpuctl_update_args_t *args, struct thread *td)
 	return (ret);
 }
 
+static int
+cpuctl_do_eval_cpu_features(int cpu, struct thread *td)
+{
+	int is_bound = 0;
+	int oldcpu;
+
+	KASSERT(cpu >= 0 && cpu <= mp_maxid,
+	    ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu));
+
+#ifdef __i386__
+	if (cpu_id == 0)
+		return (ENODEV);
+#endif
+	oldcpu = td->td_oncpu;
+	is_bound = cpu_sched_is_bound(td);
+	set_cpu(cpu, td);
+	identify_cpu1();
+	identify_cpu2();
+	restore_cpu(oldcpu, is_bound, td);
+	return (0);
+}
+
+
 int
 cpuctl_open(struct cdev *dev, int flags, int fmt __unused, struct thread *td)
 {

sys/sys/cpuctl.h

@@ -59,5 +59,6 @@ typedef struct {
 #define	CPUCTL_MSRSBIT	_IOWR('c', 5, cpuctl_msr_args_t)
 #define	CPUCTL_MSRCBIT	_IOWR('c', 6, cpuctl_msr_args_t)
 #define	CPUCTL_CPUID_COUNT _IOWR('c', 7, cpuctl_cpuid_count_args_t)
+#define	CPUCTL_EVAL_CPU_FEATURES	_IO('c', 8)
 
 #endif /* _CPUCTL_H_ */

sys/x86/include/x86_var.h

@@ -119,7 +119,8 @@ void	cpu_setregs(void);
 void	dump_add_page(vm_paddr_t);
 void	dump_drop_page(vm_paddr_t);
 void	finishidentcpu(void);
-void	identify_cpu(void);
+void	identify_cpu1(void);
+void	identify_cpu2(void);
 void	identify_hypervisor(void);
 void	initializecpu(void);
 void	initializecpucache(void);

sys/x86/x86/identcpu.c

@@ -1385,9 +1385,8 @@ fix_cpuid(void)
 	return (false);
 }
 
-#ifdef __amd64__
 void
-identify_cpu(void)
+identify_cpu1(void)
 {
 	u_int regs[4];
 
@@ -1404,7 +1403,29 @@ identify_cpu(void)
 	cpu_feature = regs[3];
 	cpu_feature2 = regs[2];
 }
-#endif
+
+void
+identify_cpu2(void)
+{
+	u_int regs[4], cpu_stdext_disable;
+
+	if (cpu_high >= 7) {
+		cpuid_count(7, 0, regs);
+		cpu_stdext_feature = regs[1];
+
+		/*
+		 * Some hypervisors failed to filter out unsupported
+		 * extended features.  Allow to disable the
+		 * extensions, activation of which requires setting a
+		 * bit in CR4, and which VM monitors do not support.
+		 */
+		cpu_stdext_disable = 0;
+		TUNABLE_INT_FETCH("hw.cpu_stdext_disable", &cpu_stdext_disable);
+		cpu_stdext_feature &= ~cpu_stdext_disable;
+
+		cpu_stdext_feature2 = regs[2];
+	}
+}
 
 /*
  * Final stage of CPU identification.
@@ -1412,7 +1433,7 @@ identify_cpu(void)
 void
 finishidentcpu(void)
 {
-	u_int regs[4], cpu_stdext_disable;
+	u_int regs[4];
 #ifdef __i386__
 	u_char ccr3;
 #endif
@@ -1431,22 +1452,7 @@ finishidentcpu(void)
 		cpu_mon_max_size = regs[1] &  CPUID5_MON_MAX_SIZE;
 	}
 
-	if (cpu_high >= 7) {
-		cpuid_count(7, 0, regs);
-		cpu_stdext_feature = regs[1];
-
-		/*
-		 * Some hypervisors failed to filter out unsupported
-		 * extended features.  Allow to disable the
-		 * extensions, activation of which requires setting a
-		 * bit in CR4, and which VM monitors do not support.
-		 */
-		cpu_stdext_disable = 0;
-		TUNABLE_INT_FETCH("hw.cpu_stdext_disable", &cpu_stdext_disable);
-		cpu_stdext_feature &= ~cpu_stdext_disable;
-
-		cpu_stdext_feature2 = regs[2];
-	}
+	identify_cpu2();
 
 #ifdef __i386__
 	if (cpu_high > 0 &&

usr.sbin/cpucontrol/cpucontrol.8

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 30, 2017
+.Dd January 5, 2018
 .Dt CPUCONTROL 8
 .Os
 .Sh NAME
@@ -34,46 +34,51 @@
 device
 .Sh SYNOPSIS
 .Nm
+.Bk
 .Op Fl v
 .Fl m Ar msr
-.Bk
 .Ar device
 .Ek
+.Bk
 .Nm
 .Op Fl v
 .Fl m Ar msr Ns = Ns Ar value
-.Bk
 .Ar device
 .Ek
+.Bk
 .Nm
 .Op Fl v
 .Fl m Ar msr Ns &= Ns Ar mask
-.Bk
 .Ar device
 .Ek
+.Bk
 .Nm
 .Op Fl v
 .Fl m Ar msr Ns |= Ns Ar mask
-.Bk
 .Ar device
 .Ek
+.Bk
 .Nm
 .Op Fl v
 .Fl i Ar level
-.Bk
 .Ar device
 .Ek
+.Bk
 .Nm
 .Op Fl v
 .Fl i Ar level,level_type
-.Bk
 .Ar device
 .Ek
+.Bk
 .Nm
 .Op Fl vn
 .Op Fl d Ar datadir
 .Fl u
+.Ar device
+.Ek
 .Bk
+.Nm
+.Fl e
 .Ar device
 .Ek
 .Sh DESCRIPTION
@@ -136,6 +141,20 @@ The
 .Nm
 utility will walk through the configured data directories
 and apply all firmware updates available for this CPU.
+.It Fl e
+Re-evaluate the kernel flags indicating the present CPU features.
+This command is typically executed after a firmware update was applied
+which changes information reported by the
+.Dv CPUID
+instruction.
+.Pp
+.Bf -symbolic
+Only execute the
+.Fl e
+command after the microcode update was applied to all CPUs in the system.
+The kernel does not operate correctly if the features of processors are
+not identical.
+.Ef
 .It Fl v
 Increase the verbosity level.
 .It Fl h

usr.sbin/cpucontrol/cpucontrol.c

@@ -63,6 +63,7 @@ int	verbosity_level = 0;
 #define	FLAG_M	0x02
 #define	FLAG_U	0x04
 #define	FLAG_N	0x08
+#define	FLAG_E	0x10
 
 #define	OP_INVAL	0x00
 #define	OP_READ		0x01
@@ -117,7 +118,7 @@ usage(void)
 	if (name == NULL)
 		name = "cpuctl";
 	fprintf(stderr, "Usage: %s [-vh] [-d datadir] [-m msr[=value] | "
-	    "-i level | -i level,level_type | -u] device\n", name);
+	    "-i level | -i level,level_type | -e | -u] device\n", name);
 	exit(EX_USAGE);
 }
 
@@ -340,6 +341,25 @@ do_msr(const char *cmdarg, const char *dev)
 	return (0);
 }
 
+static int
+do_eval_cpu_features(const char *dev)
+{
+	int fd, error;
+
+	assert(dev != NULL);
+
+	fd = open(dev, O_RDWR);
+	if (fd < 0) {
+		WARN(0, "error opening %s for writing", dev);
+		return (1);
+	}
+	error = ioctl(fd, CPUCTL_EVAL_CPU_FEATURES, NULL);
+	if (error < 0)
+		WARN(0, "ioctl(%s, CPUCTL_EVAL_CPU_FEATURES)", dev);
+	close(fd);
+	return (error);
+}
+
 static int
 do_update(const char *dev)
 {
@@ -430,11 +450,14 @@ main(int argc, char *argv[])
 	error = 0;
 	cmdarg = "";	/* To keep gcc3 happy. */
 
-	while ((c = getopt(argc, argv, "d:hi:m:nuv")) != -1) {
+	while ((c = getopt(argc, argv, "d:ehi:m:nuv")) != -1) {
 		switch (c) {
 		case 'd':
 			datadir_add(optarg);
 			break;
+		case 'e':
+			flags |= FLAG_E;
+			break;
 		case 'i':
 			flags |= FLAG_I;
 			cmdarg = optarg;
@@ -468,22 +491,25 @@ main(int argc, char *argv[])
 	if ((flags & FLAG_N) == 0)
 		datadir_add(DEFAULT_DATADIR);
 	dev = argv[0];
-	c = flags & (FLAG_I | FLAG_M | FLAG_U);
+	c = flags & (FLAG_E | FLAG_I | FLAG_M | FLAG_U);
 	switch (c) {
-		case FLAG_I:
-			if (strstr(cmdarg, ",") != NULL)
-				error = do_cpuid_count(cmdarg, dev);
-			else
-				error = do_cpuid(cmdarg, dev);
-			break;
-		case FLAG_M:
-			error = do_msr(cmdarg, dev);
-			break;
-		case FLAG_U:
-			error = do_update(dev);
-			break;
-		default:
-			usage();	/* Only one command can be selected. */
+	case FLAG_I:
+		if (strstr(cmdarg, ",") != NULL)
+			error = do_cpuid_count(cmdarg, dev);
+		else
+			error = do_cpuid(cmdarg, dev);
+		break;
+	case FLAG_M:
+		error = do_msr(cmdarg, dev);
+		break;
+	case FLAG_U:
+		error = do_update(dev);
+		break;
+	case FLAG_E:
+		error = do_eval_cpu_features(dev);
+		break;
+	default:
+		usage();	/* Only one command can be selected. */
 	}
 	SLIST_FREE(&datadirs, next, free);
 	return (error == 0 ? 0 : 1);