powerpc: Implement fpu_kern_enter/fpu_kern_leave

Summary:
Provide an implementation of fpu_kern_enter/fpu_kern_leave for PPC to
enable FPU, VSX, and Altivec usage in-kernel. The functions currently
only support FPU_KERN_NOCTX, but this is sufficient for ossl(1) and many
other users of the API.

This patchset has been tested on powerpc64le using a modified version of
the in-tree tools/tools/crypto/cryptocheck.c tool to check for FPU/Vec
register clobbering along with a follow-up patch to enable ossl(4) on
powerpc64*.

Reviewed by:	jhibbits
MFC after:	2 weeks
Differential Revision: https://reviews.freebsd.org/D41540

share/man/man9/fpu_kern.9

@@ -185,7 +185,8 @@ and false otherwise.
 .Sh NOTES
 The
 .Nm
-is currently implemented only for the i386, amd64, and arm64 architectures.
+is currently implemented only for the i386, amd64, arm64, and powerpc
+architectures.
 .Pp
 There is no way to handle floating point exceptions raised from
 kernel mode.
@@ -205,6 +206,8 @@ facitily and this manual page were written by
 .An Konstantin Belousov Aq Mt kib@FreeBSD.org .
 The arm64 support was added by
 .An Andrew Turner Aq Mt andrew@FreeBSD.org .
+The powerpc support was added by
+.An Shawn Anastasio Aq Mt sanastasio@raptorengineering.com .
 .Sh BUGS
 .Fn fpu_kern_leave
 should probably have type

sys/powerpc/include/altivec.h

@@ -35,5 +35,7 @@
 void    enable_vec(struct thread *);
 void    save_vec(struct thread *);
 void    save_vec_nodrop(struct thread *);
+void    enable_vec_kern(void);
+void    disable_vec(struct thread *td);
 
 #endif	/* _MACHINE_ALTIVEC_H_ */

sys/powerpc/include/fpu.h

@@ -76,6 +76,26 @@ void    save_fpu(struct thread *);
 void    save_fpu_nodrop(struct thread *);
 void    cleanup_fpscr(void);
 u_int   get_fpu_exception(struct thread *);
+void    enable_fpu_kern(void);
+void    disable_fpu(struct thread *td);
+
+/*
+ * Flags for fpu_kern_alloc_ctx(), fpu_kern_enter() and fpu_kern_thread().
+ */
+#define	FPU_KERN_NORMAL	0x0000
+#define	FPU_KERN_NOWAIT	0x0001
+#define	FPU_KERN_KTHR	0x0002
+#define	FPU_KERN_NOCTX	0x0004
+
+struct fpu_kern_ctx;
+
+struct fpu_kern_ctx *fpu_kern_alloc_ctx(u_int flags);
+void	fpu_kern_free_ctx(struct fpu_kern_ctx *ctx);
+void	fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx,
+	    u_int flags);
+int	fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx);
+int	fpu_kern_thread(u_int flags);
+int	is_fpu_kern_thread(u_int flags);
 
 #endif /* _KERNEL */
 

sys/powerpc/include/pcb.h

@@ -48,7 +48,7 @@ struct pcb {
 	register_t	pcb_toc;		/* toc pointer */
 	register_t	pcb_lr;			/* link register */
 	register_t	pcb_dscr;		/* dscr value */
-	register_t	pcb_fscr;		
+	register_t	pcb_fscr;
 	register_t	pcb_tar;
 	struct		pmap *pcb_pm;		/* pmap of our vmspace */
 	jmp_buf		*pcb_onfault;		/* For use during
@@ -56,11 +56,14 @@ struct pcb {
 	int		pcb_flags;
 #define	PCB_FPU		0x1	/* Process uses FPU */
 #define	PCB_FPREGS	0x2	/* Process had FPU registers initialized */
-#define	PCB_VEC		0x4	/* Process had Altivec initialized */
+#define	PCB_VEC		0x4	/* Process uses Altivec */
 #define	PCB_VSX		0x8	/* Process had VSX initialized */
 #define	PCB_CDSCR	0x10	/* Process had Custom DSCR initialized */
 #define	PCB_HTM		0x20	/* Process had HTM initialized */
 #define	PCB_CFSCR	0x40	/* Process had FSCR updated */
+#define	PCB_KERN_FPU    0x80	/* Kernel is using FPU/Vector unit */
+#define	PCB_KERN_FPU_NOSAVE 0x100 /* FPU/Vec state not saved for kernel use */
+#define	PCB_VECREGS     0x200	/* Process had Altivec registers initialized */
 	struct fpu {
 		union {
 #if _BYTE_ORDER == _BIG_ENDIAN

sys/powerpc/powerpc/altivec.c

@@ -105,10 +105,11 @@ enable_vec(struct thread *td)
 	 * the thread, initialise the vector registers and VSCR to 0, and
 	 * set the flag to indicate that the vector unit is in use.
 	 */
+	pcb->pcb_flags |= PCB_VEC;
 	tf->srr1 |= PSL_VEC;
-	if (!(pcb->pcb_flags & PCB_VEC)) {
+	if (!(pcb->pcb_flags & PCB_VECREGS)) {
 		memset(&pcb->pcb_vec, 0, sizeof pcb->pcb_vec);
-		pcb->pcb_flags |= PCB_VEC;
+		pcb->pcb_flags |= PCB_VECREGS;
 	}
 
 	/*
@@ -170,3 +171,32 @@ save_vec_nodrop(struct thread *td)
 	if (td == PCPU_GET(vecthread))
 		save_vec_int(td);
 }
+
+void
+enable_vec_kern(void)
+{
+	mtmsr(mfmsr() | PSL_VEC);
+}
+
+void
+disable_vec(struct thread *td)
+{
+	register_t msr;
+	struct pcb *pcb;
+	struct trapframe *tf;
+
+	pcb = td->td_pcb;
+	tf = trapframe(td);
+
+	/* Disable PSL_VEC in kernel (if enabled) */
+	msr = mfmsr() & ~PSL_VEC;
+	isync();
+	mtmsr(msr);
+
+	/*
+	 * Disable PSL_VEC in userspace. It will be re-enabled when
+	 * an Altivec instruction is executed.
+	 */
+	tf->srr1 &= ~PSL_VEC;
+	pcb->pcb_flags &= ~PCB_VEC;
+}

sys/powerpc/powerpc/exec_machdep.c

@@ -441,12 +441,14 @@ grab_mcontext(struct thread *td, mcontext_t *mcp, int flags)
 	 * Repeat for Altivec context
 	 */
 
-	if (pcb->pcb_flags & PCB_VEC) {
-		KASSERT(td == curthread,
-			("get_mcontext: fp save not curthread"));
-		critical_enter();
-		save_vec(td);
-		critical_exit();
+	if (pcb->pcb_flags & PCB_VECREGS) {
+		if (pcb->pcb_flags & PCB_VEC) {
+			KASSERT(td == curthread,
+				("get_mcontext: altivec save not curthread"));
+			critical_enter();
+			save_vec(td);
+			critical_exit();
+		}
 		mcp->mc_flags |= _MC_AV_VALID;
 		mcp->mc_vscr  = pcb->pcb_vec.vscr;
 		mcp->mc_vrsave =  pcb->pcb_vec.vrsave;
@@ -543,11 +545,8 @@ set_mcontext(struct thread *td, mcontext_t *mcp)
 	}
 
 	if (mcp->mc_flags & _MC_AV_VALID) {
-		if ((pcb->pcb_flags & PCB_VEC) != PCB_VEC) {
-			critical_enter();
-			enable_vec(td);
-			critical_exit();
-		}
+		/* enable_vec() will happen lazily on a fault */
+		pcb->pcb_flags |= PCB_VECREGS;
 		pcb->pcb_vec.vscr = mcp->mc_vscr;
 		pcb->pcb_vec.vrsave = mcp->mc_vrsave;
 		memcpy(pcb->pcb_vec.vr, mcp->mc_avec, sizeof(mcp->mc_avec));

sys/powerpc/powerpc/fpu.c

@@ -42,6 +42,7 @@
 #include <machine/fpu.h>
 #include <machine/pcb.h>
 #include <machine/psl.h>
+#include <machine/altivec.h>
 
 static void
 save_fpu_int(struct thread *td)
@@ -259,3 +260,127 @@ get_fpu_exception(struct thread *td)
 	return ucode;
 }
 
+void
+enable_fpu_kern(void)
+{
+	register_t msr;
+
+	msr = mfmsr() | PSL_FP;
+
+	if (cpu_features & PPC_FEATURE_HAS_VSX)
+		msr |= PSL_VSX;
+
+	mtmsr(msr);
+}
+
+void
+disable_fpu(struct thread *td)
+{
+	register_t msr;
+	struct pcb *pcb;
+	struct trapframe *tf;
+
+	pcb = td->td_pcb;
+	tf = trapframe(td);
+
+	/* Disable FPU in kernel (if enabled) */
+	msr = mfmsr() & ~(PSL_FP | PSL_VSX);
+	isync();
+	mtmsr(msr);
+
+	/*
+	 * Disable FPU in userspace. It will be re-enabled when
+	 * an FP or VSX instruction is executed.
+	 */
+	tf->srr1 &= ~(PSL_FP | PSL_VSX);
+	pcb->pcb_flags &= ~(PCB_FPU | PCB_VSX);
+}
+
+#ifndef __SPE__
+/*
+ * XXX: Implement fpu_kern_alloc_ctx/fpu_kern_free_ctx once fpu_kern_enter and
+ * fpu_kern_leave can handle !FPU_KERN_NOCTX.
+ */
+struct fpu_kern_ctx {
+#define	FPU_KERN_CTX_DUMMY	0x01	/* avoided save for the kern thread */
+#define	FPU_KERN_CTX_INUSE	0x02
+	uint32_t	 flags;
+};
+
+void
+fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
+{
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+
+	KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL,
+	    ("ctx is required when !FPU_KERN_NOCTX"));
+	KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0,
+	    ("using inuse ctx"));
+	KASSERT((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) == 0,
+	    ("recursive fpu_kern_enter while in PCB_KERN_FPU_NOSAVE state"));
+
+	if ((flags & FPU_KERN_NOCTX) != 0) {
+		critical_enter();
+
+		if (pcb->pcb_flags & PCB_FPU) {
+			save_fpu(td);
+			pcb->pcb_flags |= PCB_FPREGS;
+		}
+		enable_fpu_kern();
+
+		if (pcb->pcb_flags & PCB_VEC) {
+			save_vec(td);
+			pcb->pcb_flags |= PCB_VECREGS;
+		}
+		enable_vec_kern();
+
+		pcb->pcb_flags |= PCB_KERN_FPU | PCB_KERN_FPU_NOSAVE;
+		return;
+	}
+
+	KASSERT(0, ("fpu_kern_enter with !FPU_KERN_NOCTX not implemented!"));
+}
+
+int
+fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
+{
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+
+	if ((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) != 0) {
+		KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX"));
+		KASSERT(PCPU_GET(fpcurthread) == NULL,
+		    ("non-NULL fpcurthread for PCB_FP_NOSAVE"));
+		CRITICAL_ASSERT(td);
+
+		/* Disable FPU, VMX, and VSX */
+		disable_fpu(td);
+		disable_vec(td);
+
+		pcb->pcb_flags &= ~PCB_KERN_FPU_NOSAVE;
+
+		critical_exit();
+	} else {
+		KASSERT(0, ("fpu_kern_leave with !FPU_KERN_NOCTX not implemented!"));
+	}
+
+	pcb->pcb_flags &= ~PCB_KERN_FPU;
+
+	return 0;
+}
+
+int
+is_fpu_kern_thread(u_int flags __unused)
+{
+	struct pcb *curpcb;
+
+	if ((curthread->td_pflags & TDP_KTHREAD) == 0)
+		return (0);
+	curpcb = curthread->td_pcb;
+	return ((curpcb->pcb_flags & PCB_KERN_FPU) != 0);
+}
+
+#endif /* !__SPE__ */

sys/sys/param.h

@@ -75,7 +75,7 @@
  * cannot include sys/param.h and should only be updated here.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1500000
+#define __FreeBSD_version 1500001
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,