powerpc: Adjust priority NOPs, and make them functions

PowerISA 2.07 and PowerISA 3.0 both specify special NOPs for priority
adjustments, with "medium" priority being normal.  We had been setting
medium-low as our normal priority.  Rather than guess each time as to what
we want and the right NOP, wrap them in inline functions, and replace the
occurrances of the NOPs with the functions.  Also, make DELAY() drop to very
low priority while waiting, so we don't burn CPU.

Coupled with r346143, this shaves off a modest 5-8% on buildworld times with
-j72.  There may be more room for improvement with judicious use of these
NOPs.

MFC after:	2 weeks

sys/powerpc/include/cpufunc.h

@@ -212,6 +212,43 @@ get_pcpu(void)
 	return (ret);
 }
 
+/* "NOP" operations to signify priorities to the kernel. */
+static __inline void
+nop_prio_vlow(void)
+{
+	__asm __volatile("or 31,31,31");
+}
+
+static __inline void
+nop_prio_low(void)
+{
+	__asm __volatile("or 1,1,1");
+}
+
+static __inline void
+nop_prio_mlow(void)
+{
+	__asm __volatile("or 6,6,6");
+}
+
+static __inline void
+nop_prio_medium(void)
+{
+	__asm __volatile("or 2,2,2");
+}
+
+static __inline void
+nop_prio_mhigh(void)
+{
+	__asm __volatile("or 5,5,5");
+}
+
+static __inline void
+nop_prio_high(void)
+{
+	__asm __volatile("or 3,3,3");
+}
+
 #endif /* _KERNEL */
 
 #endif /* !_MACHINE_CPUFUNC_H_ */

sys/powerpc/powerpc/machdep.c

@@ -494,7 +494,7 @@ spinlock_enter(void)
 
 	td = curthread;
 	if (td->td_md.md_spinlock_count == 0) {
-		__asm __volatile("or 2,2,2"); /* Set high thread priority */
+		nop_prio_mhigh();
 		msr = intr_disable();
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_msr = msr;
@@ -515,7 +515,7 @@ spinlock_exit(void)
 	td->td_md.md_spinlock_count--;
 	if (td->td_md.md_spinlock_count == 0) {
 		intr_restore(msr);
-		__asm __volatile("or 6,6,6"); /* Set normal thread priority */
+		nop_prio_medium();
 	}
 }
 

sys/powerpc/powerpc/mp_machdep.c

@@ -78,8 +78,8 @@ machdep_ap_bootstrap(void)
 	__asm __volatile("msync; isync");
 
 	while (ap_letgo == 0)
-		__asm __volatile("or 31,31,31");
-	__asm __volatile("or 6,6,6");
+		nop_prio_vlow();
+	nop_prio_medium();
 
 	/*
 	 * Set timebase as soon as possible to meet an implicit rendezvous