o Put the syscall return registers in the context. Not only do we

need this for swapcontext(), KSE upcalls initiated from ast()
   also need to save them so that we properly return the syscall
   results after having had a context switch. Note that we don't
   use r11 in the kernel. However, the runtime specification has
   defined r8-r11 as return registers, so we put r11 in the context
   as well. I think deischen@ was trying to tell me that we should
   save the return registers before. I just wasn't ready for it :-)

o  The EPC syscall code has 2 return registers and 2 frame markers
   to save. The first (rp/pfs) belongs to the syscall stub itself.
   The second (iip/cfm) belongs to the caller of the syscall stub.
   We want to put the second in the context (note that iip and cfm
   relate to interrupts. They are only being misused by the syscall
   code, but are not part of a regular context).
   This way, when the context is switched to again, we return to
   the caller of setcontext(2) as one would expect.

o  Deal with dirty registers on the kernel stack. The getcontext()
   syscall will flush the RSE, so we don't expect any dirty registers
   in that case. However, in thread_userret() we also need to save
   the context in certain cases. When that happens, we are sure that
   there are dirty registers on the kernel stack.
   This implementation simply copies the registers, one at a time,
   from the kernel stack to the user stack. NAT collections are not
   dealt with. Hence we don't preserve NaT bits. A better solution
   needs to be found at some later time.
   We also don't deal with this in all cases in set_mcontext. No
   temporay solution is implemented because it's not a showstopper.
   The problem is that we need to ignore the dirty registers and we
   automaticly do that for at most 62 registers. When there are more
   than 62 dirty registers we have a memory "leak".

This commit is fundamental for KSE support.

sys/ia64/ia64/machdep.c

@@ -1038,29 +1038,73 @@ freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
 int
 get_mcontext(struct thread *td, mcontext_t *mc, int clear_ret)
 {
+	struct _special s;
 	struct trapframe *tf;
+	uint64_t bspst, *kstk, *ustk;
 
 	tf = td->td_frame;
-	mc->mc_special = tf->tf_special;
+	s = tf->tf_special;
+	s.pfs = s.cfm;
+	s.rp = s.iip;
+	s.cfm = s.iip = 0;
+	if (s.ndirty != 0) {
+		__asm __volatile("mov	ar.rsc=0;;");
+		__asm __volatile("mov	%0=ar.bspstore" : "=r"(bspst));
+		/* Make sure we have all the user registers written out. */
+		if (bspst - td->td_kstack < s.ndirty)
+			__asm __volatile("flushrs;;");
+		__asm __volatile("mov	ar.rsc=3");
+		ustk = (uint64_t*)s.bspstore;
+		kstk = (uint64_t*)td->td_kstack;
+		while (s.ndirty > 0) {
+			*ustk++ = *kstk++;
+			if (((uintptr_t)ustk & 0x1ff) == 0x1f8)
+				*ustk++ = 0;
+			if (((uintptr_t)kstk & 0x1ff) == 0x1f8) {
+				kstk++;
+				s.ndirty -= 8;
+			}
+			s.ndirty -= 8;
+		}
+		s.bspstore = (uintptr_t)ustk;
+	}
+	mc->mc_special = s;
 	save_callee_saved(&mc->mc_preserved);
 	save_callee_saved_fp(&mc->mc_preserved_fp);
+	/*
+	 * Put the syscall return values in the context. We need this
+	 * for swapcontext() to work. Note that we don't use gr11 in
+	 * the kernel, but the runtime specification defines it as a
+	 * return register, just like gr8-gr10.
+	 */
+	mc->mc_scratch.gr8 = (clear_ret) ? 0 : tf->tf_scratch.gr8;
+	mc->mc_scratch.gr9 = (clear_ret) ? 0 : tf->tf_scratch.gr9;
+	mc->mc_scratch.gr10 = (clear_ret) ? 0 : tf->tf_scratch.gr10;
+	mc->mc_scratch.gr11 = (clear_ret) ? 0 : tf->tf_scratch.gr11;
 	return (0);
 }
 
 int
 set_mcontext(struct thread *td, const mcontext_t *mc)
 {
+	struct _special s;
 	struct trapframe *tf;
-	uint64_t psr;
 
 	tf = td->td_frame;
+	s = mc->mc_special;
+	s.cfm = s.pfs;
+	s.iip = s.rp;
+	s.pfs = tf->tf_special.pfs;
+	s.rp = tf->tf_special.rp;
 	/* Only copy the user mask from the new context. */
-	psr = tf->tf_special.psr & ~0x1f;
-	psr |= mc->mc_special.psr & 0x1f;
-	tf->tf_special = mc->mc_special;
-	tf->tf_special.psr = psr;
+	s.psr = (s.psr & 0x1f) | (tf->tf_special.psr & ~0x1f);
+	tf->tf_special = s;
 	restore_callee_saved(&mc->mc_preserved);
 	restore_callee_saved_fp(&mc->mc_preserved_fp);
+	tf->tf_scratch.gr8 = mc->mc_scratch.gr8;
+	tf->tf_scratch.gr9 = mc->mc_scratch.gr9;
+	tf->tf_scratch.gr10 = mc->mc_scratch.gr10;
+	tf->tf_scratch.gr11 = mc->mc_scratch.gr11;
 	return (0);
 }