Change cpu_set_kse_upcall to more generic style, so we can reuse it

in other codes. Add cpu_set_user_tls, use it to tweak user register
and setup user TLS. I ever wanted to merge it into cpu_set_kse_upcall,
but since cpu_set_kse_upcall is also used by M:N threads which may
not need this feature, so I wrote a separated cpu_set_user_tls.

sys/alpha/alpha/vm_machdep.c

@@ -330,15 +330,16 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 }
 
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
 	struct pcb *pcb;
 	struct trapframe *tf;
-	uint64_t stack;
+	uint64_t sp;
 
 	pcb = td->td_pcb;
 	tf = td->td_frame;
-	stack = ((uint64_t)ku->ku_stack.ss_sp + ku->ku_stack.ss_size) & ~15;
+	sp = ((uint64_t)stack->ss_sp + stack->ss_size) & ~15;
 
 	bzero(tf->tf_regs, FRAME_SIZE * sizeof(tf->tf_regs[0]));
 	bzero(&pcb->pcb_fp, sizeof(pcb->pcb_fp));
@@ -346,19 +347,29 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
 	pcb->pcb_fp.fpr_cr = FPCR_DYN_NORMAL | FPCR_INVD | FPCR_DZED |
 	    FPCR_OVFD | FPCR_INED | FPCR_UNFD;
 	if (td != curthread) {
-		pcb->pcb_hw.apcb_usp = stack;
+		pcb->pcb_hw.apcb_usp = sp;
 		pcb->pcb_hw.apcb_unique = 0;
 	} else {
-		alpha_pal_wrusp(stack);
+		alpha_pal_wrusp(sp);
 		alpha_pal_wrunique(0);
 	}
 	tf->tf_regs[FRAME_PS] = ALPHA_PSL_USERSET;
-	tf->tf_regs[FRAME_PC] = (u_long)ku->ku_func;
-	tf->tf_regs[FRAME_A0] = (u_long)ku->ku_mailbox;
+	tf->tf_regs[FRAME_PC] = (u_long)entry;
+	tf->tf_regs[FRAME_A0] = (u_long)arg;
 	tf->tf_regs[FRAME_T12] = tf->tf_regs[FRAME_PC];	/* aka. PV */
 	tf->tf_regs[FRAME_FLAGS] = 0;			/* full restore */
 }
 
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	if (td != curthread)
+		td->td_pcb->pcb_hw.apcb_unique = (unsigned long)tls_base;
+	else
+		alpha_pal_wrunique((uintptr_t)tls_base);
+}
+
 /*
  * Reset back to firmware.
  */

sys/amd64/amd64/vm_machdep.c

@@ -310,7 +310,8 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
  * in thread_userret() itself can be done as well.
  */
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
 
 	/* 
@@ -326,18 +327,32 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
 	 * Set the trap frame to point at the beginning of the uts
 	 * function.
 	 */
-	td->td_frame->tf_rbp = 0; 
+	td->td_frame->tf_rbp = 0;
 	td->td_frame->tf_rsp =
-	    ((register_t)ku->ku_stack.ss_sp + ku->ku_stack.ss_size) & ~0x0f;
+	    ((register_t)stack->ss_sp + stack->ss_size) & ~0x0f;
 	td->td_frame->tf_rsp -= 8;
 	td->td_frame->tf_rbp = 0;
-	td->td_frame->tf_rip = (register_t)ku->ku_func;
+	td->td_frame->tf_rip = (register_t)entry;
 
 	/*
 	 * Pass the address of the mailbox for this kse to the uts
 	 * function as a parameter on the stack.
 	 */
-	td->td_frame->tf_rdi = (register_t)ku->ku_mailbox;
+	td->td_frame->tf_rdi = (register_t)arg;
+}
+
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	if (td == curthread) {
+		critical_enter();
+		td->td_pcb->pcb_fsbase = (register_t)tls_base;
+		wrmsr(MSR_FSBASE, td->td_pcb->pcb_fsbase);
+		critical_exit();
+	} else {
+		td->td_pcb->pcb_fsbase = (register_t)tls_base;
+	}
 }
 
 #ifdef SMP

sys/arm/arm/vm_machdep.c

@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sf_buf.h>
 #include <machine/cpu.h>
 #include <machine/pcb.h>
+#include <machine/sysarch.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #include <sys/lock.h>
@@ -279,17 +280,31 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
  * in thread_userret() itself can be done as well.
  */
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
 	struct trapframe *tf = td->td_frame;
 
-	tf->tf_usr_sp = ((int)ku->ku_stack.ss_sp + ku->ku_stack.ss_size
+	tf->tf_usr_sp = ((int)stack->ss_sp + stack->ss_size
 	    - sizeof(struct trapframe)) & ~7;
-	tf->tf_pc = (int)ku->ku_func;
-	tf->tf_r0 = (int)ku->ku_mailbox;
+	tf->tf_pc = (int)entry;
+	tf->tf_r0 = (int)arg;
 	tf->tf_spsr = PSR_USR32_MODE;
 }
 
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	if (td != curthread)
+		td->td_md.md_tp = tls_base;
+	else {
+		critical_enter();
+		*(void **)ARM_TP_ADDRESS = tls_base;
+		critical_exit();
+	}
+}
+
 void
 cpu_thread_exit(struct thread *td)
 {

sys/i386/i386/vm_machdep.c

@@ -439,7 +439,8 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
  * in thread_userret() itself can be done as well.
  */
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
 
 	/* 
@@ -457,15 +458,48 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
 	 */
 	td->td_frame->tf_ebp = 0; 
 	td->td_frame->tf_esp =
-	    (int)ku->ku_stack.ss_sp + ku->ku_stack.ss_size - 16;
-	td->td_frame->tf_eip = (int)ku->ku_func;
+	    (int)stack->ss_sp + stack->ss_size - 16;
+	td->td_frame->tf_eip = (int)entry;
 
 	/*
 	 * Pass the address of the mailbox for this kse to the uts
 	 * function as a parameter on the stack.
 	 */
 	suword((void *)(td->td_frame->tf_esp + sizeof(void *)),
-	    (int)ku->ku_mailbox);
+	    (int)arg);
+}
+
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+	struct segment_descriptor sd;
+	uint32_t base;
+
+	/*
+	 * Construct a descriptor and store it in the pcb for
+	 * the next context switch.  Also store it in the gdt
+	 * so that the load of tf_fs into %fs will activate it
+	 * at return to userland.
+	 */
+	base = (uint32_t)tls_base;
+	sd.sd_lobase = base & 0xffffff;
+	sd.sd_hibase = (base >> 24) & 0xff;
+	sd.sd_lolimit = 0xffff;	/* 4GB limit, wraps around */
+	sd.sd_hilimit = 0xf;
+	sd.sd_type  = SDT_MEMRWA;
+	sd.sd_dpl   = SEL_UPL;
+	sd.sd_p     = 1;
+	sd.sd_xx    = 0;
+	sd.sd_def32 = 1;
+	sd.sd_gran  = 1;
+	critical_enter();
+	/* set %gs */
+	td->td_pcb->pcb_gsd = sd;
+	if (td == curthread) {
+		PCPU_GET(fsgs_gdt)[1] = sd;
+		load_gs(GSEL(GUGS_SEL, SEL_UPL));
+	}
+	critical_exit();
 }
 
 /*

sys/ia64/ia64/vm_machdep.c

@@ -165,11 +165,12 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 }
 
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
 	struct ia64_fdesc *fd;
 	struct trapframe *tf;
-	uint64_t ndirty, stack;
+	uint64_t ndirty, sp;
 
 	tf = td->td_frame;
 	ndirty = tf->tf_special.ndirty + (tf->tf_special.bspstore & 0x1ffUL);
@@ -177,13 +178,13 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
 	KASSERT((ndirty & ~PAGE_MASK) == 0,
 	    ("Whoa there! We have more than 8KB of dirty registers!"));
 
-	fd = ku->ku_func;
-	stack = (uint64_t)ku->ku_stack.ss_sp;
+	fd = (struct ia64_fdesc *)entry;
+	sp = (uint64_t)stack->ss_sp;
 
 	bzero(&tf->tf_special, sizeof(tf->tf_special));
 	tf->tf_special.iip = fuword(&fd->func);
 	tf->tf_special.gp = fuword(&fd->gp);
-	tf->tf_special.sp = (stack + ku->ku_stack.ss_size - 16) & ~15;
+	tf->tf_special.sp = (sp + stack->ss_size - 16) & ~15;
 	tf->tf_special.rsc = 0xf;
 	tf->tf_special.fpsr = IA64_FPSR_DEFAULT;
 	tf->tf_special.psr = IA64_PSR_IC | IA64_PSR_I | IA64_PSR_IT |
@@ -192,22 +193,28 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
 
 	if (tf->tf_flags & FRAME_SYSCALL) {
 		tf->tf_special.cfm = (3UL<<62) | (1UL<<7) | 1UL;
-		tf->tf_special.bspstore = stack + 8;
-		suword((caddr_t)stack, (uint64_t)ku->ku_mailbox);
+		tf->tf_special.bspstore = sp + 8;
+		suword((caddr_t)sp, (uint64_t)arg);
 	} else {
 		tf->tf_special.cfm = (1UL<<63) | (1UL<<7) | 1UL;
-		tf->tf_special.bspstore = stack;
+		tf->tf_special.bspstore = sp;
 		tf->tf_special.ndirty = 8;
-		stack = td->td_kstack + ndirty - 8;
-		if ((stack & 0x1ff) == 0x1f8) {
-			*(uint64_t*)stack = 0;
+		sp = td->td_kstack + ndirty - 8;
+		if ((sp & 0x1ff) == 0x1f8) {
+			*(uint64_t*)sp = 0;
 			tf->tf_special.ndirty += 8;
-			stack -= 8;
+			sp -= 8;
 		}
-		*(uint64_t*)stack = (uint64_t)ku->ku_mailbox;
+		*(uint64_t*)sp = (uint64_t)arg;
 	}
 }
 
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+	td->td_frame->tf_special.tp = (unsigned long)tls_base;
+}
+
 /*
  * Finish a fork operation, with process p2 nearly set up.
  * Copy and update the pcb, set up the stack so that the child

sys/kern/kern_kse.c

@@ -780,7 +780,8 @@ kse_create(struct thread *td, struct kse_create_args *uap)
 			 * to the new thread, so we should clear single step
 			 * flag here.
 			 */
-			cpu_set_upcall_kse(newtd, newku);
+			cpu_set_upcall_kse(newtd, newku->ku_func,
+				newku->ku_mailbox, &newku->ku_stack);
 			if (p->p_flag & P_TRACED)
 				ptrace_clear_single_step(newtd);
 		}
@@ -1371,7 +1372,8 @@ thread_userret(struct thread *td, struct trapframe *frame)
 		 * Set user context to the UTS
 		 */
 		if (!(ku->ku_mflags & KMF_NOUPCALL)) {
-			cpu_set_upcall_kse(td, ku);
+			cpu_set_upcall_kse(td, ku->ku_func, ku->ku_mailbox,
+				&ku->ku_stack);
 			if (p->p_flag & P_TRACED)
 				ptrace_clear_single_step(td);
 			error = suword32(&ku->ku_mailbox->km_lwp,

sys/kern/kern_thread.c

@@ -258,6 +258,8 @@ ksegrp_unlink(struct ksegrp *kg)
 	/*
 	 * Aggregate stats from the KSE
 	 */
+	if (p->p_procscopegrp == kg)
+		p->p_procscopegrp = NULL;
 }
 
 /*
@@ -990,6 +992,7 @@ thread_single_end(void)
 	p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
 	mtx_lock_spin(&sched_lock);
 	p->p_singlethread = NULL;
+	p->p_procscopegrp = NULL;
 	/*
 	 * If there are other threads they mey now run,
 	 * unless of course there is a blanket 'stop order'

sys/powerpc/aim/vm_machdep.c

@@ -333,23 +333,31 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 }
 
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
         struct trapframe *tf;
         uint32_t sp;
 
 	tf = td->td_frame;
 	/* align stack and alloc space for frame ptr and saved LR */
-        sp = ((uint32_t)ku->ku_stack.ss_sp + ku->ku_stack.ss_size
+        sp = ((uint32_t)stack->ss_sp + stack->ss_size
 		- 2*sizeof(u_int32_t)) & ~0x1f;
 	bzero(tf, sizeof(struct trapframe));
 
 	tf->fixreg[1] = (register_t)sp;
-        tf->fixreg[3] = (register_t)ku->ku_mailbox;
-        tf->srr0 = (register_t)ku->ku_func;
+        tf->fixreg[3] = (register_t)arg;
+        tf->srr0 = (register_t)entry;
         tf->srr1 = PSL_MBO | PSL_USERSET | PSL_FE_DFLT;
         td->td_pcb->pcb_flags = 0;
 
-        td->td_retval[0] = (register_t)ku->ku_func;
+        td->td_retval[0] = (register_t)entry;
         td->td_retval[1] = 0;
 }
+
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	td->td_frame->fixreg[2] = (register_t)tls_base;
+}

sys/powerpc/powerpc/vm_machdep.c

@@ -333,23 +333,31 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 }
 
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
         struct trapframe *tf;
         uint32_t sp;
 
 	tf = td->td_frame;
 	/* align stack and alloc space for frame ptr and saved LR */
-        sp = ((uint32_t)ku->ku_stack.ss_sp + ku->ku_stack.ss_size
+        sp = ((uint32_t)stack->ss_sp + stack->ss_size
 		- 2*sizeof(u_int32_t)) & ~0x1f;
 	bzero(tf, sizeof(struct trapframe));
 
 	tf->fixreg[1] = (register_t)sp;
-        tf->fixreg[3] = (register_t)ku->ku_mailbox;
-        tf->srr0 = (register_t)ku->ku_func;
+        tf->fixreg[3] = (register_t)arg;
+        tf->srr0 = (register_t)entry;
         tf->srr1 = PSL_MBO | PSL_USERSET | PSL_FE_DFLT;
         td->td_pcb->pcb_flags = 0;
 
-        td->td_retval[0] = (register_t)ku->ku_func;
+        td->td_retval[0] = (register_t)entry;
         td->td_retval[1] = 0;
 }
+
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	td->td_frame->fixreg[2] = (register_t)tls_base;
+}

sys/sparc64/sparc64/vm_machdep.c

@@ -177,22 +177,34 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 }
 
 void
-cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
+cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
+	stack_t *stack)
 {
 	struct trapframe *tf;
 	uint64_t sp;
 
+	if (td == curthread)
+		flushw();
 	tf = td->td_frame;
-	sp = (uint64_t)ku->ku_stack.ss_sp + ku->ku_stack.ss_size;
-	tf->tf_out[0] = (uint64_t)ku->ku_mailbox;
+	sp = (uint64_t)stack->ss_sp + stack->ss_size;
+	tf->tf_out[0] = (uint64_t)arg;
 	tf->tf_out[6] = sp - SPOFF - sizeof(struct frame);
-	tf->tf_tpc = (uint64_t)ku->ku_func;
+	tf->tf_tpc = (uint64_t)entry;
 	tf->tf_tnpc = tf->tf_tpc + 4;
 
 	td->td_retval[0] = tf->tf_out[0];
 	td->td_retval[1] = tf->tf_out[1];
 }
 
+void
+cpu_set_user_tls(struct thread *td, void *tls_base)
+{
+
+	if (td == curthread)
+		flushw();
+	td->td_frame->tf_global[7] = (uint64_t) tls_base;
+}
+
 /*
  * Finish a fork operation, with process p2 nearly set up.
  * Copy and update the pcb, set up the stack so that the child

sys/sys/proc.h

@@ -574,6 +574,7 @@ struct proc {
 	int		p_suspcount;	/* (c) Num threads in suspended mode. */
 	struct thread	*p_xthread;	/* (c) Trap thread */
 	int		p_boundary_count;/* (c) Num threads at user boundary */
+	struct ksegrp	*p_procscopegrp;
 /* End area that is zeroed on creation. */
 #define	p_endzero	p_magic
 
@@ -894,7 +895,8 @@ void	ksegrp_stash(struct ksegrp *kg);
 void	kse_GC(void);
 void	kseinit(void);
 void	cpu_set_upcall(struct thread *td, struct thread *td0);
-void	cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku);
+void	cpu_set_upcall_kse(struct thread *, void (*)(void *), void *, stack_t *);
+void	cpu_set_user_tls(struct thread *, void *tls_base);
 void	cpu_thread_clean(struct thread *);
 void	cpu_thread_exit(struct thread *);
 void	cpu_thread_setup(struct thread *td);