2019-05-27 06:55:01 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
2005-10-10 12:36:14 +00:00
|
|
|
/*
|
|
|
|
* PowerPC version
|
|
|
|
* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
|
|
|
|
* Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
|
|
|
|
* Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
|
|
|
|
* Adapted for Power Macintosh by Paul Mackerras.
|
|
|
|
* Low-level exception handlers and MMU support
|
|
|
|
* rewritten by Paul Mackerras.
|
|
|
|
* Copyright (C) 1996 Paul Mackerras.
|
|
|
|
* MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
|
|
|
|
*
|
|
|
|
* This file contains the system call entry code, context switch
|
|
|
|
* code, and exception/interrupt return code for PowerPC.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/errno.h>
|
powerpc/kernel: Switch to using MAX_ERRNO
Currently on powerpc we have our own #define for the highest (negative)
errno value, called _LAST_ERRNO. This is defined to be 516, for reasons
which are not clear.
The generic code, and x86, use MAX_ERRNO, which is defined to be 4095.
In particular seccomp uses MAX_ERRNO to restrict the value that a
seccomp filter can return.
Currently with the mismatch between _LAST_ERRNO and MAX_ERRNO, a seccomp
tracer wanting to return 600, expecting it to be seen as an error, would
instead find on powerpc that userspace sees a successful syscall with a
return value of 600.
To avoid this inconsistency, switch powerpc to use MAX_ERRNO.
We are somewhat confident that generic syscalls that can return a
non-error value above negative MAX_ERRNO have already been updated to
use force_successful_syscall_return().
I have also checked all the powerpc specific syscalls, and believe that
none of them expect to return a non-error value between -MAX_ERRNO and
-516. So this change should be safe ...
Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Reviewed-by: Kees Cook <keescook@chromium.org>
2015-07-23 10:21:01 +00:00
|
|
|
#include <linux/err.h>
|
2005-10-10 12:36:14 +00:00
|
|
|
#include <linux/sys.h>
|
|
|
|
#include <linux/threads.h>
|
|
|
|
#include <asm/reg.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
#include <asm/mmu.h>
|
|
|
|
#include <asm/cputable.h>
|
|
|
|
#include <asm/thread_info.h>
|
|
|
|
#include <asm/ppc_asm.h>
|
|
|
|
#include <asm/asm-offsets.h>
|
|
|
|
#include <asm/unistd.h>
|
2010-11-18 15:06:17 +00:00
|
|
|
#include <asm/ptrace.h>
|
2016-01-14 04:33:46 +00:00
|
|
|
#include <asm/export.h>
|
2018-07-05 16:25:01 +00:00
|
|
|
#include <asm/feature-fixups.h>
|
2018-07-27 23:06:38 +00:00
|
|
|
#include <asm/barrier.h>
|
2019-03-11 08:30:31 +00:00
|
|
|
#include <asm/kup.h>
|
2019-04-30 12:39:01 +00:00
|
|
|
#include <asm/bug.h>
|
2005-10-10 12:36:14 +00:00
|
|
|
|
2019-04-30 12:38:51 +00:00
|
|
|
#include "head_32.h"
|
2005-10-10 12:36:14 +00:00
|
|
|
|
2020-07-16 01:35:22 +00:00
|
|
|
/*
|
|
|
|
* powerpc relies on return from interrupt/syscall being context synchronising
|
|
|
|
* (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional
|
|
|
|
* synchronisation instructions.
|
|
|
|
*/
|
|
|
|
|
2017-07-12 10:08:49 +00:00
|
|
|
/*
|
|
|
|
* Align to 4k in order to ensure that all functions modyfing srr0/srr1
|
|
|
|
* fit into one page in order to not encounter a TLB miss between the
|
|
|
|
* modification of srr0/srr1 and the associated rfi.
|
|
|
|
*/
|
|
|
|
.align 12
|
|
|
|
|
2005-10-10 12:36:14 +00:00
|
|
|
#ifdef CONFIG_BOOKE
|
|
|
|
.globl mcheck_transfer_to_handler
|
|
|
|
mcheck_transfer_to_handler:
|
2008-04-30 10:23:21 +00:00
|
|
|
/* fall through */
|
2020-03-31 16:03:45 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(mcheck_transfer_to_handler)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
.globl debug_transfer_to_handler
|
|
|
|
debug_transfer_to_handler:
|
2008-04-30 10:23:21 +00:00
|
|
|
/* fall through */
|
2020-03-31 16:03:45 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(debug_transfer_to_handler)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
.globl crit_transfer_to_handler
|
|
|
|
crit_transfer_to_handler:
|
|
|
|
/* fall through */
|
2020-03-31 16:03:45 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
|
2005-10-10 12:36:14 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_40x
|
|
|
|
.globl crit_transfer_to_handler
|
|
|
|
crit_transfer_to_handler:
|
|
|
|
/* fall through */
|
2020-03-31 16:03:45 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
|
2005-10-10 12:36:14 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This code finishes saving the registers to the exception frame
|
|
|
|
* and jumps to the appropriate handler for the exception, turning
|
|
|
|
* on address translation.
|
|
|
|
* Note that we rely on the caller having set cr0.eq iff the exception
|
|
|
|
* occurred in kernel mode (i.e. MSR:PR = 0).
|
|
|
|
*/
|
|
|
|
.globl transfer_to_handler_full
|
|
|
|
transfer_to_handler_full:
|
|
|
|
SAVE_NVGPRS(r11)
|
2020-03-31 16:03:45 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(transfer_to_handler_full)
|
2005-10-10 12:36:14 +00:00
|
|
|
/* fall through */
|
|
|
|
|
|
|
|
.globl transfer_to_handler
|
|
|
|
transfer_to_handler:
|
|
|
|
stw r2,GPR2(r11)
|
|
|
|
stw r12,_NIP(r11)
|
|
|
|
stw r9,_MSR(r11)
|
|
|
|
andi. r2,r9,MSR_PR
|
|
|
|
mfctr r12
|
|
|
|
mfspr r2,SPRN_XER
|
|
|
|
stw r12,_CTR(r11)
|
|
|
|
stw r2,_XER(r11)
|
2009-07-14 20:52:54 +00:00
|
|
|
mfspr r12,SPRN_SPRG_THREAD
|
2021-03-12 12:50:22 +00:00
|
|
|
tovirt(r12, r12)
|
2005-10-10 12:36:14 +00:00
|
|
|
beq 2f /* if from user, fix up THREAD.regs */
|
2019-03-11 08:30:31 +00:00
|
|
|
addi r2, r12, -THREAD
|
2005-10-10 12:36:14 +00:00
|
|
|
addi r11,r1,STACK_FRAME_OVERHEAD
|
|
|
|
stw r11,PT_REGS(r12)
|
2019-03-11 08:30:35 +00:00
|
|
|
#ifdef CONFIG_PPC_BOOK3S_32
|
|
|
|
kuep_lock r11, r12
|
|
|
|
#endif
|
2005-10-10 12:36:14 +00:00
|
|
|
b 3f
|
2006-04-18 11:49:11 +00:00
|
|
|
|
2021-03-12 12:50:21 +00:00
|
|
|
/* if from kernel, check interrupted DOZE/NAP mode */
|
|
|
|
2:
|
2020-01-27 10:42:04 +00:00
|
|
|
kuap_save_and_lock r11, r12, r9, r2, r6
|
2019-03-11 08:30:31 +00:00
|
|
|
addi r2, r12, -THREAD
|
2018-11-17 10:24:56 +00:00
|
|
|
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
|
2019-01-31 10:09:04 +00:00
|
|
|
lwz r12,TI_LOCAL_FLAGS(r2)
|
2006-04-18 11:49:11 +00:00
|
|
|
mtcrf 0x01,r12
|
|
|
|
bt- 31-TLF_NAPPING,4f
|
2008-05-14 04:30:48 +00:00
|
|
|
bt- 31-TLF_SLEEPING,7f
|
2018-11-17 10:24:56 +00:00
|
|
|
#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */
|
2005-10-10 12:36:14 +00:00
|
|
|
.globl transfer_to_handler_cont
|
|
|
|
transfer_to_handler_cont:
|
|
|
|
3:
|
|
|
|
mflr r9
|
|
|
|
lwz r11,0(r9) /* virtual address of handler */
|
|
|
|
lwz r9,4(r9) /* where to go when done */
|
2021-03-12 12:50:24 +00:00
|
|
|
mtctr r11
|
2019-04-30 12:39:01 +00:00
|
|
|
mtlr r9
|
2021-03-12 12:50:24 +00:00
|
|
|
bctr /* jump to handler */
|
2019-04-30 12:39:01 +00:00
|
|
|
|
2020-03-31 16:03:45 +00:00
|
|
|
#if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
|
|
|
|
4: rlwinm r12,r12,0,~_TLF_NAPPING
|
|
|
|
stw r12,TI_LOCAL_FLAGS(r2)
|
|
|
|
b power_save_ppc32_restore
|
|
|
|
|
|
|
|
7: rlwinm r12,r12,0,~_TLF_SLEEPING
|
|
|
|
stw r12,TI_LOCAL_FLAGS(r2)
|
|
|
|
lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
|
|
|
|
rlwinm r9,r9,0,~MSR_EE
|
|
|
|
lwz r12,_LINK(r11) /* and return to address in LR */
|
|
|
|
kuap_restore r11, r2, r3, r4, r5
|
|
|
|
lwz r2, GPR2(r11)
|
|
|
|
b fast_exception_return
|
|
|
|
#endif
|
|
|
|
_ASM_NOKPROBE_SYMBOL(transfer_to_handler)
|
|
|
|
_ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont)
|
|
|
|
|
2019-04-30 12:39:02 +00:00
|
|
|
.globl transfer_to_syscall
|
|
|
|
transfer_to_syscall:
|
2021-02-08 15:10:32 +00:00
|
|
|
SAVE_NVGPRS(r1)
|
2021-02-08 15:10:20 +00:00
|
|
|
#ifdef CONFIG_PPC_BOOK3S_32
|
|
|
|
kuep_lock r11, r12
|
|
|
|
#endif
|
2019-04-30 12:39:02 +00:00
|
|
|
|
2021-02-08 15:10:33 +00:00
|
|
|
/* Calling convention has r9 = orig r0, r10 = regs */
|
|
|
|
addi r10,r1,STACK_FRAME_OVERHEAD
|
|
|
|
mr r9,r0
|
|
|
|
stw r10,THREAD+PT_REGS(r2)
|
|
|
|
bl system_call_exception
|
2018-07-27 23:06:38 +00:00
|
|
|
|
2005-10-10 12:36:14 +00:00
|
|
|
ret_from_syscall:
|
2021-02-08 15:10:33 +00:00
|
|
|
addi r4,r1,STACK_FRAME_OVERHEAD
|
|
|
|
li r5,0
|
|
|
|
bl syscall_exit_prepare
|
2020-10-18 17:25:18 +00:00
|
|
|
#ifdef CONFIG_PPC_47x
|
2007-10-31 05:42:19 +00:00
|
|
|
lis r4,icache_44x_need_flush@ha
|
|
|
|
lwz r5,icache_44x_need_flush@l(r4)
|
|
|
|
cmplwi cr0,r5,0
|
|
|
|
bne- 2f
|
2020-10-18 17:25:18 +00:00
|
|
|
#endif /* CONFIG_PPC_47x */
|
2019-03-11 08:30:35 +00:00
|
|
|
#ifdef CONFIG_PPC_BOOK3S_32
|
|
|
|
kuep_unlock r5, r7
|
|
|
|
#endif
|
2019-03-11 08:30:31 +00:00
|
|
|
kuap_check r2, r4
|
2005-10-10 12:36:14 +00:00
|
|
|
lwz r4,_LINK(r1)
|
|
|
|
lwz r5,_CCR(r1)
|
|
|
|
mtlr r4
|
|
|
|
lwz r7,_NIP(r1)
|
2021-02-08 15:10:33 +00:00
|
|
|
lwz r8,_MSR(r1)
|
|
|
|
cmpwi r3,0
|
|
|
|
lwz r3,GPR3(r1)
|
2020-03-31 16:03:46 +00:00
|
|
|
syscall_exit_finish:
|
2005-10-10 12:36:14 +00:00
|
|
|
mtspr SPRN_SRR0,r7
|
|
|
|
mtspr SPRN_SRR1,r8
|
2021-02-08 15:10:33 +00:00
|
|
|
|
|
|
|
bne 3f
|
|
|
|
mtcr r5
|
|
|
|
|
|
|
|
1: lwz r2,GPR2(r1)
|
|
|
|
lwz r1,GPR1(r1)
|
2020-11-08 16:57:37 +00:00
|
|
|
rfi
|
|
|
|
#ifdef CONFIG_40x
|
|
|
|
b . /* Prevent prefetch past rfi */
|
|
|
|
#endif
|
2021-02-08 15:10:33 +00:00
|
|
|
|
|
|
|
3: mtcr r5
|
|
|
|
lwz r4,_CTR(r1)
|
|
|
|
lwz r5,_XER(r1)
|
|
|
|
REST_NVGPRS(r1)
|
|
|
|
mtctr r4
|
|
|
|
mtxer r5
|
|
|
|
lwz r0,GPR0(r1)
|
|
|
|
lwz r3,GPR3(r1)
|
|
|
|
REST_8GPRS(4,r1)
|
|
|
|
lwz r12,GPR12(r1)
|
|
|
|
b 1b
|
|
|
|
|
2007-10-31 05:42:19 +00:00
|
|
|
#ifdef CONFIG_44x
|
|
|
|
2: li r7,0
|
|
|
|
iccci r0,r0
|
|
|
|
stw r7,icache_44x_need_flush@l(r4)
|
|
|
|
b 1b
|
|
|
|
#endif /* CONFIG_44x */
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
.globl ret_from_fork
|
|
|
|
ret_from_fork:
|
|
|
|
REST_NVGPRS(r1)
|
|
|
|
bl schedule_tail
|
|
|
|
li r3,0
|
|
|
|
b ret_from_syscall
|
|
|
|
|
2012-09-12 22:32:42 +00:00
|
|
|
.globl ret_from_kernel_thread
|
|
|
|
ret_from_kernel_thread:
|
|
|
|
REST_NVGPRS(r1)
|
|
|
|
bl schedule_tail
|
|
|
|
mtlr r14
|
|
|
|
mr r3,r15
|
|
|
|
PPC440EP_ERR42
|
|
|
|
blrl
|
|
|
|
li r3,0
|
2012-08-31 19:48:05 +00:00
|
|
|
b ret_from_syscall
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Top-level page fault handling.
|
|
|
|
* This is in assembler because if do_page_fault tells us that
|
|
|
|
* it is a bad kernel page fault, we want to save the non-volatile
|
|
|
|
* registers before calling bad_page_fault.
|
|
|
|
*/
|
|
|
|
.globl handle_page_fault
|
|
|
|
handle_page_fault:
|
|
|
|
addi r3,r1,STACK_FRAME_OVERHEAD
|
2017-08-08 06:37:24 +00:00
|
|
|
bl do_page_fault
|
2005-10-10 12:36:14 +00:00
|
|
|
cmpwi r3,0
|
|
|
|
beq+ ret_from_except
|
|
|
|
SAVE_NVGPRS(r1)
|
2005-10-28 12:45:25 +00:00
|
|
|
lwz r0,_TRAP(r1)
|
2005-10-10 12:36:14 +00:00
|
|
|
clrrwi r0,r0,1
|
2005-10-28 12:45:25 +00:00
|
|
|
stw r0,_TRAP(r1)
|
2021-01-30 13:08:21 +00:00
|
|
|
mr r4,r3 /* err arg for bad_page_fault */
|
2005-10-10 12:36:14 +00:00
|
|
|
addi r3,r1,STACK_FRAME_OVERHEAD
|
2020-12-09 05:29:25 +00:00
|
|
|
bl __bad_page_fault
|
2005-10-10 12:36:14 +00:00
|
|
|
b ret_from_except_full
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This routine switches between two different tasks. The process
|
|
|
|
* state of one is saved on its kernel stack. Then the state
|
|
|
|
* of the other is restored from its kernel stack. The memory
|
|
|
|
* management hardware is updated to the second process's state.
|
|
|
|
* Finally, we can return to the second process.
|
|
|
|
* On entry, r3 points to the THREAD for the current task, r4
|
|
|
|
* points to the THREAD for the new task.
|
|
|
|
*
|
|
|
|
* This routine is always called with interrupts disabled.
|
|
|
|
*
|
|
|
|
* Note: there are two ways to get to the "going out" portion
|
|
|
|
* of this code; either by coming in via the entry (_switch)
|
|
|
|
* or via "fork" which must set up an environment equivalent
|
|
|
|
* to the "_switch" path. If you change this , you'll have to
|
|
|
|
* change the fork code also.
|
|
|
|
*
|
|
|
|
* The code which creates the new task context is in 'copy_thread'
|
|
|
|
* in arch/ppc/kernel/process.c
|
|
|
|
*/
|
|
|
|
_GLOBAL(_switch)
|
|
|
|
stwu r1,-INT_FRAME_SIZE(r1)
|
|
|
|
mflr r0
|
|
|
|
stw r0,INT_FRAME_SIZE+4(r1)
|
|
|
|
/* r3-r12 are caller saved -- Cort */
|
|
|
|
SAVE_NVGPRS(r1)
|
|
|
|
stw r0,_NIP(r1) /* Return to switch caller */
|
|
|
|
mfmsr r11
|
|
|
|
li r0,MSR_FP /* Disable floating-point */
|
|
|
|
#ifdef CONFIG_ALTIVEC
|
|
|
|
BEGIN_FTR_SECTION
|
|
|
|
oris r0,r0,MSR_VEC@h /* Disable altivec */
|
|
|
|
mfspr r12,SPRN_VRSAVE /* save vrsave register value */
|
|
|
|
stw r12,THREAD+THREAD_VRSAVE(r2)
|
|
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
|
|
#endif /* CONFIG_ALTIVEC */
|
|
|
|
#ifdef CONFIG_SPE
|
2007-09-13 06:44:20 +00:00
|
|
|
BEGIN_FTR_SECTION
|
2005-10-10 12:36:14 +00:00
|
|
|
oris r0,r0,MSR_SPE@h /* Disable SPE */
|
|
|
|
mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
|
|
|
|
stw r12,THREAD+THREAD_SPEFSCR(r2)
|
2007-09-13 06:44:20 +00:00
|
|
|
END_FTR_SECTION_IFSET(CPU_FTR_SPE)
|
2005-10-10 12:36:14 +00:00
|
|
|
#endif /* CONFIG_SPE */
|
|
|
|
and. r0,r0,r11 /* FP or altivec or SPE enabled? */
|
|
|
|
beq+ 1f
|
|
|
|
andc r11,r11,r0
|
2019-12-21 08:32:22 +00:00
|
|
|
mtmsr r11
|
2005-10-10 12:36:14 +00:00
|
|
|
isync
|
|
|
|
1: stw r11,_MSR(r1)
|
|
|
|
mfcr r10
|
|
|
|
stw r10,_CCR(r1)
|
|
|
|
stw r1,KSP(r3) /* Set old stack pointer */
|
|
|
|
|
2020-04-17 11:58:36 +00:00
|
|
|
kuap_check r2, r0
|
2005-10-10 12:36:14 +00:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/* We need a sync somewhere here to make sure that if the
|
|
|
|
* previous task gets rescheduled on another CPU, it sees all
|
|
|
|
* stores it has performed on this one.
|
|
|
|
*/
|
|
|
|
sync
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
|
|
|
tophys(r0,r4)
|
2009-07-14 20:52:54 +00:00
|
|
|
mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
|
2005-10-10 12:36:14 +00:00
|
|
|
lwz r1,KSP(r4) /* Load new stack pointer */
|
|
|
|
|
|
|
|
/* save the old current 'last' for return value */
|
|
|
|
mr r3,r2
|
|
|
|
addi r2,r4,-THREAD /* Update current */
|
|
|
|
|
|
|
|
#ifdef CONFIG_ALTIVEC
|
|
|
|
BEGIN_FTR_SECTION
|
|
|
|
lwz r0,THREAD+THREAD_VRSAVE(r2)
|
|
|
|
mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
|
|
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
|
|
#endif /* CONFIG_ALTIVEC */
|
|
|
|
#ifdef CONFIG_SPE
|
2007-09-13 06:44:20 +00:00
|
|
|
BEGIN_FTR_SECTION
|
2005-10-10 12:36:14 +00:00
|
|
|
lwz r0,THREAD+THREAD_SPEFSCR(r2)
|
|
|
|
mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
|
2007-09-13 06:44:20 +00:00
|
|
|
END_FTR_SECTION_IFSET(CPU_FTR_SPE)
|
2005-10-10 12:36:14 +00:00
|
|
|
#endif /* CONFIG_SPE */
|
2017-01-24 10:37:20 +00:00
|
|
|
|
2005-10-10 12:36:14 +00:00
|
|
|
lwz r0,_CCR(r1)
|
|
|
|
mtcrf 0xFF,r0
|
|
|
|
/* r3-r12 are destroyed -- Cort */
|
|
|
|
REST_NVGPRS(r1)
|
|
|
|
|
|
|
|
lwz r4,_NIP(r1) /* Return to _switch caller in new task */
|
|
|
|
mtlr r4
|
|
|
|
addi r1,r1,INT_FRAME_SIZE
|
|
|
|
blr
|
|
|
|
|
|
|
|
.globl fast_exception_return
|
|
|
|
fast_exception_return:
|
|
|
|
#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
|
|
|
|
andi. r10,r9,MSR_RI /* check for recoverable interrupt */
|
|
|
|
beq 1f /* if not, we've got problems */
|
|
|
|
#endif
|
|
|
|
|
|
|
|
2: REST_4GPRS(3, r11)
|
|
|
|
lwz r10,_CCR(r11)
|
|
|
|
REST_GPR(1, r11)
|
|
|
|
mtcr r10
|
|
|
|
lwz r10,_LINK(r11)
|
|
|
|
mtlr r10
|
2019-02-27 11:45:30 +00:00
|
|
|
/* Clear the exception_marker on the stack to avoid confusing stacktrace */
|
|
|
|
li r10, 0
|
|
|
|
stw r10, 8(r11)
|
2005-10-10 12:36:14 +00:00
|
|
|
REST_GPR(10, r11)
|
2018-01-12 12:45:23 +00:00
|
|
|
#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
|
powerpc/8xx: Perf events on PPC 8xx
This patch has been reworked since RFC version. In the RFC, this patch
was preceded by a patch clearing MSR RI for all PPC32 at all time at
exception prologs. Now MSR RI clearing is done only when this 8xx perf
events functionality is compiled in, it is therefore limited to 8xx
and merged inside this patch.
Other main changes have been to take into account detailed review from
Peter Zijlstra. The instructions counter has been reworked to behave
as a free running counter like the three other counters.
The 8xx has no PMU, however some events can be emulated by other means.
This patch implements the following events (as reported by 'perf list'):
cpu-cycles OR cycles [Hardware event]
instructions [Hardware event]
dTLB-load-misses [Hardware cache event]
iTLB-load-misses [Hardware cache event]
'cycles' event is implemented using the timebase clock. Timebase clock
corresponds to CPU clock divided by 16, so number of cycles is
approximatly 16 times the number of TB ticks
On the 8xx, TLB misses are handled by software. It is therefore
easy to count all TLB misses each time the TLB miss exception is
called.
'instructions' is calculated by using instruction watchpoint counter.
This patch sets counter A to count instructions at address greater
than 0, hence we count all instructions executed while MSR RI bit is
set. The counter is set to the maximum which is 0xffff. Every 65535
instructions, debug instruction breakpoint exception fires. The
exception handler increments a counter in memory which then
represent the upper part of the instruction counter. We therefore
end up with a 48 bits counter. In order to avoid unnecessary overhead
while no perf event is active, this counter is started when the first
event referring to this counter is added, and the counter is stopped
when the last event referring to it is deleted. In order to properly
support breakpoint exceptions, MSR RI bit has to be unset in exception
epilogs in order to avoid breakpoint exceptions during critical
sections during changes to SRR0 and SRR1 would be problematic.
All counters are handled as free running counters.
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: Scott Wood <oss@buserror.net>
2016-12-15 12:42:18 +00:00
|
|
|
mtspr SPRN_NRI, r0
|
|
|
|
#endif
|
2005-10-10 12:36:14 +00:00
|
|
|
mtspr SPRN_SRR1,r9
|
|
|
|
mtspr SPRN_SRR0,r12
|
|
|
|
REST_GPR(9, r11)
|
|
|
|
REST_GPR(12, r11)
|
|
|
|
lwz r11,GPR11(r11)
|
2020-11-08 16:57:37 +00:00
|
|
|
rfi
|
|
|
|
#ifdef CONFIG_40x
|
|
|
|
b . /* Prevent prefetch past rfi */
|
|
|
|
#endif
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(fast_exception_return)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
|
|
|
|
/* check if the exception happened in a restartable section */
|
|
|
|
1: lis r3,exc_exit_restart_end@ha
|
|
|
|
addi r3,r3,exc_exit_restart_end@l
|
|
|
|
cmplw r12,r3
|
|
|
|
bge 3f
|
|
|
|
lis r4,exc_exit_restart@ha
|
|
|
|
addi r4,r4,exc_exit_restart@l
|
|
|
|
cmplw r12,r4
|
|
|
|
blt 3f
|
|
|
|
lis r3,fee_restarts@ha
|
|
|
|
tophys(r3,r3)
|
|
|
|
lwz r5,fee_restarts@l(r3)
|
|
|
|
addi r5,r5,1
|
|
|
|
stw r5,fee_restarts@l(r3)
|
|
|
|
mr r12,r4 /* restart at exc_exit_restart */
|
|
|
|
b 2b
|
|
|
|
|
2007-05-14 22:11:58 +00:00
|
|
|
.section .bss
|
|
|
|
.align 2
|
|
|
|
fee_restarts:
|
|
|
|
.space 4
|
|
|
|
.previous
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
/* aargh, a nonrecoverable interrupt, panic */
|
|
|
|
/* aargh, we don't know which trap this is */
|
|
|
|
3:
|
|
|
|
li r10,-1
|
2005-10-28 12:45:25 +00:00
|
|
|
stw r10,_TRAP(r11)
|
2005-10-10 12:36:14 +00:00
|
|
|
addi r3,r1,STACK_FRAME_OVERHEAD
|
|
|
|
bl transfer_to_handler_full
|
2018-09-25 14:10:04 +00:00
|
|
|
.long unrecoverable_exception
|
2005-10-10 12:36:14 +00:00
|
|
|
.long ret_from_except
|
|
|
|
#endif
|
|
|
|
|
|
|
|
.globl ret_from_except_full
|
|
|
|
ret_from_except_full:
|
|
|
|
REST_NVGPRS(r1)
|
|
|
|
/* fall through */
|
|
|
|
|
|
|
|
.globl ret_from_except
|
|
|
|
ret_from_except:
|
|
|
|
/* Hard-disable interrupts so that current_thread_info()->flags
|
|
|
|
* can't change between when we test it and when we return
|
|
|
|
* from the interrupt. */
|
2009-06-17 17:43:59 +00:00
|
|
|
/* Note: We don't bother telling lockdep about it */
|
2019-08-20 14:34:13 +00:00
|
|
|
LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
|
2019-12-21 08:32:22 +00:00
|
|
|
mtmsr r10 /* disable interrupts */
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
lwz r3,_MSR(r1) /* Returning to user mode? */
|
|
|
|
andi. r0,r3,MSR_PR
|
|
|
|
beq resume_kernel
|
|
|
|
|
|
|
|
user_exc_return: /* r10 contains MSR_KERNEL here */
|
|
|
|
/* Check current_thread_info()->flags */
|
2019-01-31 10:09:04 +00:00
|
|
|
lwz r9,TI_FLAGS(r2)
|
2008-04-28 07:30:37 +00:00
|
|
|
andi. r0,r9,_TIF_USER_WORK_MASK
|
2005-10-10 12:36:14 +00:00
|
|
|
bne do_work
|
|
|
|
|
|
|
|
restore_user:
|
|
|
|
#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
|
2008-04-09 21:15:40 +00:00
|
|
|
/* Check whether this process has its own DBCR0 value. The internal
|
|
|
|
debug mode bit tells us that dbcr0 should be loaded. */
|
2005-10-10 12:36:14 +00:00
|
|
|
lwz r0,THREAD+THREAD_DBCR0(r2)
|
2008-07-25 19:27:33 +00:00
|
|
|
andis. r10,r0,DBCR0_IDM@h
|
2005-10-10 12:36:14 +00:00
|
|
|
bnel- load_dbcr0
|
|
|
|
#endif
|
2019-01-31 10:09:04 +00:00
|
|
|
ACCOUNT_CPU_USER_EXIT(r2, r10, r11)
|
2019-03-11 08:30:35 +00:00
|
|
|
#ifdef CONFIG_PPC_BOOK3S_32
|
|
|
|
kuep_unlock r10, r11
|
|
|
|
#endif
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
b restore
|
|
|
|
|
|
|
|
/* N.B. the only way to get here is from the beq following ret_from_except. */
|
|
|
|
resume_kernel:
|
2012-09-16 23:54:30 +00:00
|
|
|
/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
|
2019-01-31 10:09:04 +00:00
|
|
|
lwz r8,TI_FLAGS(r2)
|
2013-05-31 01:20:02 +00:00
|
|
|
andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
|
2012-09-16 23:54:30 +00:00
|
|
|
beq+ 1f
|
|
|
|
|
|
|
|
addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
|
|
|
|
|
|
|
|
lwz r3,GPR1(r1)
|
|
|
|
subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
|
|
|
|
mr r4,r1 /* src: current exception frame */
|
|
|
|
mr r1,r3 /* Reroute the trampoline frame to r1 */
|
|
|
|
|
|
|
|
/* Copy from the original to the trampoline. */
|
|
|
|
li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
|
|
|
|
li r6,0 /* start offset: 0 */
|
|
|
|
mtctr r5
|
|
|
|
2: lwzx r0,r6,r4
|
|
|
|
stwx r0,r6,r3
|
|
|
|
addi r6,r6,4
|
|
|
|
bdnz 2b
|
|
|
|
|
|
|
|
/* Do real store operation to complete stwu */
|
|
|
|
lwz r5,GPR1(r1)
|
|
|
|
stw r8,0(r5)
|
|
|
|
|
|
|
|
/* Clear _TIF_EMULATE_STACK_STORE flag */
|
|
|
|
lis r11,_TIF_EMULATE_STACK_STORE@h
|
2019-01-31 10:09:04 +00:00
|
|
|
addi r5,r2,TI_FLAGS
|
2012-09-16 23:54:30 +00:00
|
|
|
0: lwarx r8,0,r5
|
|
|
|
andc r8,r8,r11
|
|
|
|
stwcx. r8,0,r5
|
|
|
|
bne- 0b
|
|
|
|
1:
|
|
|
|
|
2019-10-24 16:04:58 +00:00
|
|
|
#ifdef CONFIG_PREEMPTION
|
2012-09-16 23:54:30 +00:00
|
|
|
/* check current_thread_info->preempt_count */
|
2019-01-31 10:09:04 +00:00
|
|
|
lwz r0,TI_PREEMPT(r2)
|
2005-10-10 12:36:14 +00:00
|
|
|
cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
|
2019-03-11 08:30:31 +00:00
|
|
|
bne restore_kuap
|
2012-09-16 23:54:30 +00:00
|
|
|
andi. r8,r8,_TIF_NEED_RESCHED
|
2019-03-11 08:30:31 +00:00
|
|
|
beq+ restore_kuap
|
2012-09-16 23:54:30 +00:00
|
|
|
lwz r3,_MSR(r1)
|
2005-10-10 12:36:14 +00:00
|
|
|
andi. r0,r3,MSR_EE /* interrupts off? */
|
2019-03-11 08:30:31 +00:00
|
|
|
beq restore_kuap /* don't schedule if so */
|
2009-06-17 17:43:59 +00:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
/* Lockdep thinks irqs are enabled, we need to call
|
|
|
|
* preempt_schedule_irq with IRQs off, so we inform lockdep
|
|
|
|
* now that we -did- turn them off already
|
|
|
|
*/
|
|
|
|
bl trace_hardirqs_off
|
|
|
|
#endif
|
2019-03-11 22:47:46 +00:00
|
|
|
bl preempt_schedule_irq
|
2009-06-17 17:43:59 +00:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
/* And now, to properly rebalance the above, we tell lockdep they
|
|
|
|
* are being turned back on, which will happen when we return
|
|
|
|
*/
|
|
|
|
bl trace_hardirqs_on
|
|
|
|
#endif
|
2019-10-24 16:04:58 +00:00
|
|
|
#endif /* CONFIG_PREEMPTION */
|
2019-03-11 08:30:31 +00:00
|
|
|
restore_kuap:
|
|
|
|
kuap_restore r1, r2, r9, r10, r0
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
/* interrupts are hard-disabled at this point */
|
|
|
|
restore:
|
2020-10-18 17:25:18 +00:00
|
|
|
#if defined(CONFIG_44x) && !defined(CONFIG_PPC_47x)
|
2007-10-31 05:42:19 +00:00
|
|
|
lis r4,icache_44x_need_flush@ha
|
|
|
|
lwz r5,icache_44x_need_flush@l(r4)
|
|
|
|
cmplwi cr0,r5,0
|
|
|
|
beq+ 1f
|
|
|
|
li r6,0
|
|
|
|
iccci r0,r0
|
|
|
|
stw r6,icache_44x_need_flush@l(r4)
|
|
|
|
1:
|
|
|
|
#endif /* CONFIG_44x */
|
2009-06-17 17:43:59 +00:00
|
|
|
|
|
|
|
lwz r9,_MSR(r1)
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
/* Lockdep doesn't know about the fact that IRQs are temporarily turned
|
|
|
|
* off in this assembly code while peeking at TI_FLAGS() and such. However
|
|
|
|
* we need to inform it if the exception turned interrupts off, and we
|
|
|
|
* are about to trun them back on.
|
|
|
|
*/
|
|
|
|
andi. r10,r9,MSR_EE
|
|
|
|
beq 1f
|
2010-12-22 16:42:56 +00:00
|
|
|
stwu r1,-32(r1)
|
|
|
|
mflr r0
|
|
|
|
stw r0,4(r1)
|
2009-06-17 17:43:59 +00:00
|
|
|
bl trace_hardirqs_on
|
2019-04-30 12:39:05 +00:00
|
|
|
addi r1, r1, 32
|
2009-06-17 17:43:59 +00:00
|
|
|
lwz r9,_MSR(r1)
|
|
|
|
1:
|
|
|
|
#endif /* CONFIG_TRACE_IRQFLAGS */
|
|
|
|
|
2005-10-10 12:36:14 +00:00
|
|
|
lwz r0,GPR0(r1)
|
|
|
|
lwz r2,GPR2(r1)
|
|
|
|
REST_4GPRS(3, r1)
|
|
|
|
REST_2GPRS(7, r1)
|
|
|
|
|
|
|
|
lwz r10,_XER(r1)
|
|
|
|
lwz r11,_CTR(r1)
|
|
|
|
mtspr SPRN_XER,r10
|
|
|
|
mtctr r11
|
|
|
|
|
2007-11-09 22:17:49 +00:00
|
|
|
BEGIN_FTR_SECTION
|
|
|
|
lwarx r11,0,r1
|
|
|
|
END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
|
2005-10-10 12:36:14 +00:00
|
|
|
stwcx. r0,0,r1 /* to clear the reservation */
|
|
|
|
|
|
|
|
#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
|
|
|
|
andi. r10,r9,MSR_RI /* check if this exception occurred */
|
|
|
|
beql nonrecoverable /* at a bad place (MSR:RI = 0) */
|
|
|
|
|
|
|
|
lwz r10,_CCR(r1)
|
|
|
|
lwz r11,_LINK(r1)
|
|
|
|
mtcrf 0xFF,r10
|
|
|
|
mtlr r11
|
|
|
|
|
2019-02-27 11:45:30 +00:00
|
|
|
/* Clear the exception_marker on the stack to avoid confusing stacktrace */
|
|
|
|
li r10, 0
|
|
|
|
stw r10, 8(r1)
|
2005-10-10 12:36:14 +00:00
|
|
|
/*
|
|
|
|
* Once we put values in SRR0 and SRR1, we are in a state
|
|
|
|
* where exceptions are not recoverable, since taking an
|
|
|
|
* exception will trash SRR0 and SRR1. Therefore we clear the
|
|
|
|
* MSR:RI bit to indicate this. If we do take an exception,
|
|
|
|
* we can't return to the point of the exception but we
|
|
|
|
* can restart the exception exit path at the label
|
|
|
|
* exc_exit_restart below. -- paulus
|
|
|
|
*/
|
2019-08-20 14:34:13 +00:00
|
|
|
LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI)
|
2019-12-21 08:32:22 +00:00
|
|
|
mtmsr r10 /* clear the RI bit */
|
2005-10-10 12:36:14 +00:00
|
|
|
.globl exc_exit_restart
|
|
|
|
exc_exit_restart:
|
|
|
|
lwz r12,_NIP(r1)
|
|
|
|
mtspr SPRN_SRR0,r12
|
|
|
|
mtspr SPRN_SRR1,r9
|
|
|
|
REST_4GPRS(9, r1)
|
|
|
|
lwz r1,GPR1(r1)
|
|
|
|
.globl exc_exit_restart_end
|
|
|
|
exc_exit_restart_end:
|
2020-11-08 16:57:36 +00:00
|
|
|
rfi
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(exc_exit_restart)
|
|
|
|
_ASM_NOKPROBE_SYMBOL(exc_exit_restart_end)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
#else /* !(CONFIG_4xx || CONFIG_BOOKE) */
|
|
|
|
/*
|
|
|
|
* This is a bit different on 4xx/Book-E because it doesn't have
|
|
|
|
* the RI bit in the MSR.
|
|
|
|
* The TLB miss handler checks if we have interrupted
|
|
|
|
* the exception exit path and restarts it if so
|
|
|
|
* (well maybe one day it will... :).
|
|
|
|
*/
|
|
|
|
lwz r11,_LINK(r1)
|
|
|
|
mtlr r11
|
|
|
|
lwz r10,_CCR(r1)
|
|
|
|
mtcrf 0xff,r10
|
2019-02-27 11:45:30 +00:00
|
|
|
/* Clear the exception_marker on the stack to avoid confusing stacktrace */
|
|
|
|
li r10, 0
|
|
|
|
stw r10, 8(r1)
|
2005-10-10 12:36:14 +00:00
|
|
|
REST_2GPRS(9, r1)
|
|
|
|
.globl exc_exit_restart
|
|
|
|
exc_exit_restart:
|
|
|
|
lwz r11,_NIP(r1)
|
|
|
|
lwz r12,_MSR(r1)
|
|
|
|
mtspr SPRN_SRR0,r11
|
|
|
|
mtspr SPRN_SRR1,r12
|
|
|
|
REST_2GPRS(11, r1)
|
|
|
|
lwz r1,GPR1(r1)
|
|
|
|
.globl exc_exit_restart_end
|
|
|
|
exc_exit_restart_end:
|
|
|
|
rfi
|
|
|
|
b . /* prevent prefetch past rfi */
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(exc_exit_restart)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Returning from a critical interrupt in user mode doesn't need
|
|
|
|
* to be any different from a normal exception. For a critical
|
|
|
|
* interrupt in the kernel, we just return (without checking for
|
|
|
|
* preemption) since the interrupt may have happened at some crucial
|
|
|
|
* place (e.g. inside the TLB miss handler), and because we will be
|
|
|
|
* running with r1 pointing into critical_stack, not the current
|
|
|
|
* process's kernel stack (and therefore current_thread_info() will
|
|
|
|
* give the wrong answer).
|
|
|
|
* We have to restore various SPRs that may have been in use at the
|
|
|
|
* time of the critical interrupt.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_40x
|
|
|
|
#define PPC_40x_TURN_OFF_MSR_DR \
|
|
|
|
/* avoid any possible TLB misses here by turning off MSR.DR, we \
|
|
|
|
* assume the instructions here are mapped by a pinned TLB entry */ \
|
|
|
|
li r10,MSR_IR; \
|
|
|
|
mtmsr r10; \
|
|
|
|
isync; \
|
|
|
|
tophys(r1, r1);
|
|
|
|
#else
|
|
|
|
#define PPC_40x_TURN_OFF_MSR_DR
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
|
|
|
|
REST_NVGPRS(r1); \
|
|
|
|
lwz r3,_MSR(r1); \
|
|
|
|
andi. r3,r3,MSR_PR; \
|
2019-08-20 14:34:13 +00:00
|
|
|
LOAD_REG_IMMEDIATE(r10,MSR_KERNEL); \
|
2005-10-10 12:36:14 +00:00
|
|
|
bne user_exc_return; \
|
|
|
|
lwz r0,GPR0(r1); \
|
|
|
|
lwz r2,GPR2(r1); \
|
|
|
|
REST_4GPRS(3, r1); \
|
|
|
|
REST_2GPRS(7, r1); \
|
|
|
|
lwz r10,_XER(r1); \
|
|
|
|
lwz r11,_CTR(r1); \
|
|
|
|
mtspr SPRN_XER,r10; \
|
|
|
|
mtctr r11; \
|
|
|
|
stwcx. r0,0,r1; /* to clear the reservation */ \
|
|
|
|
lwz r11,_LINK(r1); \
|
|
|
|
mtlr r11; \
|
|
|
|
lwz r10,_CCR(r1); \
|
|
|
|
mtcrf 0xff,r10; \
|
|
|
|
PPC_40x_TURN_OFF_MSR_DR; \
|
|
|
|
lwz r9,_DEAR(r1); \
|
|
|
|
lwz r10,_ESR(r1); \
|
|
|
|
mtspr SPRN_DEAR,r9; \
|
|
|
|
mtspr SPRN_ESR,r10; \
|
|
|
|
lwz r11,_NIP(r1); \
|
|
|
|
lwz r12,_MSR(r1); \
|
|
|
|
mtspr exc_lvl_srr0,r11; \
|
|
|
|
mtspr exc_lvl_srr1,r12; \
|
|
|
|
lwz r9,GPR9(r1); \
|
|
|
|
lwz r12,GPR12(r1); \
|
|
|
|
lwz r10,GPR10(r1); \
|
|
|
|
lwz r11,GPR11(r1); \
|
|
|
|
lwz r1,GPR1(r1); \
|
|
|
|
exc_lvl_rfi; \
|
|
|
|
b .; /* prevent prefetch past exc_lvl_rfi */
|
|
|
|
|
2008-04-30 10:23:21 +00:00
|
|
|
#define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
|
|
|
|
lwz r9,_##exc_lvl_srr0(r1); \
|
|
|
|
lwz r10,_##exc_lvl_srr1(r1); \
|
|
|
|
mtspr SPRN_##exc_lvl_srr0,r9; \
|
|
|
|
mtspr SPRN_##exc_lvl_srr1,r10;
|
|
|
|
|
2009-02-12 22:12:40 +00:00
|
|
|
#if defined(CONFIG_PPC_BOOK3E_MMU)
|
2008-04-30 10:23:21 +00:00
|
|
|
#ifdef CONFIG_PHYS_64BIT
|
|
|
|
#define RESTORE_MAS7 \
|
|
|
|
lwz r11,MAS7(r1); \
|
|
|
|
mtspr SPRN_MAS7,r11;
|
|
|
|
#else
|
|
|
|
#define RESTORE_MAS7
|
|
|
|
#endif /* CONFIG_PHYS_64BIT */
|
|
|
|
#define RESTORE_MMU_REGS \
|
|
|
|
lwz r9,MAS0(r1); \
|
|
|
|
lwz r10,MAS1(r1); \
|
|
|
|
lwz r11,MAS2(r1); \
|
|
|
|
mtspr SPRN_MAS0,r9; \
|
|
|
|
lwz r9,MAS3(r1); \
|
|
|
|
mtspr SPRN_MAS1,r10; \
|
|
|
|
lwz r10,MAS6(r1); \
|
|
|
|
mtspr SPRN_MAS2,r11; \
|
|
|
|
mtspr SPRN_MAS3,r9; \
|
|
|
|
mtspr SPRN_MAS6,r10; \
|
|
|
|
RESTORE_MAS7;
|
|
|
|
#elif defined(CONFIG_44x)
|
|
|
|
#define RESTORE_MMU_REGS \
|
|
|
|
lwz r9,MMUCR(r1); \
|
|
|
|
mtspr SPRN_MMUCR,r9;
|
|
|
|
#else
|
|
|
|
#define RESTORE_MMU_REGS
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_40x
|
2005-10-10 12:36:14 +00:00
|
|
|
.globl ret_from_crit_exc
|
|
|
|
ret_from_crit_exc:
|
2008-04-30 10:23:21 +00:00
|
|
|
lis r9,crit_srr0@ha;
|
|
|
|
lwz r9,crit_srr0@l(r9);
|
|
|
|
lis r10,crit_srr1@ha;
|
|
|
|
lwz r10,crit_srr1@l(r10);
|
|
|
|
mtspr SPRN_SRR0,r9;
|
|
|
|
mtspr SPRN_SRR1,r10;
|
2009-02-10 20:10:44 +00:00
|
|
|
RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(ret_from_crit_exc)
|
2008-04-30 10:23:21 +00:00
|
|
|
#endif /* CONFIG_40x */
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_BOOKE
|
2008-04-30 10:23:21 +00:00
|
|
|
.globl ret_from_crit_exc
|
|
|
|
ret_from_crit_exc:
|
|
|
|
RESTORE_xSRR(SRR0,SRR1);
|
|
|
|
RESTORE_MMU_REGS;
|
2009-02-10 20:10:44 +00:00
|
|
|
RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(ret_from_crit_exc)
|
2008-04-30 10:23:21 +00:00
|
|
|
|
2005-10-10 12:36:14 +00:00
|
|
|
.globl ret_from_debug_exc
|
|
|
|
ret_from_debug_exc:
|
2008-04-30 10:23:21 +00:00
|
|
|
RESTORE_xSRR(SRR0,SRR1);
|
|
|
|
RESTORE_xSRR(CSRR0,CSRR1);
|
|
|
|
RESTORE_MMU_REGS;
|
2009-02-10 20:10:44 +00:00
|
|
|
RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(ret_from_debug_exc)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
.globl ret_from_mcheck_exc
|
|
|
|
ret_from_mcheck_exc:
|
2008-04-30 10:23:21 +00:00
|
|
|
RESTORE_xSRR(SRR0,SRR1);
|
|
|
|
RESTORE_xSRR(CSRR0,CSRR1);
|
|
|
|
RESTORE_xSRR(DSRR0,DSRR1);
|
|
|
|
RESTORE_MMU_REGS;
|
2009-02-10 20:10:44 +00:00
|
|
|
RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
|
2020-03-31 16:03:47 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)
|
2005-10-10 12:36:14 +00:00
|
|
|
#endif /* CONFIG_BOOKE */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Load the DBCR0 value for a task that is being ptraced,
|
|
|
|
* having first saved away the global DBCR0. Note that r0
|
|
|
|
* has the dbcr0 value to set upon entry to this.
|
|
|
|
*/
|
|
|
|
load_dbcr0:
|
|
|
|
mfmsr r10 /* first disable debug exceptions */
|
|
|
|
rlwinm r10,r10,0,~MSR_DE
|
|
|
|
mtmsr r10
|
|
|
|
isync
|
|
|
|
mfspr r10,SPRN_DBCR0
|
|
|
|
lis r11,global_dbcr0@ha
|
|
|
|
addi r11,r11,global_dbcr0@l
|
2008-04-09 21:15:40 +00:00
|
|
|
#ifdef CONFIG_SMP
|
2019-01-31 10:09:04 +00:00
|
|
|
lwz r9,TASK_CPU(r2)
|
2021-02-08 15:10:41 +00:00
|
|
|
slwi r9,r9,2
|
2008-04-09 21:15:40 +00:00
|
|
|
add r11,r11,r9
|
|
|
|
#endif
|
2005-10-10 12:36:14 +00:00
|
|
|
stw r10,0(r11)
|
|
|
|
mtspr SPRN_DBCR0,r0
|
|
|
|
li r11,-1
|
|
|
|
mtspr SPRN_DBSR,r11 /* clear all pending debug events */
|
|
|
|
blr
|
|
|
|
|
2007-05-14 22:11:58 +00:00
|
|
|
.section .bss
|
|
|
|
.align 4
|
2019-04-30 12:39:02 +00:00
|
|
|
.global global_dbcr0
|
2007-05-14 22:11:58 +00:00
|
|
|
global_dbcr0:
|
2021-02-08 15:10:41 +00:00
|
|
|
.space 4*NR_CPUS
|
2007-05-14 22:11:58 +00:00
|
|
|
.previous
|
2005-10-10 12:36:14 +00:00
|
|
|
#endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
|
|
|
|
|
|
|
|
do_work: /* r10 contains MSR_KERNEL here */
|
|
|
|
andi. r0,r9,_TIF_NEED_RESCHED
|
|
|
|
beq do_user_signal
|
|
|
|
|
|
|
|
do_resched: /* r10 contains MSR_KERNEL here */
|
2019-04-30 12:39:01 +00:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
bl trace_hardirqs_on
|
|
|
|
mfmsr r10
|
|
|
|
#endif
|
2005-10-10 12:36:14 +00:00
|
|
|
ori r10,r10,MSR_EE
|
2019-12-21 08:32:22 +00:00
|
|
|
mtmsr r10 /* hard-enable interrupts */
|
2005-10-10 12:36:14 +00:00
|
|
|
bl schedule
|
|
|
|
recheck:
|
2009-06-17 17:43:59 +00:00
|
|
|
/* Note: And we don't tell it we are disabling them again
|
|
|
|
* neither. Those disable/enable cycles used to peek at
|
|
|
|
* TI_FLAGS aren't advertised.
|
|
|
|
*/
|
2019-08-20 14:34:13 +00:00
|
|
|
LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
|
2019-12-21 08:32:22 +00:00
|
|
|
mtmsr r10 /* disable interrupts */
|
2019-01-31 10:09:04 +00:00
|
|
|
lwz r9,TI_FLAGS(r2)
|
2005-10-10 12:36:14 +00:00
|
|
|
andi. r0,r9,_TIF_NEED_RESCHED
|
|
|
|
bne- do_resched
|
2008-04-28 07:30:37 +00:00
|
|
|
andi. r0,r9,_TIF_USER_WORK_MASK
|
2005-10-10 12:36:14 +00:00
|
|
|
beq restore_user
|
|
|
|
do_user_signal: /* r10 contains MSR_KERNEL here */
|
|
|
|
ori r10,r10,MSR_EE
|
2019-12-21 08:32:22 +00:00
|
|
|
mtmsr r10 /* hard-enable interrupts */
|
2005-10-10 12:36:14 +00:00
|
|
|
/* save r13-r31 in the exception frame, if not already done */
|
2005-10-28 12:45:25 +00:00
|
|
|
lwz r3,_TRAP(r1)
|
2005-10-10 12:36:14 +00:00
|
|
|
andi. r0,r3,1
|
|
|
|
beq 2f
|
|
|
|
SAVE_NVGPRS(r1)
|
|
|
|
rlwinm r3,r3,0,0,30
|
2005-10-28 12:45:25 +00:00
|
|
|
stw r3,_TRAP(r1)
|
2008-07-27 06:52:52 +00:00
|
|
|
2: addi r3,r1,STACK_FRAME_OVERHEAD
|
|
|
|
mr r4,r9
|
2012-02-22 05:48:32 +00:00
|
|
|
bl do_notify_resume
|
2005-10-10 12:36:14 +00:00
|
|
|
REST_NVGPRS(r1)
|
|
|
|
b recheck
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We come here when we are at the end of handling an exception
|
|
|
|
* that occurred at a place where taking an exception will lose
|
|
|
|
* state information, such as the contents of SRR0 and SRR1.
|
|
|
|
*/
|
|
|
|
nonrecoverable:
|
|
|
|
lis r10,exc_exit_restart_end@ha
|
|
|
|
addi r10,r10,exc_exit_restart_end@l
|
|
|
|
cmplw r12,r10
|
|
|
|
bge 3f
|
|
|
|
lis r11,exc_exit_restart@ha
|
|
|
|
addi r11,r11,exc_exit_restart@l
|
|
|
|
cmplw r12,r11
|
|
|
|
blt 3f
|
|
|
|
lis r10,ee_restarts@ha
|
|
|
|
lwz r12,ee_restarts@l(r10)
|
|
|
|
addi r12,r12,1
|
|
|
|
stw r12,ee_restarts@l(r10)
|
|
|
|
mr r12,r11 /* restart at exc_exit_restart */
|
|
|
|
blr
|
|
|
|
3: /* OK, we can't recover, kill this process */
|
2005-10-28 12:45:25 +00:00
|
|
|
lwz r3,_TRAP(r1)
|
2005-10-10 12:36:14 +00:00
|
|
|
andi. r0,r3,1
|
2019-01-31 10:08:58 +00:00
|
|
|
beq 5f
|
2005-10-10 12:36:14 +00:00
|
|
|
SAVE_NVGPRS(r1)
|
|
|
|
rlwinm r3,r3,0,0,30
|
2005-10-28 12:45:25 +00:00
|
|
|
stw r3,_TRAP(r1)
|
2019-01-31 10:08:58 +00:00
|
|
|
5: mfspr r2,SPRN_SPRG_THREAD
|
|
|
|
addi r2,r2,-THREAD
|
|
|
|
tovirt(r2,r2) /* set back r2 to current */
|
2005-10-10 12:36:14 +00:00
|
|
|
4: addi r3,r1,STACK_FRAME_OVERHEAD
|
2018-09-25 14:10:04 +00:00
|
|
|
bl unrecoverable_exception
|
2005-10-10 12:36:14 +00:00
|
|
|
/* shouldn't return */
|
|
|
|
b 4b
|
2020-03-31 16:03:44 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(nonrecoverable)
|
2005-10-10 12:36:14 +00:00
|
|
|
|
2007-05-14 22:11:58 +00:00
|
|
|
.section .bss
|
|
|
|
.align 2
|
|
|
|
ee_restarts:
|
|
|
|
.space 4
|
|
|
|
.previous
|
2005-10-10 12:36:14 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* PROM code for specific machines follows. Put it
|
|
|
|
* here so it's easy to add arch-specific sections later.
|
|
|
|
* -- Cort
|
|
|
|
*/
|
2005-10-26 07:05:24 +00:00
|
|
|
#ifdef CONFIG_PPC_RTAS
|
2005-10-10 12:36:14 +00:00
|
|
|
/*
|
|
|
|
* On CHRP, the Run-Time Abstraction Services (RTAS) have to be
|
|
|
|
* called with the MMU off.
|
|
|
|
*/
|
|
|
|
_GLOBAL(enter_rtas)
|
|
|
|
stwu r1,-INT_FRAME_SIZE(r1)
|
|
|
|
mflr r0
|
|
|
|
stw r0,INT_FRAME_SIZE+4(r1)
|
2006-01-13 03:56:25 +00:00
|
|
|
LOAD_REG_ADDR(r4, rtas)
|
2005-10-10 12:36:14 +00:00
|
|
|
lis r6,1f@ha /* physical return address for rtas */
|
|
|
|
addi r6,r6,1f@l
|
|
|
|
tophys(r6,r6)
|
2005-10-26 07:05:24 +00:00
|
|
|
lwz r8,RTASENTRY(r4)
|
|
|
|
lwz r4,RTASBASE(r4)
|
2005-10-10 12:36:14 +00:00
|
|
|
mfmsr r9
|
|
|
|
stw r9,8(r1)
|
2019-08-20 14:34:13 +00:00
|
|
|
LOAD_REG_IMMEDIATE(r0,MSR_KERNEL)
|
2020-09-29 06:48:34 +00:00
|
|
|
mtmsr r0 /* disable interrupts so SRR0/1 don't get trashed */
|
2005-10-10 12:36:14 +00:00
|
|
|
li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
|
|
|
|
mtlr r6
|
2021-03-12 12:50:22 +00:00
|
|
|
stw r1, THREAD + RTAS_SP(r2)
|
2005-10-10 12:36:14 +00:00
|
|
|
mtspr SPRN_SRR0,r8
|
|
|
|
mtspr SPRN_SRR1,r9
|
2020-11-08 16:57:36 +00:00
|
|
|
rfi
|
2021-03-12 12:50:22 +00:00
|
|
|
1:
|
2021-03-12 12:50:24 +00:00
|
|
|
lis r8, 1f@h
|
|
|
|
ori r8, r8, 1f@l
|
|
|
|
LOAD_REG_IMMEDIATE(r9,MSR_KERNEL)
|
|
|
|
mtspr SPRN_SRR0,r8
|
|
|
|
mtspr SPRN_SRR1,r9
|
|
|
|
rfi /* Reactivate MMU translation */
|
|
|
|
1:
|
2021-03-12 12:50:22 +00:00
|
|
|
lwz r8,INT_FRAME_SIZE+4(r1) /* get return address */
|
|
|
|
lwz r9,8(r1) /* original msr value */
|
2005-10-10 12:36:14 +00:00
|
|
|
addi r1,r1,INT_FRAME_SIZE
|
|
|
|
li r0,0
|
2021-03-12 12:50:22 +00:00
|
|
|
stw r0, THREAD + RTAS_SP(r2)
|
2021-03-12 12:50:24 +00:00
|
|
|
mtlr r8
|
|
|
|
mtmsr r9
|
|
|
|
blr /* return to caller */
|
2020-03-31 16:03:44 +00:00
|
|
|
_ASM_NOKPROBE_SYMBOL(enter_rtas)
|
2005-10-26 07:05:24 +00:00
|
|
|
#endif /* CONFIG_PPC_RTAS */
|