mirror of
https://github.com/torvalds/linux
synced 2024-10-03 01:43:05 +00:00
d51f86cfd8
The dssall ("Data Stream Stop All") instruction is obsolete altogether
with other Data Cache Instructions since ISA 2.03 (year 2006).
LLVM IAS does not support it but PPC970 seems to be using it.
This switches dssall to .long as there is no much point in fixing LLVM.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20211221055904.555763-6-aik@ozlabs.ru
357 lines
7.2 KiB
ArmAsm
357 lines
7.2 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
/*
|
|
* This file contains low-level cache management functions
|
|
* used for sleep and CPU speed changes on Apple machines.
|
|
* (In fact the only thing that is Apple-specific is that we assume
|
|
* that we can read from ROM at physical address 0xfff00000.)
|
|
*
|
|
* Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and
|
|
* Benjamin Herrenschmidt (benh@kernel.crashing.org)
|
|
*/
|
|
|
|
#include <asm/processor.h>
|
|
#include <asm/ppc_asm.h>
|
|
#include <asm/cputable.h>
|
|
#include <asm/feature-fixups.h>
|
|
|
|
/*
|
|
* Flush and disable all data caches (dL1, L2, L3). This is used
|
|
* when going to sleep, when doing a PMU based cpufreq transition,
|
|
* or when "offlining" a CPU on SMP machines. This code is over
|
|
* paranoid, but I've had enough issues with various CPU revs and
|
|
* bugs that I decided it was worth being over cautious
|
|
*/
|
|
|
|
_GLOBAL(flush_disable_caches)
|
|
#ifndef CONFIG_PPC_BOOK3S_32
|
|
blr
|
|
#else
|
|
BEGIN_FTR_SECTION
|
|
b flush_disable_745x
|
|
END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
|
|
BEGIN_FTR_SECTION
|
|
b flush_disable_75x
|
|
END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
|
|
b __flush_disable_L1
|
|
|
|
/* This is the code for G3 and 74[01]0 */
|
|
flush_disable_75x:
|
|
mflr r10
|
|
|
|
/* Turn off EE and DR in MSR */
|
|
mfmsr r11
|
|
rlwinm r0,r11,0,~MSR_EE
|
|
rlwinm r0,r0,0,~MSR_DR
|
|
sync
|
|
mtmsr r0
|
|
isync
|
|
|
|
/* Stop DST streams */
|
|
BEGIN_FTR_SECTION
|
|
PPC_DSSALL
|
|
sync
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
|
|
/* Stop DPM */
|
|
mfspr r8,SPRN_HID0 /* Save SPRN_HID0 in r8 */
|
|
rlwinm r4,r8,0,12,10 /* Turn off HID0[DPM] */
|
|
sync
|
|
mtspr SPRN_HID0,r4 /* Disable DPM */
|
|
sync
|
|
|
|
/* Disp-flush L1. We have a weird problem here that I never
|
|
* totally figured out. On 750FX, using the ROM for the flush
|
|
* results in a non-working flush. We use that workaround for
|
|
* now until I finally understand what's going on. --BenH
|
|
*/
|
|
|
|
/* ROM base by default */
|
|
lis r4,0xfff0
|
|
mfpvr r3
|
|
srwi r3,r3,16
|
|
cmplwi cr0,r3,0x7000
|
|
bne+ 1f
|
|
/* RAM base on 750FX */
|
|
li r4,0
|
|
1: li r4,0x4000
|
|
mtctr r4
|
|
1: lwz r0,0(r4)
|
|
addi r4,r4,32
|
|
bdnz 1b
|
|
sync
|
|
isync
|
|
|
|
/* Disable / invalidate / enable L1 data */
|
|
mfspr r3,SPRN_HID0
|
|
rlwinm r3,r3,0,~(HID0_DCE | HID0_ICE)
|
|
mtspr SPRN_HID0,r3
|
|
sync
|
|
isync
|
|
ori r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI)
|
|
sync
|
|
isync
|
|
mtspr SPRN_HID0,r3
|
|
xori r3,r3,(HID0_DCI|HID0_ICFI)
|
|
mtspr SPRN_HID0,r3
|
|
sync
|
|
|
|
/* Get the current enable bit of the L2CR into r4 */
|
|
mfspr r5,SPRN_L2CR
|
|
/* Set to data-only (pre-745x bit) */
|
|
oris r3,r5,L2CR_L2DO@h
|
|
b 2f
|
|
/* When disabling L2, code must be in L1 */
|
|
.balign 32
|
|
1: mtspr SPRN_L2CR,r3
|
|
3: sync
|
|
isync
|
|
b 1f
|
|
2: b 3f
|
|
3: sync
|
|
isync
|
|
b 1b
|
|
1: /* disp-flush L2. The interesting thing here is that the L2 can be
|
|
* up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
|
|
* but that is probbaly fine. We disp-flush over 4Mb to be safe
|
|
*/
|
|
lis r4,2
|
|
mtctr r4
|
|
lis r4,0xfff0
|
|
1: lwz r0,0(r4)
|
|
addi r4,r4,32
|
|
bdnz 1b
|
|
sync
|
|
isync
|
|
lis r4,2
|
|
mtctr r4
|
|
lis r4,0xfff0
|
|
1: dcbf 0,r4
|
|
addi r4,r4,32
|
|
bdnz 1b
|
|
sync
|
|
isync
|
|
|
|
/* now disable L2 */
|
|
rlwinm r5,r5,0,~L2CR_L2E
|
|
b 2f
|
|
/* When disabling L2, code must be in L1 */
|
|
.balign 32
|
|
1: mtspr SPRN_L2CR,r5
|
|
3: sync
|
|
isync
|
|
b 1f
|
|
2: b 3f
|
|
3: sync
|
|
isync
|
|
b 1b
|
|
1: sync
|
|
isync
|
|
/* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
|
|
oris r4,r5,L2CR_L2I@h
|
|
mtspr SPRN_L2CR,r4
|
|
sync
|
|
isync
|
|
|
|
/* Wait for the invalidation to complete */
|
|
1: mfspr r3,SPRN_L2CR
|
|
rlwinm. r0,r3,0,31,31
|
|
bne 1b
|
|
|
|
/* Clear L2I */
|
|
xoris r4,r4,L2CR_L2I@h
|
|
sync
|
|
mtspr SPRN_L2CR,r4
|
|
sync
|
|
|
|
/* now disable the L1 data cache */
|
|
mfspr r0,SPRN_HID0
|
|
rlwinm r0,r0,0,~(HID0_DCE|HID0_ICE)
|
|
mtspr SPRN_HID0,r0
|
|
sync
|
|
isync
|
|
|
|
/* Restore HID0[DPM] to whatever it was before */
|
|
sync
|
|
mfspr r0,SPRN_HID0
|
|
rlwimi r0,r8,0,11,11 /* Turn back HID0[DPM] */
|
|
mtspr SPRN_HID0,r0
|
|
sync
|
|
|
|
/* restore DR and EE */
|
|
sync
|
|
mtmsr r11
|
|
isync
|
|
|
|
mtlr r10
|
|
blr
|
|
_ASM_NOKPROBE_SYMBOL(flush_disable_75x)
|
|
|
|
/* This code is for 745x processors */
|
|
flush_disable_745x:
|
|
/* Turn off EE and DR in MSR */
|
|
mfmsr r11
|
|
rlwinm r0,r11,0,~MSR_EE
|
|
rlwinm r0,r0,0,~MSR_DR
|
|
sync
|
|
mtmsr r0
|
|
isync
|
|
|
|
/* Stop prefetch streams */
|
|
PPC_DSSALL
|
|
sync
|
|
|
|
/* Disable L2 prefetching */
|
|
mfspr r0,SPRN_MSSCR0
|
|
rlwinm r0,r0,0,0,29
|
|
mtspr SPRN_MSSCR0,r0
|
|
sync
|
|
isync
|
|
lis r4,0
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
dcbf 0,r4
|
|
|
|
/* Due to a bug with the HW flush on some CPU revs, we occasionally
|
|
* experience data corruption. I'm adding a displacement flush along
|
|
* with a dcbf loop over a few Mb to "help". The problem isn't totally
|
|
* fixed by this in theory, but at least, in practice, I couldn't reproduce
|
|
* it even with a big hammer...
|
|
*/
|
|
|
|
lis r4,0x0002
|
|
mtctr r4
|
|
li r4,0
|
|
1:
|
|
lwz r0,0(r4)
|
|
addi r4,r4,32 /* Go to start of next cache line */
|
|
bdnz 1b
|
|
isync
|
|
|
|
/* Now, flush the first 4MB of memory */
|
|
lis r4,0x0002
|
|
mtctr r4
|
|
li r4,0
|
|
sync
|
|
1:
|
|
dcbf 0,r4
|
|
addi r4,r4,32 /* Go to start of next cache line */
|
|
bdnz 1b
|
|
|
|
/* Flush and disable the L1 data cache */
|
|
mfspr r6,SPRN_LDSTCR
|
|
lis r3,0xfff0 /* read from ROM for displacement flush */
|
|
li r4,0xfe /* start with only way 0 unlocked */
|
|
li r5,128 /* 128 lines in each way */
|
|
1: mtctr r5
|
|
rlwimi r6,r4,0,24,31
|
|
mtspr SPRN_LDSTCR,r6
|
|
sync
|
|
isync
|
|
2: lwz r0,0(r3) /* touch each cache line */
|
|
addi r3,r3,32
|
|
bdnz 2b
|
|
rlwinm r4,r4,1,24,30 /* move on to the next way */
|
|
ori r4,r4,1
|
|
cmpwi r4,0xff /* all done? */
|
|
bne 1b
|
|
/* now unlock the L1 data cache */
|
|
li r4,0
|
|
rlwimi r6,r4,0,24,31
|
|
sync
|
|
mtspr SPRN_LDSTCR,r6
|
|
sync
|
|
isync
|
|
|
|
/* Flush the L2 cache using the hardware assist */
|
|
mfspr r3,SPRN_L2CR
|
|
cmpwi r3,0 /* check if it is enabled first */
|
|
bge 4f
|
|
oris r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h
|
|
b 2f
|
|
/* When disabling/locking L2, code must be in L1 */
|
|
.balign 32
|
|
1: mtspr SPRN_L2CR,r0 /* lock the L2 cache */
|
|
3: sync
|
|
isync
|
|
b 1f
|
|
2: b 3f
|
|
3: sync
|
|
isync
|
|
b 1b
|
|
1: sync
|
|
isync
|
|
ori r0,r3,L2CR_L2HWF_745x
|
|
sync
|
|
mtspr SPRN_L2CR,r0 /* set the hardware flush bit */
|
|
3: mfspr r0,SPRN_L2CR /* wait for it to go to 0 */
|
|
andi. r0,r0,L2CR_L2HWF_745x
|
|
bne 3b
|
|
sync
|
|
rlwinm r3,r3,0,~L2CR_L2E
|
|
b 2f
|
|
/* When disabling L2, code must be in L1 */
|
|
.balign 32
|
|
1: mtspr SPRN_L2CR,r3 /* disable the L2 cache */
|
|
3: sync
|
|
isync
|
|
b 1f
|
|
2: b 3f
|
|
3: sync
|
|
isync
|
|
b 1b
|
|
1: sync
|
|
isync
|
|
oris r4,r3,L2CR_L2I@h
|
|
mtspr SPRN_L2CR,r4
|
|
sync
|
|
isync
|
|
1: mfspr r4,SPRN_L2CR
|
|
andis. r0,r4,L2CR_L2I@h
|
|
bne 1b
|
|
sync
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* Flush the L3 cache using the hardware assist */
|
|
4: mfspr r3,SPRN_L3CR
|
|
cmpwi r3,0 /* check if it is enabled */
|
|
bge 6f
|
|
oris r0,r3,L3CR_L3IO@h
|
|
ori r0,r0,L3CR_L3DO
|
|
sync
|
|
mtspr SPRN_L3CR,r0 /* lock the L3 cache */
|
|
sync
|
|
isync
|
|
ori r0,r0,L3CR_L3HWF
|
|
sync
|
|
mtspr SPRN_L3CR,r0 /* set the hardware flush bit */
|
|
5: mfspr r0,SPRN_L3CR /* wait for it to go to zero */
|
|
andi. r0,r0,L3CR_L3HWF
|
|
bne 5b
|
|
rlwinm r3,r3,0,~L3CR_L3E
|
|
sync
|
|
mtspr SPRN_L3CR,r3 /* disable the L3 cache */
|
|
sync
|
|
ori r4,r3,L3CR_L3I
|
|
mtspr SPRN_L3CR,r4
|
|
1: mfspr r4,SPRN_L3CR
|
|
andi. r0,r4,L3CR_L3I
|
|
bne 1b
|
|
sync
|
|
END_FTR_SECTION_IFSET(CPU_FTR_L3CR)
|
|
|
|
6: mfspr r0,SPRN_HID0 /* now disable the L1 data cache */
|
|
rlwinm r0,r0,0,~HID0_DCE
|
|
mtspr SPRN_HID0,r0
|
|
sync
|
|
isync
|
|
mtmsr r11 /* restore DR and EE */
|
|
isync
|
|
blr
|
|
_ASM_NOKPROBE_SYMBOL(flush_disable_745x)
|
|
#endif /* CONFIG_PPC_BOOK3S_32 */
|