Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * This file contains low-level cache management functions
4 * used for sleep and CPU speed changes on Apple machines.
5 * (In fact the only thing that is Apple-specific is that we assume
6 * that we can read from ROM at physical address 0xfff00000.)
7 *
8 * Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and
9 * Benjamin Herrenschmidt (benh@kernel.crashing.org)
10 */
11
12#include <asm/processor.h>
13#include <asm/ppc_asm.h>
14#include <asm/cputable.h>
15#include <asm/feature-fixups.h>
16
17/*
18 * Flush and disable all data caches (dL1, L2, L3). This is used
19 * when going to sleep, when doing a PMU based cpufreq transition,
20 * or when "offlining" a CPU on SMP machines. This code is over
21 * paranoid, but I've had enough issues with various CPU revs and
22 * bugs that I decided it was worth being over cautious
23 */
24
25_GLOBAL(flush_disable_caches)
26#ifndef CONFIG_PPC_BOOK3S_32
27 blr
28#else
29BEGIN_FTR_SECTION
30 b flush_disable_745x
31END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
32BEGIN_FTR_SECTION
33 b flush_disable_75x
34END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
35 b __flush_disable_L1
36
37/* This is the code for G3 and 74[01]0 */
38flush_disable_75x:
39 mflr r10
40
41 /* Turn off EE and DR in MSR */
42 mfmsr r11
43 rlwinm r0,r11,0,~MSR_EE
44 rlwinm r0,r0,0,~MSR_DR
45 sync
46 mtmsr r0
47 isync
48
49 /* Stop DST streams */
50BEGIN_FTR_SECTION
51 PPC_DSSALL
52 sync
53END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
54
55 /* Stop DPM */
56 mfspr r8,SPRN_HID0 /* Save SPRN_HID0 in r8 */
57 rlwinm r4,r8,0,12,10 /* Turn off HID0[DPM] */
58 sync
59 mtspr SPRN_HID0,r4 /* Disable DPM */
60 sync
61
62 /* Disp-flush L1. We have a weird problem here that I never
63 * totally figured out. On 750FX, using the ROM for the flush
64 * results in a non-working flush. We use that workaround for
65 * now until I finally understand what's going on. --BenH
66 */
67
68 /* ROM base by default */
69 lis r4,0xfff0
70 mfpvr r3
71 srwi r3,r3,16
72 cmplwi cr0,r3,0x7000
73 bne+ 1f
74 /* RAM base on 750FX */
75 li r4,0
761: li r4,0x4000
77 mtctr r4
781: lwz r0,0(r4)
79 addi r4,r4,32
80 bdnz 1b
81 sync
82 isync
83
84 /* Disable / invalidate / enable L1 data */
85 mfspr r3,SPRN_HID0
86 rlwinm r3,r3,0,~(HID0_DCE | HID0_ICE)
87 mtspr SPRN_HID0,r3
88 sync
89 isync
90 ori r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI)
91 sync
92 isync
93 mtspr SPRN_HID0,r3
94 xori r3,r3,(HID0_DCI|HID0_ICFI)
95 mtspr SPRN_HID0,r3
96 sync
97
98 /* Get the current enable bit of the L2CR into r4 */
99 mfspr r5,SPRN_L2CR
100 /* Set to data-only (pre-745x bit) */
101 oris r3,r5,L2CR_L2DO@h
102 b 2f
103 /* When disabling L2, code must be in L1 */
104 .balign 32
1051: mtspr SPRN_L2CR,r3
1063: sync
107 isync
108 b 1f
1092: b 3f
1103: sync
111 isync
112 b 1b
1131: /* disp-flush L2. The interesting thing here is that the L2 can be
114 * up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
115 * but that is probbaly fine. We disp-flush over 4Mb to be safe
116 */
117 lis r4,2
118 mtctr r4
119 lis r4,0xfff0
1201: lwz r0,0(r4)
121 addi r4,r4,32
122 bdnz 1b
123 sync
124 isync
125 lis r4,2
126 mtctr r4
127 lis r4,0xfff0
1281: dcbf 0,r4
129 addi r4,r4,32
130 bdnz 1b
131 sync
132 isync
133
134 /* now disable L2 */
135 rlwinm r5,r5,0,~L2CR_L2E
136 b 2f
137 /* When disabling L2, code must be in L1 */
138 .balign 32
1391: mtspr SPRN_L2CR,r5
1403: sync
141 isync
142 b 1f
1432: b 3f
1443: sync
145 isync
146 b 1b
1471: sync
148 isync
149 /* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
150 oris r4,r5,L2CR_L2I@h
151 mtspr SPRN_L2CR,r4
152 sync
153 isync
154
155 /* Wait for the invalidation to complete */
1561: mfspr r3,SPRN_L2CR
157 rlwinm. r0,r3,0,31,31
158 bne 1b
159
160 /* Clear L2I */
161 xoris r4,r4,L2CR_L2I@h
162 sync
163 mtspr SPRN_L2CR,r4
164 sync
165
166 /* now disable the L1 data cache */
167 mfspr r0,SPRN_HID0
168 rlwinm r0,r0,0,~(HID0_DCE|HID0_ICE)
169 mtspr SPRN_HID0,r0
170 sync
171 isync
172
173 /* Restore HID0[DPM] to whatever it was before */
174 sync
175 mfspr r0,SPRN_HID0
176 rlwimi r0,r8,0,11,11 /* Turn back HID0[DPM] */
177 mtspr SPRN_HID0,r0
178 sync
179
180 /* restore DR and EE */
181 sync
182 mtmsr r11
183 isync
184
185 mtlr r10
186 blr
187_ASM_NOKPROBE_SYMBOL(flush_disable_75x)
188
189/* This code is for 745x processors */
190flush_disable_745x:
191 /* Turn off EE and DR in MSR */
192 mfmsr r11
193 rlwinm r0,r11,0,~MSR_EE
194 rlwinm r0,r0,0,~MSR_DR
195 sync
196 mtmsr r0
197 isync
198
199 /* Stop prefetch streams */
200 PPC_DSSALL
201 sync
202
203 /* Disable L2 prefetching */
204 mfspr r0,SPRN_MSSCR0
205 rlwinm r0,r0,0,0,29
206 mtspr SPRN_MSSCR0,r0
207 sync
208 isync
209 lis r4,0
210 dcbf 0,r4
211 dcbf 0,r4
212 dcbf 0,r4
213 dcbf 0,r4
214 dcbf 0,r4
215 dcbf 0,r4
216 dcbf 0,r4
217 dcbf 0,r4
218
219 /* Due to a bug with the HW flush on some CPU revs, we occasionally
220 * experience data corruption. I'm adding a displacement flush along
221 * with a dcbf loop over a few Mb to "help". The problem isn't totally
222 * fixed by this in theory, but at least, in practice, I couldn't reproduce
223 * it even with a big hammer...
224 */
225
226 lis r4,0x0002
227 mtctr r4
228 li r4,0
2291:
230 lwz r0,0(r4)
231 addi r4,r4,32 /* Go to start of next cache line */
232 bdnz 1b
233 isync
234
235 /* Now, flush the first 4MB of memory */
236 lis r4,0x0002
237 mtctr r4
238 li r4,0
239 sync
2401:
241 dcbf 0,r4
242 addi r4,r4,32 /* Go to start of next cache line */
243 bdnz 1b
244
245 /* Flush and disable the L1 data cache */
246 mfspr r6,SPRN_LDSTCR
247 lis r3,0xfff0 /* read from ROM for displacement flush */
248 li r4,0xfe /* start with only way 0 unlocked */
249 li r5,128 /* 128 lines in each way */
2501: mtctr r5
251 rlwimi r6,r4,0,24,31
252 mtspr SPRN_LDSTCR,r6
253 sync
254 isync
2552: lwz r0,0(r3) /* touch each cache line */
256 addi r3,r3,32
257 bdnz 2b
258 rlwinm r4,r4,1,24,30 /* move on to the next way */
259 ori r4,r4,1
260 cmpwi r4,0xff /* all done? */
261 bne 1b
262 /* now unlock the L1 data cache */
263 li r4,0
264 rlwimi r6,r4,0,24,31
265 sync
266 mtspr SPRN_LDSTCR,r6
267 sync
268 isync
269
270 /* Flush the L2 cache using the hardware assist */
271 mfspr r3,SPRN_L2CR
272 cmpwi r3,0 /* check if it is enabled first */
273 bge 4f
274 oris r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h
275 b 2f
276 /* When disabling/locking L2, code must be in L1 */
277 .balign 32
2781: mtspr SPRN_L2CR,r0 /* lock the L2 cache */
2793: sync
280 isync
281 b 1f
2822: b 3f
2833: sync
284 isync
285 b 1b
2861: sync
287 isync
288 ori r0,r3,L2CR_L2HWF_745x
289 sync
290 mtspr SPRN_L2CR,r0 /* set the hardware flush bit */
2913: mfspr r0,SPRN_L2CR /* wait for it to go to 0 */
292 andi. r0,r0,L2CR_L2HWF_745x
293 bne 3b
294 sync
295 rlwinm r3,r3,0,~L2CR_L2E
296 b 2f
297 /* When disabling L2, code must be in L1 */
298 .balign 32
2991: mtspr SPRN_L2CR,r3 /* disable the L2 cache */
3003: sync
301 isync
302 b 1f
3032: b 3f
3043: sync
305 isync
306 b 1b
3071: sync
308 isync
309 oris r4,r3,L2CR_L2I@h
310 mtspr SPRN_L2CR,r4
311 sync
312 isync
3131: mfspr r4,SPRN_L2CR
314 andis. r0,r4,L2CR_L2I@h
315 bne 1b
316 sync
317
318BEGIN_FTR_SECTION
319 /* Flush the L3 cache using the hardware assist */
3204: mfspr r3,SPRN_L3CR
321 cmpwi r3,0 /* check if it is enabled */
322 bge 6f
323 oris r0,r3,L3CR_L3IO@h
324 ori r0,r0,L3CR_L3DO
325 sync
326 mtspr SPRN_L3CR,r0 /* lock the L3 cache */
327 sync
328 isync
329 ori r0,r0,L3CR_L3HWF
330 sync
331 mtspr SPRN_L3CR,r0 /* set the hardware flush bit */
3325: mfspr r0,SPRN_L3CR /* wait for it to go to zero */
333 andi. r0,r0,L3CR_L3HWF
334 bne 5b
335 rlwinm r3,r3,0,~L3CR_L3E
336 sync
337 mtspr SPRN_L3CR,r3 /* disable the L3 cache */
338 sync
339 ori r4,r3,L3CR_L3I
340 mtspr SPRN_L3CR,r4
3411: mfspr r4,SPRN_L3CR
342 andi. r0,r4,L3CR_L3I
343 bne 1b
344 sync
345END_FTR_SECTION_IFSET(CPU_FTR_L3CR)
346
3476: mfspr r0,SPRN_HID0 /* now disable the L1 data cache */
348 rlwinm r0,r0,0,~HID0_DCE
349 mtspr SPRN_HID0,r0
350 sync
351 isync
352 mtmsr r11 /* restore DR and EE */
353 isync
354 blr
355_ASM_NOKPROBE_SYMBOL(flush_disable_745x)
356#endif /* CONFIG_PPC_BOOK3S_32 */