tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
12
13#include <asm/reg.h>
14#include <asm/pgtable.h>
15#include <asm/mmu.h>
16#include <asm/page.h>
17#include <asm/types.h>
18#include <asm/ppc_asm.h>
19#include <asm/asm-offsets.h>
20#include <asm/cputable.h>
21
22 .text
23
24/*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
37#define STACKFRAMESIZE 256
38
39/* Save parameters offsets */
40#define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
41
42/* Save non-volatile offsets */
43#define STK_REG(i) (112 + ((i)-14)*8)
44
45
46#ifndef CONFIG_PPC_64K_PAGES
47
48/*****************************************************************************
49 * *
50 * 4K SW & 4K HW pages implementation *
51 * *
52 *****************************************************************************/
53
54
55/*
56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57 * pte_t *ptep, unsigned long trap, int local, int ssize)
58 *
59 * Adds a 4K page to the hash table in a segment of 4K pages only
60 */
61
62_GLOBAL(__hash_page_4K)
63 mflr r0
64 std r0,16(r1)
65 stdu r1,-STACKFRAMESIZE(r1)
66 /* Save all params that we need after a function call */
67 std r6,STK_PARM(r6)(r1)
68 std r8,STK_PARM(r8)(r1)
69 std r9,STK_PARM(r9)(r1)
70
71 /* Add _PAGE_PRESENT to access */
72 ori r4,r4,_PAGE_PRESENT
73
74 /* Save non-volatile registers.
75 * r31 will hold "old PTE"
76 * r30 is "new PTE"
77 * r29 is "va"
78 * r28 is a hash value
79 * r27 is hashtab mask (maybe dynamic patched instead ?)
80 */
81 std r27,STK_REG(r27)(r1)
82 std r28,STK_REG(r28)(r1)
83 std r29,STK_REG(r29)(r1)
84 std r30,STK_REG(r30)(r1)
85 std r31,STK_REG(r31)(r1)
86
87 /* Step 1:
88 *
89 * Check permissions, atomically mark the linux PTE busy
90 * and hashed.
91 */
921:
93 ldarx r31,0,r6
94 /* Check access rights (access & ~(pte_val(*ptep))) */
95 andc. r0,r4,r31
96 bne- htab_wrong_access
97 /* Check if PTE is busy */
98 andi. r0,r31,_PAGE_BUSY
99 /* If so, just bail out and refault if needed. Someone else
100 * is changing this PTE anyway and might hash it.
101 */
102 bne- htab_bail_ok
103
104 /* Prepare new PTE value (turn access RW into DIRTY, then
105 * add BUSY,HASHPTE and ACCESSED)
106 */
107 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108 or r30,r30,r31
109 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110 /* Write the linux PTE atomically (setting busy) */
111 stdcx. r30,0,r6
112 bne- 1b
113 isync
114
115 /* Step 2:
116 *
117 * Insert/Update the HPTE in the hash table. At this point,
118 * r4 (access) is re-useable, we use it for the new HPTE flags
119 */
120
121BEGIN_FTR_SECTION
122 cmpdi r9,0 /* check segment size */
123 bne 3f
124END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
125 /* Calc va and put it in r29 */
126 rldicr r29,r5,28,63-28
127 rldicl r3,r3,0,36
128 or r29,r3,r29
129
130 /* Calculate hash value for primary slot and store it in r28 */
131 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
132 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
133 xor r28,r5,r0
134 b 4f
135
1363: /* Calc VA and hash in r29 and r28 for 1T segment */
137 sldi r29,r5,40 /* vsid << 40 */
138 clrldi r3,r3,24 /* ea & 0xffffffffff */
139 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
140 clrldi r5,r5,40 /* vsid & 0xffffff */
141 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
142 xor r28,r28,r5
143 or r29,r3,r29 /* VA */
144 xor r28,r28,r0 /* hash */
145
146 /* Convert linux PTE bits into HW equivalents */
1474: andi. r3,r30,0x1fe /* Get basic set of flags */
148 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
149 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
150 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
151 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
152 andc r0,r30,r0 /* r0 = pte & ~r0 */
153 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
154 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
155
156 /* We eventually do the icache sync here (maybe inline that
157 * code rather than call a C function...)
158 */
159BEGIN_FTR_SECTION
160 mr r4,r30
161 mr r5,r7
162 bl .hash_page_do_lazy_icache
163END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
164
165 /* At this point, r3 contains new PP bits, save them in
166 * place of "access" in the param area (sic)
167 */
168 std r3,STK_PARM(r4)(r1)
169
170 /* Get htab_hash_mask */
171 ld r4,htab_hash_mask@got(2)
172 ld r27,0(r4) /* htab_hash_mask -> r27 */
173
174 /* Check if we may already be in the hashtable, in this case, we
175 * go to out-of-line code to try to modify the HPTE
176 */
177 andi. r0,r31,_PAGE_HASHPTE
178 bne htab_modify_pte
179
180htab_insert_pte:
181 /* Clear hpte bits in new pte (we also clear BUSY btw) and
182 * add _PAGE_HASHPTE
183 */
184 lis r0,_PAGE_HPTEFLAGS@h
185 ori r0,r0,_PAGE_HPTEFLAGS@l
186 andc r30,r30,r0
187 ori r30,r30,_PAGE_HASHPTE
188
189 /* physical address r5 */
190 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
191 sldi r5,r5,PAGE_SHIFT
192
193 /* Calculate primary group hash */
194 and r0,r28,r27
195 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
196
197 /* Call ppc_md.hpte_insert */
198 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
199 mr r4,r29 /* Retreive va */
200 li r7,0 /* !bolted, !secondary */
201 li r8,MMU_PAGE_4K /* page size */
202 ld r9,STK_PARM(r9)(r1) /* segment size */
203_GLOBAL(htab_call_hpte_insert1)
204 bl . /* Patched by htab_finish_init() */
205 cmpdi 0,r3,0
206 bge htab_pte_insert_ok /* Insertion successful */
207 cmpdi 0,r3,-2 /* Critical failure */
208 beq- htab_pte_insert_failure
209
210 /* Now try secondary slot */
211
212 /* physical address r5 */
213 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
214 sldi r5,r5,PAGE_SHIFT
215
216 /* Calculate secondary group hash */
217 andc r0,r27,r28
218 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
219
220 /* Call ppc_md.hpte_insert */
221 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
222 mr r4,r29 /* Retreive va */
223 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
224 li r8,MMU_PAGE_4K /* page size */
225 ld r9,STK_PARM(r9)(r1) /* segment size */
226_GLOBAL(htab_call_hpte_insert2)
227 bl . /* Patched by htab_finish_init() */
228 cmpdi 0,r3,0
229 bge+ htab_pte_insert_ok /* Insertion successful */
230 cmpdi 0,r3,-2 /* Critical failure */
231 beq- htab_pte_insert_failure
232
233 /* Both are full, we need to evict something */
234 mftb r0
235 /* Pick a random group based on TB */
236 andi. r0,r0,1
237 mr r5,r28
238 bne 2f
239 not r5,r5
2402: and r0,r5,r27
241 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
242 /* Call ppc_md.hpte_remove */
243_GLOBAL(htab_call_hpte_remove)
244 bl . /* Patched by htab_finish_init() */
245
246 /* Try all again */
247 b htab_insert_pte
248
249htab_bail_ok:
250 li r3,0
251 b htab_bail
252
253htab_pte_insert_ok:
254 /* Insert slot number & secondary bit in PTE */
255 rldimi r30,r3,12,63-15
256
257 /* Write out the PTE with a normal write
258 * (maybe add eieio may be good still ?)
259 */
260htab_write_out_pte:
261 ld r6,STK_PARM(r6)(r1)
262 std r30,0(r6)
263 li r3, 0
264htab_bail:
265 ld r27,STK_REG(r27)(r1)
266 ld r28,STK_REG(r28)(r1)
267 ld r29,STK_REG(r29)(r1)
268 ld r30,STK_REG(r30)(r1)
269 ld r31,STK_REG(r31)(r1)
270 addi r1,r1,STACKFRAMESIZE
271 ld r0,16(r1)
272 mtlr r0
273 blr
274
275htab_modify_pte:
276 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
277 mr r4,r3
278 rlwinm r3,r31,32-12,29,31
279
280 /* Secondary group ? if yes, get a inverted hash value */
281 mr r5,r28
282 andi. r0,r31,_PAGE_SECONDARY
283 beq 1f
284 not r5,r5
2851:
286 /* Calculate proper slot value for ppc_md.hpte_updatepp */
287 and r0,r5,r27
288 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
289 add r3,r0,r3 /* add slot idx */
290
291 /* Call ppc_md.hpte_updatepp */
292 mr r5,r29 /* va */
293 li r6,MMU_PAGE_4K /* page size */
294 ld r7,STK_PARM(r9)(r1) /* segment size */
295 ld r8,STK_PARM(r8)(r1) /* get "local" param */
296_GLOBAL(htab_call_hpte_updatepp)
297 bl . /* Patched by htab_finish_init() */
298
299 /* if we failed because typically the HPTE wasn't really here
300 * we try an insertion.
301 */
302 cmpdi 0,r3,-1
303 beq- htab_insert_pte
304
305 /* Clear the BUSY bit and Write out the PTE */
306 li r0,_PAGE_BUSY
307 andc r30,r30,r0
308 b htab_write_out_pte
309
310htab_wrong_access:
311 /* Bail out clearing reservation */
312 stdcx. r31,0,r6
313 li r3,1
314 b htab_bail
315
316htab_pte_insert_failure:
317 /* Bail out restoring old PTE */
318 ld r6,STK_PARM(r6)(r1)
319 std r31,0(r6)
320 li r3,-1
321 b htab_bail
322
323
324#else /* CONFIG_PPC_64K_PAGES */
325
326
327/*****************************************************************************
328 * *
329 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
330 * *
331 *****************************************************************************/
332
333/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
334 * pte_t *ptep, unsigned long trap, int local, int ssize,
335 * int subpg_prot)
336 */
337
338/*
339 * For now, we do NOT implement Admixed pages
340 */
341_GLOBAL(__hash_page_4K)
342 mflr r0
343 std r0,16(r1)
344 stdu r1,-STACKFRAMESIZE(r1)
345 /* Save all params that we need after a function call */
346 std r6,STK_PARM(r6)(r1)
347 std r8,STK_PARM(r8)(r1)
348 std r9,STK_PARM(r9)(r1)
349
350 /* Add _PAGE_PRESENT to access */
351 ori r4,r4,_PAGE_PRESENT
352
353 /* Save non-volatile registers.
354 * r31 will hold "old PTE"
355 * r30 is "new PTE"
356 * r29 is "va"
357 * r28 is a hash value
358 * r27 is hashtab mask (maybe dynamic patched instead ?)
359 * r26 is the hidx mask
360 * r25 is the index in combo page
361 */
362 std r25,STK_REG(r25)(r1)
363 std r26,STK_REG(r26)(r1)
364 std r27,STK_REG(r27)(r1)
365 std r28,STK_REG(r28)(r1)
366 std r29,STK_REG(r29)(r1)
367 std r30,STK_REG(r30)(r1)
368 std r31,STK_REG(r31)(r1)
369
370 /* Step 1:
371 *
372 * Check permissions, atomically mark the linux PTE busy
373 * and hashed.
374 */
3751:
376 ldarx r31,0,r6
377 /* Check access rights (access & ~(pte_val(*ptep))) */
378 andc. r0,r4,r31
379 bne- htab_wrong_access
380 /* Check if PTE is busy */
381 andi. r0,r31,_PAGE_BUSY
382 /* If so, just bail out and refault if needed. Someone else
383 * is changing this PTE anyway and might hash it.
384 */
385 bne- htab_bail_ok
386 /* Prepare new PTE value (turn access RW into DIRTY, then
387 * add BUSY and ACCESSED)
388 */
389 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
390 or r30,r30,r31
391 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
392 oris r30,r30,_PAGE_COMBO@h
393 /* Write the linux PTE atomically (setting busy) */
394 stdcx. r30,0,r6
395 bne- 1b
396 isync
397
398 /* Step 2:
399 *
400 * Insert/Update the HPTE in the hash table. At this point,
401 * r4 (access) is re-useable, we use it for the new HPTE flags
402 */
403
404 /* Load the hidx index */
405 rldicl r25,r3,64-12,60
406
407BEGIN_FTR_SECTION
408 cmpdi r9,0 /* check segment size */
409 bne 3f
410END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
411 /* Calc va and put it in r29 */
412 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
413 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
414 or r29,r3,r29 /* r29 = va */
415
416 /* Calculate hash value for primary slot and store it in r28 */
417 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
418 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
419 xor r28,r5,r0
420 b 4f
421
4223: /* Calc VA and hash in r29 and r28 for 1T segment */
423 sldi r29,r5,40 /* vsid << 40 */
424 clrldi r3,r3,24 /* ea & 0xffffffffff */
425 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
426 clrldi r5,r5,40 /* vsid & 0xffffff */
427 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
428 xor r28,r28,r5
429 or r29,r3,r29 /* VA */
430 xor r28,r28,r0 /* hash */
431
432 /* Convert linux PTE bits into HW equivalents */
4334:
434#ifdef CONFIG_PPC_SUBPAGE_PROT
435 andc r10,r30,r10
436 andi. r3,r10,0x1fe /* Get basic set of flags */
437 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
438#else
439 andi. r3,r30,0x1fe /* Get basic set of flags */
440 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
441#endif
442 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
443 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
444 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
445 andc r0,r3,r0 /* r0 = pte & ~r0 */
446 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
447 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
448
449 /* We eventually do the icache sync here (maybe inline that
450 * code rather than call a C function...)
451 */
452BEGIN_FTR_SECTION
453 mr r4,r30
454 mr r5,r7
455 bl .hash_page_do_lazy_icache
456END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
457
458 /* At this point, r3 contains new PP bits, save them in
459 * place of "access" in the param area (sic)
460 */
461 std r3,STK_PARM(r4)(r1)
462
463 /* Get htab_hash_mask */
464 ld r4,htab_hash_mask@got(2)
465 ld r27,0(r4) /* htab_hash_mask -> r27 */
466
467 /* Check if we may already be in the hashtable, in this case, we
468 * go to out-of-line code to try to modify the HPTE. We look for
469 * the bit at (1 >> (index + 32))
470 */
471 andi. r0,r31,_PAGE_HASHPTE
472 li r26,0 /* Default hidx */
473 beq htab_insert_pte
474
475 /*
476 * Check if the pte was already inserted into the hash table
477 * as a 64k HW page, and invalidate the 64k HPTE if so.
478 */
479 andis. r0,r31,_PAGE_COMBO@h
480 beq htab_inval_old_hpte
481
482 ld r6,STK_PARM(r6)(r1)
483 ori r26,r6,0x8000 /* Load the hidx mask */
484 ld r26,0(r26)
485 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
486 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
487 bne htab_modify_pte
488
489htab_insert_pte:
490 /* real page number in r5, PTE RPN value + index */
491 andis. r0,r31,_PAGE_4K_PFN@h
492 srdi r5,r31,PTE_RPN_SHIFT
493 bne- htab_special_pfn
494 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
495 add r5,r5,r25
496htab_special_pfn:
497 sldi r5,r5,HW_PAGE_SHIFT
498
499 /* Calculate primary group hash */
500 and r0,r28,r27
501 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
502
503 /* Call ppc_md.hpte_insert */
504 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
505 mr r4,r29 /* Retreive va */
506 li r7,0 /* !bolted, !secondary */
507 li r8,MMU_PAGE_4K /* page size */
508 ld r9,STK_PARM(r9)(r1) /* segment size */
509_GLOBAL(htab_call_hpte_insert1)
510 bl . /* patched by htab_finish_init() */
511 cmpdi 0,r3,0
512 bge htab_pte_insert_ok /* Insertion successful */
513 cmpdi 0,r3,-2 /* Critical failure */
514 beq- htab_pte_insert_failure
515
516 /* Now try secondary slot */
517
518 /* real page number in r5, PTE RPN value + index */
519 andis. r0,r31,_PAGE_4K_PFN@h
520 srdi r5,r31,PTE_RPN_SHIFT
521 bne- 3f
522 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
523 add r5,r5,r25
5243: sldi r5,r5,HW_PAGE_SHIFT
525
526 /* Calculate secondary group hash */
527 andc r0,r27,r28
528 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
529
530 /* Call ppc_md.hpte_insert */
531 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
532 mr r4,r29 /* Retreive va */
533 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
534 li r8,MMU_PAGE_4K /* page size */
535 ld r9,STK_PARM(r9)(r1) /* segment size */
536_GLOBAL(htab_call_hpte_insert2)
537 bl . /* patched by htab_finish_init() */
538 cmpdi 0,r3,0
539 bge+ htab_pte_insert_ok /* Insertion successful */
540 cmpdi 0,r3,-2 /* Critical failure */
541 beq- htab_pte_insert_failure
542
543 /* Both are full, we need to evict something */
544 mftb r0
545 /* Pick a random group based on TB */
546 andi. r0,r0,1
547 mr r5,r28
548 bne 2f
549 not r5,r5
5502: and r0,r5,r27
551 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
552 /* Call ppc_md.hpte_remove */
553_GLOBAL(htab_call_hpte_remove)
554 bl . /* patched by htab_finish_init() */
555
556 /* Try all again */
557 b htab_insert_pte
558
559 /*
560 * Call out to C code to invalidate an 64k HW HPTE that is
561 * useless now that the segment has been switched to 4k pages.
562 */
563htab_inval_old_hpte:
564 mr r3,r29 /* virtual addr */
565 mr r4,r31 /* PTE.pte */
566 li r5,0 /* PTE.hidx */
567 li r6,MMU_PAGE_64K /* psize */
568 ld r7,STK_PARM(r9)(r1) /* ssize */
569 ld r8,STK_PARM(r8)(r1) /* local */
570 bl .flush_hash_page
571 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
572 lis r0,_PAGE_HPTE_SUB@h
573 ori r0,r0,_PAGE_HPTE_SUB@l
574 andc r30,r30,r0
575 b htab_insert_pte
576
577htab_bail_ok:
578 li r3,0
579 b htab_bail
580
581htab_pte_insert_ok:
582 /* Insert slot number & secondary bit in PTE second half,
583 * clear _PAGE_BUSY and set approriate HPTE slot bit
584 */
585 ld r6,STK_PARM(r6)(r1)
586 li r0,_PAGE_BUSY
587 andc r30,r30,r0
588 /* HPTE SUB bit */
589 li r0,1
590 subfic r5,r25,27 /* Must match bit position in */
591 sld r0,r0,r5 /* pgtable.h */
592 or r30,r30,r0
593 /* hindx */
594 sldi r5,r25,2
595 sld r3,r3,r5
596 li r4,0xf
597 sld r4,r4,r5
598 andc r26,r26,r4
599 or r26,r26,r3
600 ori r5,r6,0x8000
601 std r26,0(r5)
602 lwsync
603 std r30,0(r6)
604 li r3, 0
605htab_bail:
606 ld r25,STK_REG(r25)(r1)
607 ld r26,STK_REG(r26)(r1)
608 ld r27,STK_REG(r27)(r1)
609 ld r28,STK_REG(r28)(r1)
610 ld r29,STK_REG(r29)(r1)
611 ld r30,STK_REG(r30)(r1)
612 ld r31,STK_REG(r31)(r1)
613 addi r1,r1,STACKFRAMESIZE
614 ld r0,16(r1)
615 mtlr r0
616 blr
617
618htab_modify_pte:
619 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
620 mr r4,r3
621 sldi r5,r25,2
622 srd r3,r26,r5
623
624 /* Secondary group ? if yes, get a inverted hash value */
625 mr r5,r28
626 andi. r0,r3,0x8 /* page secondary ? */
627 beq 1f
628 not r5,r5
6291: andi. r3,r3,0x7 /* extract idx alone */
630
631 /* Calculate proper slot value for ppc_md.hpte_updatepp */
632 and r0,r5,r27
633 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
634 add r3,r0,r3 /* add slot idx */
635
636 /* Call ppc_md.hpte_updatepp */
637 mr r5,r29 /* va */
638 li r6,MMU_PAGE_4K /* page size */
639 ld r7,STK_PARM(r9)(r1) /* segment size */
640 ld r8,STK_PARM(r8)(r1) /* get "local" param */
641_GLOBAL(htab_call_hpte_updatepp)
642 bl . /* patched by htab_finish_init() */
643
644 /* if we failed because typically the HPTE wasn't really here
645 * we try an insertion.
646 */
647 cmpdi 0,r3,-1
648 beq- htab_insert_pte
649
650 /* Clear the BUSY bit and Write out the PTE */
651 li r0,_PAGE_BUSY
652 andc r30,r30,r0
653 ld r6,STK_PARM(r6)(r1)
654 std r30,0(r6)
655 li r3,0
656 b htab_bail
657
658htab_wrong_access:
659 /* Bail out clearing reservation */
660 stdcx. r31,0,r6
661 li r3,1
662 b htab_bail
663
664htab_pte_insert_failure:
665 /* Bail out restoring old PTE */
666 ld r6,STK_PARM(r6)(r1)
667 std r31,0(r6)
668 li r3,-1
669 b htab_bail
670
671#endif /* CONFIG_PPC_64K_PAGES */
672
673#ifdef CONFIG_PPC_HAS_HASH_64K
674
675/*****************************************************************************
676 * *
677 * 64K SW & 64K HW in a 64K segment pages implementation *
678 * *
679 *****************************************************************************/
680
681_GLOBAL(__hash_page_64K)
682 mflr r0
683 std r0,16(r1)
684 stdu r1,-STACKFRAMESIZE(r1)
685 /* Save all params that we need after a function call */
686 std r6,STK_PARM(r6)(r1)
687 std r8,STK_PARM(r8)(r1)
688 std r9,STK_PARM(r9)(r1)
689
690 /* Add _PAGE_PRESENT to access */
691 ori r4,r4,_PAGE_PRESENT
692
693 /* Save non-volatile registers.
694 * r31 will hold "old PTE"
695 * r30 is "new PTE"
696 * r29 is "va"
697 * r28 is a hash value
698 * r27 is hashtab mask (maybe dynamic patched instead ?)
699 */
700 std r27,STK_REG(r27)(r1)
701 std r28,STK_REG(r28)(r1)
702 std r29,STK_REG(r29)(r1)
703 std r30,STK_REG(r30)(r1)
704 std r31,STK_REG(r31)(r1)
705
706 /* Step 1:
707 *
708 * Check permissions, atomically mark the linux PTE busy
709 * and hashed.
710 */
7111:
712 ldarx r31,0,r6
713 /* Check access rights (access & ~(pte_val(*ptep))) */
714 andc. r0,r4,r31
715 bne- ht64_wrong_access
716 /* Check if PTE is busy */
717 andi. r0,r31,_PAGE_BUSY
718 /* If so, just bail out and refault if needed. Someone else
719 * is changing this PTE anyway and might hash it.
720 */
721 bne- ht64_bail_ok
722BEGIN_FTR_SECTION
723 /* Check if PTE has the cache-inhibit bit set */
724 andi. r0,r31,_PAGE_NO_CACHE
725 /* If so, bail out and refault as a 4k page */
726 bne- ht64_bail_ok
727END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
728 /* Prepare new PTE value (turn access RW into DIRTY, then
729 * add BUSY,HASHPTE and ACCESSED)
730 */
731 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
732 or r30,r30,r31
733 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
734 /* Write the linux PTE atomically (setting busy) */
735 stdcx. r30,0,r6
736 bne- 1b
737 isync
738
739 /* Step 2:
740 *
741 * Insert/Update the HPTE in the hash table. At this point,
742 * r4 (access) is re-useable, we use it for the new HPTE flags
743 */
744
745BEGIN_FTR_SECTION
746 cmpdi r9,0 /* check segment size */
747 bne 3f
748END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
749 /* Calc va and put it in r29 */
750 rldicr r29,r5,28,63-28
751 rldicl r3,r3,0,36
752 or r29,r3,r29
753
754 /* Calculate hash value for primary slot and store it in r28 */
755 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
756 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
757 xor r28,r5,r0
758 b 4f
759
7603: /* Calc VA and hash in r29 and r28 for 1T segment */
761 sldi r29,r5,40 /* vsid << 40 */
762 clrldi r3,r3,24 /* ea & 0xffffffffff */
763 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
764 clrldi r5,r5,40 /* vsid & 0xffffff */
765 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
766 xor r28,r28,r5
767 or r29,r3,r29 /* VA */
768 xor r28,r28,r0 /* hash */
769
770 /* Convert linux PTE bits into HW equivalents */
7714: andi. r3,r30,0x1fe /* Get basic set of flags */
772 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
773 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
774 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
775 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
776 andc r0,r30,r0 /* r0 = pte & ~r0 */
777 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
778 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
779
780 /* We eventually do the icache sync here (maybe inline that
781 * code rather than call a C function...)
782 */
783BEGIN_FTR_SECTION
784 mr r4,r30
785 mr r5,r7
786 bl .hash_page_do_lazy_icache
787END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
788
789 /* At this point, r3 contains new PP bits, save them in
790 * place of "access" in the param area (sic)
791 */
792 std r3,STK_PARM(r4)(r1)
793
794 /* Get htab_hash_mask */
795 ld r4,htab_hash_mask@got(2)
796 ld r27,0(r4) /* htab_hash_mask -> r27 */
797
798 /* Check if we may already be in the hashtable, in this case, we
799 * go to out-of-line code to try to modify the HPTE
800 */
801 andi. r0,r31,_PAGE_HASHPTE
802 bne ht64_modify_pte
803
804ht64_insert_pte:
805 /* Clear hpte bits in new pte (we also clear BUSY btw) and
806 * add _PAGE_HASHPTE
807 */
808 lis r0,_PAGE_HPTEFLAGS@h
809 ori r0,r0,_PAGE_HPTEFLAGS@l
810 andc r30,r30,r0
811 ori r30,r30,_PAGE_HASHPTE
812
813 /* Phyical address in r5 */
814 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
815 sldi r5,r5,PAGE_SHIFT
816
817 /* Calculate primary group hash */
818 and r0,r28,r27
819 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
820
821 /* Call ppc_md.hpte_insert */
822 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
823 mr r4,r29 /* Retreive va */
824 li r7,0 /* !bolted, !secondary */
825 li r8,MMU_PAGE_64K
826 ld r9,STK_PARM(r9)(r1) /* segment size */
827_GLOBAL(ht64_call_hpte_insert1)
828 bl . /* patched by htab_finish_init() */
829 cmpdi 0,r3,0
830 bge ht64_pte_insert_ok /* Insertion successful */
831 cmpdi 0,r3,-2 /* Critical failure */
832 beq- ht64_pte_insert_failure
833
834 /* Now try secondary slot */
835
836 /* Phyical address in r5 */
837 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
838 sldi r5,r5,PAGE_SHIFT
839
840 /* Calculate secondary group hash */
841 andc r0,r27,r28
842 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
843
844 /* Call ppc_md.hpte_insert */
845 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
846 mr r4,r29 /* Retreive va */
847 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
848 li r8,MMU_PAGE_64K
849 ld r9,STK_PARM(r9)(r1) /* segment size */
850_GLOBAL(ht64_call_hpte_insert2)
851 bl . /* patched by htab_finish_init() */
852 cmpdi 0,r3,0
853 bge+ ht64_pte_insert_ok /* Insertion successful */
854 cmpdi 0,r3,-2 /* Critical failure */
855 beq- ht64_pte_insert_failure
856
857 /* Both are full, we need to evict something */
858 mftb r0
859 /* Pick a random group based on TB */
860 andi. r0,r0,1
861 mr r5,r28
862 bne 2f
863 not r5,r5
8642: and r0,r5,r27
865 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
866 /* Call ppc_md.hpte_remove */
867_GLOBAL(ht64_call_hpte_remove)
868 bl . /* patched by htab_finish_init() */
869
870 /* Try all again */
871 b ht64_insert_pte
872
873ht64_bail_ok:
874 li r3,0
875 b ht64_bail
876
877ht64_pte_insert_ok:
878 /* Insert slot number & secondary bit in PTE */
879 rldimi r30,r3,12,63-15
880
881 /* Write out the PTE with a normal write
882 * (maybe add eieio may be good still ?)
883 */
884ht64_write_out_pte:
885 ld r6,STK_PARM(r6)(r1)
886 std r30,0(r6)
887 li r3, 0
888ht64_bail:
889 ld r27,STK_REG(r27)(r1)
890 ld r28,STK_REG(r28)(r1)
891 ld r29,STK_REG(r29)(r1)
892 ld r30,STK_REG(r30)(r1)
893 ld r31,STK_REG(r31)(r1)
894 addi r1,r1,STACKFRAMESIZE
895 ld r0,16(r1)
896 mtlr r0
897 blr
898
899ht64_modify_pte:
900 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
901 mr r4,r3
902 rlwinm r3,r31,32-12,29,31
903
904 /* Secondary group ? if yes, get a inverted hash value */
905 mr r5,r28
906 andi. r0,r31,_PAGE_F_SECOND
907 beq 1f
908 not r5,r5
9091:
910 /* Calculate proper slot value for ppc_md.hpte_updatepp */
911 and r0,r5,r27
912 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
913 add r3,r0,r3 /* add slot idx */
914
915 /* Call ppc_md.hpte_updatepp */
916 mr r5,r29 /* va */
917 li r6,MMU_PAGE_64K
918 ld r7,STK_PARM(r9)(r1) /* segment size */
919 ld r8,STK_PARM(r8)(r1) /* get "local" param */
920_GLOBAL(ht64_call_hpte_updatepp)
921 bl . /* patched by htab_finish_init() */
922
923 /* if we failed because typically the HPTE wasn't really here
924 * we try an insertion.
925 */
926 cmpdi 0,r3,-1
927 beq- ht64_insert_pte
928
929 /* Clear the BUSY bit and Write out the PTE */
930 li r0,_PAGE_BUSY
931 andc r30,r30,r0
932 b ht64_write_out_pte
933
934ht64_wrong_access:
935 /* Bail out clearing reservation */
936 stdcx. r31,0,r6
937 li r3,1
938 b ht64_bail
939
940ht64_pte_insert_failure:
941 /* Bail out restoring old PTE */
942 ld r6,STK_PARM(r6)(r1)
943 std r31,0(r6)
944 li r3,-1
945 b ht64_bail
946
947
948#endif /* CONFIG_PPC_HAS_HASH_64K */
949
950
951/*****************************************************************************
952 * *
953 * Huge pages implementation is in hugetlbpage.c *
954 * *
955 *****************************************************************************/