tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
12
13#include <asm/reg.h>
14#include <asm/pgtable.h>
15#include <asm/mmu.h>
16#include <asm/page.h>
17#include <asm/types.h>
18#include <asm/ppc_asm.h>
19#include <asm/asm-offsets.h>
20#include <asm/cputable.h>
21
22 .text
23
24/*
25 * Stackframe:
26 *
27 * +-> Back chain (SP + 256)
28 * | General register save area (SP + 112)
29 * | Parameter save area (SP + 48)
30 * | TOC save area (SP + 40)
31 * | link editor doubleword (SP + 32)
32 * | compiler doubleword (SP + 24)
33 * | LR save area (SP + 16)
34 * | CR save area (SP + 8)
35 * SP ---> +-- Back chain (SP + 0)
36 */
37#define STACKFRAMESIZE 256
38
39/* Save parameters offsets */
40#define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
41
42/* Save non-volatile offsets */
43#define STK_REG(i) (112 + ((i)-14)*8)
44
45
46#ifndef CONFIG_PPC_64K_PAGES
47
48/*****************************************************************************
49 * *
50 * 4K SW & 4K HW pages implementation *
51 * *
52 *****************************************************************************/
53
54
55/*
56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57 * pte_t *ptep, unsigned long trap, int local)
58 *
59 * Adds a 4K page to the hash table in a segment of 4K pages only
60 */
61
62_GLOBAL(__hash_page_4K)
63 mflr r0
64 std r0,16(r1)
65 stdu r1,-STACKFRAMESIZE(r1)
66 /* Save all params that we need after a function call */
67 std r6,STK_PARM(r6)(r1)
68 std r8,STK_PARM(r8)(r1)
69
70 /* Add _PAGE_PRESENT to access */
71 ori r4,r4,_PAGE_PRESENT
72
73 /* Save non-volatile registers.
74 * r31 will hold "old PTE"
75 * r30 is "new PTE"
76 * r29 is "va"
77 * r28 is a hash value
78 * r27 is hashtab mask (maybe dynamic patched instead ?)
79 */
80 std r27,STK_REG(r27)(r1)
81 std r28,STK_REG(r28)(r1)
82 std r29,STK_REG(r29)(r1)
83 std r30,STK_REG(r30)(r1)
84 std r31,STK_REG(r31)(r1)
85
86 /* Step 1:
87 *
88 * Check permissions, atomically mark the linux PTE busy
89 * and hashed.
90 */
911:
92 ldarx r31,0,r6
93 /* Check access rights (access & ~(pte_val(*ptep))) */
94 andc. r0,r4,r31
95 bne- htab_wrong_access
96 /* Check if PTE is busy */
97 andi. r0,r31,_PAGE_BUSY
98 /* If so, just bail out and refault if needed. Someone else
99 * is changing this PTE anyway and might hash it.
100 */
101 bne- htab_bail_ok
102
103 /* Prepare new PTE value (turn access RW into DIRTY, then
104 * add BUSY,HASHPTE and ACCESSED)
105 */
106 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
107 or r30,r30,r31
108 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
109 /* Write the linux PTE atomically (setting busy) */
110 stdcx. r30,0,r6
111 bne- 1b
112 isync
113
114 /* Step 2:
115 *
116 * Insert/Update the HPTE in the hash table. At this point,
117 * r4 (access) is re-useable, we use it for the new HPTE flags
118 */
119
120 /* Calc va and put it in r29 */
121 rldicr r29,r5,28,63-28
122 rldicl r3,r3,0,36
123 or r29,r3,r29
124
125 /* Calculate hash value for primary slot and store it in r28 */
126 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
127 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
128 xor r28,r5,r0
129
130 /* Convert linux PTE bits into HW equivalents */
131 andi. r3,r30,0x1fe /* Get basic set of flags */
132 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
133 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
134 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
135 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
136 andc r0,r30,r0 /* r0 = pte & ~r0 */
137 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
138 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
139
140 /* We eventually do the icache sync here (maybe inline that
141 * code rather than call a C function...)
142 */
143BEGIN_FTR_SECTION
144 mr r4,r30
145 mr r5,r7
146 bl .hash_page_do_lazy_icache
147END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
148
149 /* At this point, r3 contains new PP bits, save them in
150 * place of "access" in the param area (sic)
151 */
152 std r3,STK_PARM(r4)(r1)
153
154 /* Get htab_hash_mask */
155 ld r4,htab_hash_mask@got(2)
156 ld r27,0(r4) /* htab_hash_mask -> r27 */
157
158 /* Check if we may already be in the hashtable, in this case, we
159 * go to out-of-line code to try to modify the HPTE
160 */
161 andi. r0,r31,_PAGE_HASHPTE
162 bne htab_modify_pte
163
164htab_insert_pte:
165 /* Clear hpte bits in new pte (we also clear BUSY btw) and
166 * add _PAGE_HASHPTE
167 */
168 lis r0,_PAGE_HPTEFLAGS@h
169 ori r0,r0,_PAGE_HPTEFLAGS@l
170 andc r30,r30,r0
171 ori r30,r30,_PAGE_HASHPTE
172
173 /* physical address r5 */
174 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
175 sldi r5,r5,PAGE_SHIFT
176
177 /* Calculate primary group hash */
178 and r0,r28,r27
179 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
180
181 /* Call ppc_md.hpte_insert */
182 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
183 mr r4,r29 /* Retreive va */
184 li r7,0 /* !bolted, !secondary */
185 li r8,MMU_PAGE_4K /* page size */
186_GLOBAL(htab_call_hpte_insert1)
187 bl . /* Patched by htab_finish_init() */
188 cmpdi 0,r3,0
189 bge htab_pte_insert_ok /* Insertion successful */
190 cmpdi 0,r3,-2 /* Critical failure */
191 beq- htab_pte_insert_failure
192
193 /* Now try secondary slot */
194
195 /* physical address r5 */
196 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
197 sldi r5,r5,PAGE_SHIFT
198
199 /* Calculate secondary group hash */
200 andc r0,r27,r28
201 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
202
203 /* Call ppc_md.hpte_insert */
204 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
205 mr r4,r29 /* Retreive va */
206 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
207 li r8,MMU_PAGE_4K /* page size */
208_GLOBAL(htab_call_hpte_insert2)
209 bl . /* Patched by htab_finish_init() */
210 cmpdi 0,r3,0
211 bge+ htab_pte_insert_ok /* Insertion successful */
212 cmpdi 0,r3,-2 /* Critical failure */
213 beq- htab_pte_insert_failure
214
215 /* Both are full, we need to evict something */
216 mftb r0
217 /* Pick a random group based on TB */
218 andi. r0,r0,1
219 mr r5,r28
220 bne 2f
221 not r5,r5
2222: and r0,r5,r27
223 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
224 /* Call ppc_md.hpte_remove */
225_GLOBAL(htab_call_hpte_remove)
226 bl . /* Patched by htab_finish_init() */
227
228 /* Try all again */
229 b htab_insert_pte
230
231htab_bail_ok:
232 li r3,0
233 b htab_bail
234
235htab_pte_insert_ok:
236 /* Insert slot number & secondary bit in PTE */
237 rldimi r30,r3,12,63-15
238
239 /* Write out the PTE with a normal write
240 * (maybe add eieio may be good still ?)
241 */
242htab_write_out_pte:
243 ld r6,STK_PARM(r6)(r1)
244 std r30,0(r6)
245 li r3, 0
246htab_bail:
247 ld r27,STK_REG(r27)(r1)
248 ld r28,STK_REG(r28)(r1)
249 ld r29,STK_REG(r29)(r1)
250 ld r30,STK_REG(r30)(r1)
251 ld r31,STK_REG(r31)(r1)
252 addi r1,r1,STACKFRAMESIZE
253 ld r0,16(r1)
254 mtlr r0
255 blr
256
257htab_modify_pte:
258 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
259 mr r4,r3
260 rlwinm r3,r31,32-12,29,31
261
262 /* Secondary group ? if yes, get a inverted hash value */
263 mr r5,r28
264 andi. r0,r31,_PAGE_SECONDARY
265 beq 1f
266 not r5,r5
2671:
268 /* Calculate proper slot value for ppc_md.hpte_updatepp */
269 and r0,r5,r27
270 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
271 add r3,r0,r3 /* add slot idx */
272
273 /* Call ppc_md.hpte_updatepp */
274 mr r5,r29 /* va */
275 li r6,MMU_PAGE_4K /* page size */
276 ld r7,STK_PARM(r8)(r1) /* get "local" param */
277_GLOBAL(htab_call_hpte_updatepp)
278 bl . /* Patched by htab_finish_init() */
279
280 /* if we failed because typically the HPTE wasn't really here
281 * we try an insertion.
282 */
283 cmpdi 0,r3,-1
284 beq- htab_insert_pte
285
286 /* Clear the BUSY bit and Write out the PTE */
287 li r0,_PAGE_BUSY
288 andc r30,r30,r0
289 b htab_write_out_pte
290
291htab_wrong_access:
292 /* Bail out clearing reservation */
293 stdcx. r31,0,r6
294 li r3,1
295 b htab_bail
296
297htab_pte_insert_failure:
298 /* Bail out restoring old PTE */
299 ld r6,STK_PARM(r6)(r1)
300 std r31,0(r6)
301 li r3,-1
302 b htab_bail
303
304
305#else /* CONFIG_PPC_64K_PAGES */
306
307
308/*****************************************************************************
309 * *
310 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
311 * *
312 *****************************************************************************/
313
314/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
315 * pte_t *ptep, unsigned long trap, int local)
316 */
317
318/*
319 * For now, we do NOT implement Admixed pages
320 */
321_GLOBAL(__hash_page_4K)
322 mflr r0
323 std r0,16(r1)
324 stdu r1,-STACKFRAMESIZE(r1)
325 /* Save all params that we need after a function call */
326 std r6,STK_PARM(r6)(r1)
327 std r8,STK_PARM(r8)(r1)
328
329 /* Add _PAGE_PRESENT to access */
330 ori r4,r4,_PAGE_PRESENT
331
332 /* Save non-volatile registers.
333 * r31 will hold "old PTE"
334 * r30 is "new PTE"
335 * r29 is "va"
336 * r28 is a hash value
337 * r27 is hashtab mask (maybe dynamic patched instead ?)
338 * r26 is the hidx mask
339 * r25 is the index in combo page
340 */
341 std r25,STK_REG(r25)(r1)
342 std r26,STK_REG(r26)(r1)
343 std r27,STK_REG(r27)(r1)
344 std r28,STK_REG(r28)(r1)
345 std r29,STK_REG(r29)(r1)
346 std r30,STK_REG(r30)(r1)
347 std r31,STK_REG(r31)(r1)
348
349 /* Step 1:
350 *
351 * Check permissions, atomically mark the linux PTE busy
352 * and hashed.
353 */
3541:
355 ldarx r31,0,r6
356 /* Check access rights (access & ~(pte_val(*ptep))) */
357 andc. r0,r4,r31
358 bne- htab_wrong_access
359 /* Check if PTE is busy */
360 andi. r0,r31,_PAGE_BUSY
361 /* If so, just bail out and refault if needed. Someone else
362 * is changing this PTE anyway and might hash it.
363 */
364 bne- htab_bail_ok
365 /* Prepare new PTE value (turn access RW into DIRTY, then
366 * add BUSY and ACCESSED)
367 */
368 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
369 or r30,r30,r31
370 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
371 oris r30,r30,_PAGE_COMBO@h
372 /* Write the linux PTE atomically (setting busy) */
373 stdcx. r30,0,r6
374 bne- 1b
375 isync
376
377 /* Step 2:
378 *
379 * Insert/Update the HPTE in the hash table. At this point,
380 * r4 (access) is re-useable, we use it for the new HPTE flags
381 */
382
383 /* Load the hidx index */
384 rldicl r25,r3,64-12,60
385
386 /* Calc va and put it in r29 */
387 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
388 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
389 or r29,r3,r29 /* r29 = va
390
391 /* Calculate hash value for primary slot and store it in r28 */
392 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
393 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
394 xor r28,r5,r0
395
396 /* Convert linux PTE bits into HW equivalents */
397 andi. r3,r30,0x1fe /* Get basic set of flags */
398 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
399 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
400 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
401 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
402 andc r0,r30,r0 /* r0 = pte & ~r0 */
403 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
404 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
405
406 /* We eventually do the icache sync here (maybe inline that
407 * code rather than call a C function...)
408 */
409BEGIN_FTR_SECTION
410 mr r4,r30
411 mr r5,r7
412 bl .hash_page_do_lazy_icache
413END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
414
415 /* At this point, r3 contains new PP bits, save them in
416 * place of "access" in the param area (sic)
417 */
418 std r3,STK_PARM(r4)(r1)
419
420 /* Get htab_hash_mask */
421 ld r4,htab_hash_mask@got(2)
422 ld r27,0(r4) /* htab_hash_mask -> r27 */
423
424 /* Check if we may already be in the hashtable, in this case, we
425 * go to out-of-line code to try to modify the HPTE. We look for
426 * the bit at (1 >> (index + 32))
427 */
428 andi. r0,r31,_PAGE_HASHPTE
429 li r26,0 /* Default hidx */
430 beq htab_insert_pte
431
432 /*
433 * Check if the pte was already inserted into the hash table
434 * as a 64k HW page, and invalidate the 64k HPTE if so.
435 */
436 andis. r0,r31,_PAGE_COMBO@h
437 beq htab_inval_old_hpte
438
439 ld r6,STK_PARM(r6)(r1)
440 ori r26,r6,0x8000 /* Load the hidx mask */
441 ld r26,0(r26)
442 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
443 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
444 bne htab_modify_pte
445
446htab_insert_pte:
447 /* real page number in r5, PTE RPN value + index */
448 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
449 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
450 add r5,r5,r25
451 sldi r5,r5,HW_PAGE_SHIFT
452
453 /* Calculate primary group hash */
454 and r0,r28,r27
455 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
456
457 /* Call ppc_md.hpte_insert */
458 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
459 mr r4,r29 /* Retreive va */
460 li r7,0 /* !bolted, !secondary */
461 li r8,MMU_PAGE_4K /* page size */
462_GLOBAL(htab_call_hpte_insert1)
463 bl . /* patched by htab_finish_init() */
464 cmpdi 0,r3,0
465 bge htab_pte_insert_ok /* Insertion successful */
466 cmpdi 0,r3,-2 /* Critical failure */
467 beq- htab_pte_insert_failure
468
469 /* Now try secondary slot */
470
471 /* real page number in r5, PTE RPN value + index */
472 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
473 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
474 add r5,r5,r25
475 sldi r5,r5,HW_PAGE_SHIFT
476
477 /* Calculate secondary group hash */
478 andc r0,r27,r28
479 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
480
481 /* Call ppc_md.hpte_insert */
482 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
483 mr r4,r29 /* Retreive va */
484 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
485 li r8,MMU_PAGE_4K /* page size */
486_GLOBAL(htab_call_hpte_insert2)
487 bl . /* patched by htab_finish_init() */
488 cmpdi 0,r3,0
489 bge+ htab_pte_insert_ok /* Insertion successful */
490 cmpdi 0,r3,-2 /* Critical failure */
491 beq- htab_pte_insert_failure
492
493 /* Both are full, we need to evict something */
494 mftb r0
495 /* Pick a random group based on TB */
496 andi. r0,r0,1
497 mr r5,r28
498 bne 2f
499 not r5,r5
5002: and r0,r5,r27
501 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
502 /* Call ppc_md.hpte_remove */
503_GLOBAL(htab_call_hpte_remove)
504 bl . /* patched by htab_finish_init() */
505
506 /* Try all again */
507 b htab_insert_pte
508
509 /*
510 * Call out to C code to invalidate an 64k HW HPTE that is
511 * useless now that the segment has been switched to 4k pages.
512 */
513htab_inval_old_hpte:
514 mr r3,r29 /* virtual addr */
515 mr r4,r31 /* PTE.pte */
516 li r5,0 /* PTE.hidx */
517 li r6,MMU_PAGE_64K /* psize */
518 ld r7,STK_PARM(r8)(r1) /* local */
519 bl .flush_hash_page
520 b htab_insert_pte
521
522htab_bail_ok:
523 li r3,0
524 b htab_bail
525
526htab_pte_insert_ok:
527 /* Insert slot number & secondary bit in PTE second half,
528 * clear _PAGE_BUSY and set approriate HPTE slot bit
529 */
530 ld r6,STK_PARM(r6)(r1)
531 li r0,_PAGE_BUSY
532 andc r30,r30,r0
533 /* HPTE SUB bit */
534 li r0,1
535 subfic r5,r25,27 /* Must match bit position in */
536 sld r0,r0,r5 /* pgtable.h */
537 or r30,r30,r0
538 /* hindx */
539 sldi r5,r25,2
540 sld r3,r3,r5
541 li r4,0xf
542 sld r4,r4,r5
543 andc r26,r26,r4
544 or r26,r26,r3
545 ori r5,r6,0x8000
546 std r26,0(r5)
547 lwsync
548 std r30,0(r6)
549 li r3, 0
550htab_bail:
551 ld r25,STK_REG(r25)(r1)
552 ld r26,STK_REG(r26)(r1)
553 ld r27,STK_REG(r27)(r1)
554 ld r28,STK_REG(r28)(r1)
555 ld r29,STK_REG(r29)(r1)
556 ld r30,STK_REG(r30)(r1)
557 ld r31,STK_REG(r31)(r1)
558 addi r1,r1,STACKFRAMESIZE
559 ld r0,16(r1)
560 mtlr r0
561 blr
562
563htab_modify_pte:
564 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
565 mr r4,r3
566 sldi r5,r25,2
567 srd r3,r26,r5
568
569 /* Secondary group ? if yes, get a inverted hash value */
570 mr r5,r28
571 andi. r0,r3,0x8 /* page secondary ? */
572 beq 1f
573 not r5,r5
5741: andi. r3,r3,0x7 /* extract idx alone */
575
576 /* Calculate proper slot value for ppc_md.hpte_updatepp */
577 and r0,r5,r27
578 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
579 add r3,r0,r3 /* add slot idx */
580
581 /* Call ppc_md.hpte_updatepp */
582 mr r5,r29 /* va */
583 li r6,MMU_PAGE_4K /* page size */
584 ld r7,STK_PARM(r8)(r1) /* get "local" param */
585_GLOBAL(htab_call_hpte_updatepp)
586 bl . /* patched by htab_finish_init() */
587
588 /* if we failed because typically the HPTE wasn't really here
589 * we try an insertion.
590 */
591 cmpdi 0,r3,-1
592 beq- htab_insert_pte
593
594 /* Clear the BUSY bit and Write out the PTE */
595 li r0,_PAGE_BUSY
596 andc r30,r30,r0
597 ld r6,STK_PARM(r6)(r1)
598 std r30,0(r6)
599 li r3,0
600 b htab_bail
601
602htab_wrong_access:
603 /* Bail out clearing reservation */
604 stdcx. r31,0,r6
605 li r3,1
606 b htab_bail
607
608htab_pte_insert_failure:
609 /* Bail out restoring old PTE */
610 ld r6,STK_PARM(r6)(r1)
611 std r31,0(r6)
612 li r3,-1
613 b htab_bail
614
615
616/*****************************************************************************
617 * *
618 * 64K SW & 64K HW in a 64K segment pages implementation *
619 * *
620 *****************************************************************************/
621
622_GLOBAL(__hash_page_64K)
623 mflr r0
624 std r0,16(r1)
625 stdu r1,-STACKFRAMESIZE(r1)
626 /* Save all params that we need after a function call */
627 std r6,STK_PARM(r6)(r1)
628 std r8,STK_PARM(r8)(r1)
629
630 /* Add _PAGE_PRESENT to access */
631 ori r4,r4,_PAGE_PRESENT
632
633 /* Save non-volatile registers.
634 * r31 will hold "old PTE"
635 * r30 is "new PTE"
636 * r29 is "va"
637 * r28 is a hash value
638 * r27 is hashtab mask (maybe dynamic patched instead ?)
639 */
640 std r27,STK_REG(r27)(r1)
641 std r28,STK_REG(r28)(r1)
642 std r29,STK_REG(r29)(r1)
643 std r30,STK_REG(r30)(r1)
644 std r31,STK_REG(r31)(r1)
645
646 /* Step 1:
647 *
648 * Check permissions, atomically mark the linux PTE busy
649 * and hashed.
650 */
6511:
652 ldarx r31,0,r6
653 /* Check access rights (access & ~(pte_val(*ptep))) */
654 andc. r0,r4,r31
655 bne- ht64_wrong_access
656 /* Check if PTE is busy */
657 andi. r0,r31,_PAGE_BUSY
658 /* If so, just bail out and refault if needed. Someone else
659 * is changing this PTE anyway and might hash it.
660 */
661 bne- ht64_bail_ok
662BEGIN_FTR_SECTION
663 /* Check if PTE has the cache-inhibit bit set */
664 andi. r0,r31,_PAGE_NO_CACHE
665 /* If so, bail out and refault as a 4k page */
666 bne- ht64_bail_ok
667END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
668 /* Prepare new PTE value (turn access RW into DIRTY, then
669 * add BUSY,HASHPTE and ACCESSED)
670 */
671 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
672 or r30,r30,r31
673 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
674 /* Write the linux PTE atomically (setting busy) */
675 stdcx. r30,0,r6
676 bne- 1b
677 isync
678
679 /* Step 2:
680 *
681 * Insert/Update the HPTE in the hash table. At this point,
682 * r4 (access) is re-useable, we use it for the new HPTE flags
683 */
684
685 /* Calc va and put it in r29 */
686 rldicr r29,r5,28,63-28
687 rldicl r3,r3,0,36
688 or r29,r3,r29
689
690 /* Calculate hash value for primary slot and store it in r28 */
691 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
692 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
693 xor r28,r5,r0
694
695 /* Convert linux PTE bits into HW equivalents */
696 andi. r3,r30,0x1fe /* Get basic set of flags */
697 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
698 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
699 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
700 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
701 andc r0,r30,r0 /* r0 = pte & ~r0 */
702 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
703 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
704
705 /* We eventually do the icache sync here (maybe inline that
706 * code rather than call a C function...)
707 */
708BEGIN_FTR_SECTION
709 mr r4,r30
710 mr r5,r7
711 bl .hash_page_do_lazy_icache
712END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
713
714 /* At this point, r3 contains new PP bits, save them in
715 * place of "access" in the param area (sic)
716 */
717 std r3,STK_PARM(r4)(r1)
718
719 /* Get htab_hash_mask */
720 ld r4,htab_hash_mask@got(2)
721 ld r27,0(r4) /* htab_hash_mask -> r27 */
722
723 /* Check if we may already be in the hashtable, in this case, we
724 * go to out-of-line code to try to modify the HPTE
725 */
726 andi. r0,r31,_PAGE_HASHPTE
727 bne ht64_modify_pte
728
729ht64_insert_pte:
730 /* Clear hpte bits in new pte (we also clear BUSY btw) and
731 * add _PAGE_HASHPTE
732 */
733 lis r0,_PAGE_HPTEFLAGS@h
734 ori r0,r0,_PAGE_HPTEFLAGS@l
735 andc r30,r30,r0
736 ori r30,r30,_PAGE_HASHPTE
737
738 /* Phyical address in r5 */
739 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
740 sldi r5,r5,PAGE_SHIFT
741
742 /* Calculate primary group hash */
743 and r0,r28,r27
744 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
745
746 /* Call ppc_md.hpte_insert */
747 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
748 mr r4,r29 /* Retreive va */
749 li r7,0 /* !bolted, !secondary */
750 li r8,MMU_PAGE_64K
751_GLOBAL(ht64_call_hpte_insert1)
752 bl . /* patched by htab_finish_init() */
753 cmpdi 0,r3,0
754 bge ht64_pte_insert_ok /* Insertion successful */
755 cmpdi 0,r3,-2 /* Critical failure */
756 beq- ht64_pte_insert_failure
757
758 /* Now try secondary slot */
759
760 /* Phyical address in r5 */
761 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
762 sldi r5,r5,PAGE_SHIFT
763
764 /* Calculate secondary group hash */
765 andc r0,r27,r28
766 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
767
768 /* Call ppc_md.hpte_insert */
769 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
770 mr r4,r29 /* Retreive va */
771 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
772 li r8,MMU_PAGE_64K
773_GLOBAL(ht64_call_hpte_insert2)
774 bl . /* patched by htab_finish_init() */
775 cmpdi 0,r3,0
776 bge+ ht64_pte_insert_ok /* Insertion successful */
777 cmpdi 0,r3,-2 /* Critical failure */
778 beq- ht64_pte_insert_failure
779
780 /* Both are full, we need to evict something */
781 mftb r0
782 /* Pick a random group based on TB */
783 andi. r0,r0,1
784 mr r5,r28
785 bne 2f
786 not r5,r5
7872: and r0,r5,r27
788 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
789 /* Call ppc_md.hpte_remove */
790_GLOBAL(ht64_call_hpte_remove)
791 bl . /* patched by htab_finish_init() */
792
793 /* Try all again */
794 b ht64_insert_pte
795
796ht64_bail_ok:
797 li r3,0
798 b ht64_bail
799
800ht64_pte_insert_ok:
801 /* Insert slot number & secondary bit in PTE */
802 rldimi r30,r3,12,63-15
803
804 /* Write out the PTE with a normal write
805 * (maybe add eieio may be good still ?)
806 */
807ht64_write_out_pte:
808 ld r6,STK_PARM(r6)(r1)
809 std r30,0(r6)
810 li r3, 0
811ht64_bail:
812 ld r27,STK_REG(r27)(r1)
813 ld r28,STK_REG(r28)(r1)
814 ld r29,STK_REG(r29)(r1)
815 ld r30,STK_REG(r30)(r1)
816 ld r31,STK_REG(r31)(r1)
817 addi r1,r1,STACKFRAMESIZE
818 ld r0,16(r1)
819 mtlr r0
820 blr
821
822ht64_modify_pte:
823 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
824 mr r4,r3
825 rlwinm r3,r31,32-12,29,31
826
827 /* Secondary group ? if yes, get a inverted hash value */
828 mr r5,r28
829 andi. r0,r31,_PAGE_F_SECOND
830 beq 1f
831 not r5,r5
8321:
833 /* Calculate proper slot value for ppc_md.hpte_updatepp */
834 and r0,r5,r27
835 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
836 add r3,r0,r3 /* add slot idx */
837
838 /* Call ppc_md.hpte_updatepp */
839 mr r5,r29 /* va */
840 li r6,MMU_PAGE_64K
841 ld r7,STK_PARM(r8)(r1) /* get "local" param */
842_GLOBAL(ht64_call_hpte_updatepp)
843 bl . /* patched by htab_finish_init() */
844
845 /* if we failed because typically the HPTE wasn't really here
846 * we try an insertion.
847 */
848 cmpdi 0,r3,-1
849 beq- ht64_insert_pte
850
851 /* Clear the BUSY bit and Write out the PTE */
852 li r0,_PAGE_BUSY
853 andc r30,r30,r0
854 b ht64_write_out_pte
855
856ht64_wrong_access:
857 /* Bail out clearing reservation */
858 stdcx. r31,0,r6
859 li r3,1
860 b ht64_bail
861
862ht64_pte_insert_failure:
863 /* Bail out restoring old PTE */
864 ld r6,STK_PARM(r6)(r1)
865 std r31,0(r6)
866 li r3,-1
867 b ht64_bail
868
869
870#endif /* CONFIG_PPC_64K_PAGES */
871
872
873/*****************************************************************************
874 * *
875 * Huge pages implementation is in hugetlbpage.c *
876 * *
877 *****************************************************************************/