arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37
  38 #ifndef CONFIG_PPC_64K_PAGES
  39
  40 /*****************************************************************************
  41  *                                                                           *
  42  *           4K SW & 4K HW pages implementation                              *
  43  *                                                                           *
  44  *****************************************************************************/
  45
  46
  47 /*
  48  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  49  *               pte_t *ptep, unsigned long trap, int local, int ssize)
  50  *
  51  * Adds a 4K page to the hash table in a segment of 4K pages only
  52  */
  53
  54 _GLOBAL(__hash_page_4K)
  55         mflr    r0
  56         std     r0,16(r1)
  57         stdu    r1,-STACKFRAMESIZE(r1)
  58         /* Save all params that we need after a function call */
  59         std     r6,STK_PARAM(R6)(r1)
  60         std     r8,STK_PARAM(R8)(r1)
  61         std     r9,STK_PARAM(R9)(r1)
  62
  63         /* Save non-volatile registers.
  64          * r31 will hold "old PTE"
  65          * r30 is "new PTE"
  66          * r29 is "va"
  67          * r28 is a hash value
  68          * r27 is hashtab mask (maybe dynamic patched instead ?)
  69          */
  70         std     r27,STK_REG(R27)(r1)
  71         std     r28,STK_REG(R28)(r1)
  72         std     r29,STK_REG(R29)(r1)
  73         std     r30,STK_REG(R30)(r1)
  74         std     r31,STK_REG(R31)(r1)
  75
  76         /* Step 1:
  77          *
  78          * Check permissions, atomically mark the linux PTE busy
  79          * and hashed.
  80          */
  81 1:
  82         ldarx   r31,0,r6
  83         /* Check access rights (access & ~(pte_val(*ptep))) */
  84         andc.   r0,r4,r31
  85         bne-    htab_wrong_access
  86         /* Check if PTE is busy */
  87         andi.   r0,r31,_PAGE_BUSY
  88         /* If so, just bail out and refault if needed. Someone else
  89          * is changing this PTE anyway and might hash it.
  90          */
  91         bne-    htab_bail_ok
  92
  93         /* Prepare new PTE value (turn access RW into DIRTY, then
  94          * add BUSY,HASHPTE and ACCESSED)
  95          */
  96         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  97         or      r30,r30,r31
  98         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
  99         /* Write the linux PTE atomically (setting busy) */
 100         stdcx.  r30,0,r6
 101         bne-    1b
 102         isync
 103
 104         /* Step 2:
 105          *
 106          * Insert/Update the HPTE in the hash table. At this point,
 107          * r4 (access) is re-useable, we use it for the new HPTE flags
 108          */
 109
 110 BEGIN_FTR_SECTION
 111         cmpdi   r9,0                    /* check segment size */
 112         bne     3f
 113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 114         /* Calc va and put it in r29 */
 115         rldicr  r29,r5,28,63-28
 116         rldicl  r3,r3,0,36
 117         or      r29,r3,r29
 118
 119         /* Calculate hash value for primary slot and store it in r28 */
 120         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 121         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 122         xor     r28,r5,r0
 123         b       4f
 124
 125 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 126         sldi    r29,r5,40               /* vsid << 40 */
 127         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 128         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 129         clrldi  r5,r5,40                /* vsid & 0xffffff */
 130         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 131         xor     r28,r28,r5
 132         or      r29,r3,r29              /* VA */
 133         xor     r28,r28,r0              /* hash */
 134
 135         /* Convert linux PTE bits into HW equivalents */
 136 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 137         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 138         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 139         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 140         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 141         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 142         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 143         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 144
 145         /* We eventually do the icache sync here (maybe inline that
 146          * code rather than call a C function...)
 147          */
 148 BEGIN_FTR_SECTION
 149         mr      r4,r30
 150         mr      r5,r7
 151         bl      .hash_page_do_lazy_icache
 152 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 153
 154         /* At this point, r3 contains new PP bits, save them in
 155          * place of "access" in the param area (sic)
 156          */
 157         std     r3,STK_PARAM(R4)(r1)
 158
 159         /* Get htab_hash_mask */
 160         ld      r4,htab_hash_mask@got(2)
 161         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 162
 163         /* Check if we may already be in the hashtable, in this case, we
 164          * go to out-of-line code to try to modify the HPTE
 165          */
 166         andi.   r0,r31,_PAGE_HASHPTE
 167         bne     htab_modify_pte
 168
 169 htab_insert_pte:
 170         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 171          * add _PAGE_HASHPTE
 172          */
 173         lis     r0,_PAGE_HPTEFLAGS@h
 174         ori     r0,r0,_PAGE_HPTEFLAGS@l
 175         andc    r30,r30,r0
 176         ori     r30,r30,_PAGE_HASHPTE
 177
 178         /* physical address r5 */
 179         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 180         sldi    r5,r5,PAGE_SHIFT
 181
 182         /* Calculate primary group hash */
 183         and     r0,r28,r27
 184         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 185
 186         /* Call ppc_md.hpte_insert */
 187         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 188         mr      r4,r29                  /* Retrieve va */
 189         li      r7,0                    /* !bolted, !secondary */
 190         li      r8,MMU_PAGE_4K          /* page size */
 191         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 192 _GLOBAL(htab_call_hpte_insert1)
 193         bl      .                       /* Patched by htab_finish_init() */
 194         cmpdi   0,r3,0
 195         bge     htab_pte_insert_ok      /* Insertion successful */
 196         cmpdi   0,r3,-2                 /* Critical failure */
 197         beq-    htab_pte_insert_failure
 198
 199         /* Now try secondary slot */
 200
 201         /* physical address r5 */
 202         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 203         sldi    r5,r5,PAGE_SHIFT
 204
 205         /* Calculate secondary group hash */
 206         andc    r0,r27,r28
 207         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 208
 209         /* Call ppc_md.hpte_insert */
 210         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 211         mr      r4,r29                  /* Retrieve va */
 212         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 213         li      r8,MMU_PAGE_4K          /* page size */
 214         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 215 _GLOBAL(htab_call_hpte_insert2)
 216         bl      .                       /* Patched by htab_finish_init() */
 217         cmpdi   0,r3,0
 218         bge+    htab_pte_insert_ok      /* Insertion successful */
 219         cmpdi   0,r3,-2                 /* Critical failure */
 220         beq-    htab_pte_insert_failure
 221
 222         /* Both are full, we need to evict something */
 223         mftb    r0
 224         /* Pick a random group based on TB */
 225         andi.   r0,r0,1
 226         mr      r5,r28
 227         bne     2f
 228         not     r5,r5
 229 2:      and     r0,r5,r27
 230         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 231         /* Call ppc_md.hpte_remove */
 232 _GLOBAL(htab_call_hpte_remove)
 233         bl      .                       /* Patched by htab_finish_init() */
 234
 235         /* Try all again */
 236         b       htab_insert_pte
 237
 238 htab_bail_ok:
 239         li      r3,0
 240         b       htab_bail
 241
 242 htab_pte_insert_ok:
 243         /* Insert slot number & secondary bit in PTE */
 244         rldimi  r30,r3,12,63-15
 245
 246         /* Write out the PTE with a normal write
 247          * (maybe add eieio may be good still ?)
 248          */
 249 htab_write_out_pte:
 250         ld      r6,STK_PARAM(R6)(r1)
 251         std     r30,0(r6)
 252         li      r3, 0
 253 htab_bail:
 254         ld      r27,STK_REG(R27)(r1)
 255         ld      r28,STK_REG(R28)(r1)
 256         ld      r29,STK_REG(R29)(r1)
 257         ld      r30,STK_REG(R30)(r1)
 258         ld      r31,STK_REG(R31)(r1)
 259         addi    r1,r1,STACKFRAMESIZE
 260         ld      r0,16(r1)
 261         mtlr    r0
 262         blr
 263
 264 htab_modify_pte:
 265         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 266         mr      r4,r3
 267         rlwinm  r3,r31,32-12,29,31
 268
 269         /* Secondary group ? if yes, get a inverted hash value */
 270         mr      r5,r28
 271         andi.   r0,r31,_PAGE_SECONDARY
 272         beq     1f
 273         not     r5,r5
 274 1:
 275         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 276         and     r0,r5,r27
 277         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 278         add     r3,r0,r3        /* add slot idx */
 279
 280         /* Call ppc_md.hpte_updatepp */
 281         mr      r5,r29                  /* va */
 282         li      r6,MMU_PAGE_4K          /* page size */
 283         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 284         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 285 _GLOBAL(htab_call_hpte_updatepp)
 286         bl      .                       /* Patched by htab_finish_init() */
 287
 288         /* if we failed because typically the HPTE wasn't really here
 289          * we try an insertion.
 290          */
 291         cmpdi   0,r3,-1
 292         beq-    htab_insert_pte
 293
 294         /* Clear the BUSY bit and Write out the PTE */
 295         li      r0,_PAGE_BUSY
 296         andc    r30,r30,r0
 297         b       htab_write_out_pte
 298
 299 htab_wrong_access:
 300         /* Bail out clearing reservation */
 301         stdcx.  r31,0,r6
 302         li      r3,1
 303         b       htab_bail
 304
 305 htab_pte_insert_failure:
 306         /* Bail out restoring old PTE */
 307         ld      r6,STK_PARAM(R6)(r1)
 308         std     r31,0(r6)
 309         li      r3,-1
 310         b       htab_bail
 311
 312
 313 #else /* CONFIG_PPC_64K_PAGES */
 314
 315
 316 /*****************************************************************************
 317  *                                                                           *
 318  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 319  *                                                                           *
 320  *****************************************************************************/
 321
 322 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 323  *               pte_t *ptep, unsigned long trap, int local, int ssize,
 324  *               int subpg_prot)
 325  */
 326
 327 /*
 328  * For now, we do NOT implement Admixed pages
 329  */
 330 _GLOBAL(__hash_page_4K)
 331         mflr    r0
 332         std     r0,16(r1)
 333         stdu    r1,-STACKFRAMESIZE(r1)
 334         /* Save all params that we need after a function call */
 335         std     r6,STK_PARAM(R6)(r1)
 336         std     r8,STK_PARAM(R8)(r1)
 337         std     r9,STK_PARAM(R9)(r1)
 338
 339         /* Save non-volatile registers.
 340          * r31 will hold "old PTE"
 341          * r30 is "new PTE"
 342          * r29 is "va"
 343          * r28 is a hash value
 344          * r27 is hashtab mask (maybe dynamic patched instead ?)
 345          * r26 is the hidx mask
 346          * r25 is the index in combo page
 347          */
 348         std     r25,STK_REG(R25)(r1)
 349         std     r26,STK_REG(R26)(r1)
 350         std     r27,STK_REG(R27)(r1)
 351         std     r28,STK_REG(R28)(r1)
 352         std     r29,STK_REG(R29)(r1)
 353         std     r30,STK_REG(R30)(r1)
 354         std     r31,STK_REG(R31)(r1)
 355
 356         /* Step 1:
 357          *
 358          * Check permissions, atomically mark the linux PTE busy
 359          * and hashed.
 360          */
 361 1:
 362         ldarx   r31,0,r6
 363         /* Check access rights (access & ~(pte_val(*ptep))) */
 364         andc.   r0,r4,r31
 365         bne-    htab_wrong_access
 366         /* Check if PTE is busy */
 367         andi.   r0,r31,_PAGE_BUSY
 368         /* If so, just bail out and refault if needed. Someone else
 369          * is changing this PTE anyway and might hash it.
 370          */
 371         bne-    htab_bail_ok
 372         /* Prepare new PTE value (turn access RW into DIRTY, then
 373          * add BUSY and ACCESSED)
 374          */
 375         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 376         or      r30,r30,r31
 377         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 378         oris    r30,r30,_PAGE_COMBO@h
 379         /* Write the linux PTE atomically (setting busy) */
 380         stdcx.  r30,0,r6
 381         bne-    1b
 382         isync
 383
 384         /* Step 2:
 385          *
 386          * Insert/Update the HPTE in the hash table. At this point,
 387          * r4 (access) is re-useable, we use it for the new HPTE flags
 388          */
 389
 390         /* Load the hidx index */
 391         rldicl  r25,r3,64-12,60
 392
 393 BEGIN_FTR_SECTION
 394         cmpdi   r9,0                    /* check segment size */
 395         bne     3f
 396 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 397         /* Calc va and put it in r29 */
 398         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
 399         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
 400         or      r29,r3,r29              /* r29 = va */
 401
 402         /* Calculate hash value for primary slot and store it in r28 */
 403         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 404         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 405         xor     r28,r5,r0
 406         b       4f
 407
 408 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 409         sldi    r29,r5,40               /* vsid << 40 */
 410         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 411         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 412         clrldi  r5,r5,40                /* vsid & 0xffffff */
 413         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 414         xor     r28,r28,r5
 415         or      r29,r3,r29              /* VA */
 416         xor     r28,r28,r0              /* hash */
 417
 418         /* Convert linux PTE bits into HW equivalents */
 419 4:
 420 #ifdef CONFIG_PPC_SUBPAGE_PROT
 421         andc    r10,r30,r10
 422         andi.   r3,r10,0x1fe            /* Get basic set of flags */
 423         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 424 #else
 425         andi.   r3,r30,0x1fe            /* Get basic set of flags */
 426         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 427 #endif
 428         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 429         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 430         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 431         andc    r0,r3,r0                /* r0 = pte & ~r0 */
 432         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 433         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 434
 435         /* We eventually do the icache sync here (maybe inline that
 436          * code rather than call a C function...)
 437          */
 438 BEGIN_FTR_SECTION
 439         mr      r4,r30
 440         mr      r5,r7
 441         bl      .hash_page_do_lazy_icache
 442 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 443
 444         /* At this point, r3 contains new PP bits, save them in
 445          * place of "access" in the param area (sic)
 446          */
 447         std     r3,STK_PARAM(R4)(r1)
 448
 449         /* Get htab_hash_mask */
 450         ld      r4,htab_hash_mask@got(2)
 451         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 452
 453         /* Check if we may already be in the hashtable, in this case, we
 454          * go to out-of-line code to try to modify the HPTE. We look for
 455          * the bit at (1 >> (index + 32))
 456          */
 457         rldicl. r0,r31,64-12,48
 458         li      r26,0                   /* Default hidx */
 459         beq     htab_insert_pte
 460
 461         /*
 462          * Check if the pte was already inserted into the hash table
 463          * as a 64k HW page, and invalidate the 64k HPTE if so.
 464          */
 465         andis.  r0,r31,_PAGE_COMBO@h
 466         beq     htab_inval_old_hpte
 467
 468         ld      r6,STK_PARAM(R6)(r1)
 469         ori     r26,r6,0x8000           /* Load the hidx mask */
 470         ld      r26,0(r26)
 471         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 472         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 473         bne     htab_modify_pte
 474
 475 htab_insert_pte:
 476         /* real page number in r5, PTE RPN value + index */
 477         andis.  r0,r31,_PAGE_4K_PFN@h
 478         srdi    r5,r31,PTE_RPN_SHIFT
 479         bne-    htab_special_pfn
 480         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 481         add     r5,r5,r25
 482 htab_special_pfn:
 483         sldi    r5,r5,HW_PAGE_SHIFT
 484
 485         /* Calculate primary group hash */
 486         and     r0,r28,r27
 487         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 488
 489         /* Call ppc_md.hpte_insert */
 490         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 491         mr      r4,r29                  /* Retrieve va */
 492         li      r7,0                    /* !bolted, !secondary */
 493         li      r8,MMU_PAGE_4K          /* page size */
 494         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 495 _GLOBAL(htab_call_hpte_insert1)
 496         bl      .                       /* patched by htab_finish_init() */
 497         cmpdi   0,r3,0
 498         bge     htab_pte_insert_ok      /* Insertion successful */
 499         cmpdi   0,r3,-2                 /* Critical failure */
 500         beq-    htab_pte_insert_failure
 501
 502         /* Now try secondary slot */
 503
 504         /* real page number in r5, PTE RPN value + index */
 505         andis.  r0,r31,_PAGE_4K_PFN@h
 506         srdi    r5,r31,PTE_RPN_SHIFT
 507         bne-    3f
 508         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 509         add     r5,r5,r25
 510 3:      sldi    r5,r5,HW_PAGE_SHIFT
 511
 512         /* Calculate secondary group hash */
 513         andc    r0,r27,r28
 514         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 515
 516         /* Call ppc_md.hpte_insert */
 517         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 518         mr      r4,r29                  /* Retrieve va */
 519         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 520         li      r8,MMU_PAGE_4K          /* page size */
 521         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 522 _GLOBAL(htab_call_hpte_insert2)
 523         bl      .                       /* patched by htab_finish_init() */
 524         cmpdi   0,r3,0
 525         bge+    htab_pte_insert_ok      /* Insertion successful */
 526         cmpdi   0,r3,-2                 /* Critical failure */
 527         beq-    htab_pte_insert_failure
 528
 529         /* Both are full, we need to evict something */
 530         mftb    r0
 531         /* Pick a random group based on TB */
 532         andi.   r0,r0,1
 533         mr      r5,r28
 534         bne     2f
 535         not     r5,r5
 536 2:      and     r0,r5,r27
 537         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 538         /* Call ppc_md.hpte_remove */
 539 _GLOBAL(htab_call_hpte_remove)
 540         bl      .                       /* patched by htab_finish_init() */
 541
 542         /* Try all again */
 543         b       htab_insert_pte
 544
 545         /*
 546          * Call out to C code to invalidate an 64k HW HPTE that is
 547          * useless now that the segment has been switched to 4k pages.
 548          */
 549 htab_inval_old_hpte:
 550         mr      r3,r29                  /* virtual addr */
 551         mr      r4,r31                  /* PTE.pte */
 552         li      r5,0                    /* PTE.hidx */
 553         li      r6,MMU_PAGE_64K         /* psize */
 554         ld      r7,STK_PARAM(R9)(r1)    /* ssize */
 555         ld      r8,STK_PARAM(R8)(r1)    /* local */
 556         bl      .flush_hash_page
 557         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 558         lis     r0,_PAGE_HPTE_SUB@h
 559         ori     r0,r0,_PAGE_HPTE_SUB@l
 560         andc    r30,r30,r0
 561         b       htab_insert_pte
 562
 563 htab_bail_ok:
 564         li      r3,0
 565         b       htab_bail
 566
 567 htab_pte_insert_ok:
 568         /* Insert slot number & secondary bit in PTE second half,
 569          * clear _PAGE_BUSY and set approriate HPTE slot bit
 570          */
 571         ld      r6,STK_PARAM(R6)(r1)
 572         li      r0,_PAGE_BUSY
 573         andc    r30,r30,r0
 574         /* HPTE SUB bit */
 575         li      r0,1
 576         subfic  r5,r25,27               /* Must match bit position in */
 577         sld     r0,r0,r5                /* pgtable.h */
 578         or      r30,r30,r0
 579         /* hindx */
 580         sldi    r5,r25,2
 581         sld     r3,r3,r5
 582         li      r4,0xf
 583         sld     r4,r4,r5
 584         andc    r26,r26,r4
 585         or      r26,r26,r3
 586         ori     r5,r6,0x8000
 587         std     r26,0(r5)
 588         lwsync
 589         std     r30,0(r6)
 590         li      r3, 0
 591 htab_bail:
 592         ld      r25,STK_REG(R25)(r1)
 593         ld      r26,STK_REG(R26)(r1)
 594         ld      r27,STK_REG(R27)(r1)
 595         ld      r28,STK_REG(R28)(r1)
 596         ld      r29,STK_REG(R29)(r1)
 597         ld      r30,STK_REG(R30)(r1)
 598         ld      r31,STK_REG(R31)(r1)
 599         addi    r1,r1,STACKFRAMESIZE
 600         ld      r0,16(r1)
 601         mtlr    r0
 602         blr
 603
 604 htab_modify_pte:
 605         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 606         mr      r4,r3
 607         sldi    r5,r25,2
 608         srd     r3,r26,r5
 609
 610         /* Secondary group ? if yes, get a inverted hash value */
 611         mr      r5,r28
 612         andi.   r0,r3,0x8 /* page secondary ? */
 613         beq     1f
 614         not     r5,r5
 615 1:      andi.   r3,r3,0x7 /* extract idx alone */
 616
 617         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 618         and     r0,r5,r27
 619         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 620         add     r3,r0,r3        /* add slot idx */
 621
 622         /* Call ppc_md.hpte_updatepp */
 623         mr      r5,r29                  /* va */
 624         li      r6,MMU_PAGE_4K          /* page size */
 625         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 626         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 627 _GLOBAL(htab_call_hpte_updatepp)
 628         bl      .                       /* patched by htab_finish_init() */
 629
 630         /* if we failed because typically the HPTE wasn't really here
 631          * we try an insertion.
 632          */
 633         cmpdi   0,r3,-1
 634         beq-    htab_insert_pte
 635
 636         /* Clear the BUSY bit and Write out the PTE */
 637         li      r0,_PAGE_BUSY
 638         andc    r30,r30,r0
 639         ld      r6,STK_PARAM(R6)(r1)
 640         std     r30,0(r6)
 641         li      r3,0
 642         b       htab_bail
 643
 644 htab_wrong_access:
 645         /* Bail out clearing reservation */
 646         stdcx.  r31,0,r6
 647         li      r3,1
 648         b       htab_bail
 649
 650 htab_pte_insert_failure:
 651         /* Bail out restoring old PTE */
 652         ld      r6,STK_PARAM(R6)(r1)
 653         std     r31,0(r6)
 654         li      r3,-1
 655         b       htab_bail
 656
 657 #endif /* CONFIG_PPC_64K_PAGES */
 658
 659 #ifdef CONFIG_PPC_HAS_HASH_64K
 660
 661 /*****************************************************************************
 662  *                                                                           *
 663  *           64K SW & 64K HW in a 64K segment pages implementation           *
 664  *                                                                           *
 665  *****************************************************************************/
 666
 667 _GLOBAL(__hash_page_64K)
 668         mflr    r0
 669         std     r0,16(r1)
 670         stdu    r1,-STACKFRAMESIZE(r1)
 671         /* Save all params that we need after a function call */
 672         std     r6,STK_PARAM(R6)(r1)
 673         std     r8,STK_PARAM(R8)(r1)
 674         std     r9,STK_PARAM(R9)(r1)
 675
 676         /* Save non-volatile registers.
 677          * r31 will hold "old PTE"
 678          * r30 is "new PTE"
 679          * r29 is "va"
 680          * r28 is a hash value
 681          * r27 is hashtab mask (maybe dynamic patched instead ?)
 682          */
 683         std     r27,STK_REG(R27)(r1)
 684         std     r28,STK_REG(R28)(r1)
 685         std     r29,STK_REG(R29)(r1)
 686         std     r30,STK_REG(R30)(r1)
 687         std     r31,STK_REG(R31)(r1)
 688
 689         /* Step 1:
 690          *
 691          * Check permissions, atomically mark the linux PTE busy
 692          * and hashed.
 693          */
 694 1:
 695         ldarx   r31,0,r6
 696         /* Check access rights (access & ~(pte_val(*ptep))) */
 697         andc.   r0,r4,r31
 698         bne-    ht64_wrong_access
 699         /* Check if PTE is busy */
 700         andi.   r0,r31,_PAGE_BUSY
 701         /* If so, just bail out and refault if needed. Someone else
 702          * is changing this PTE anyway and might hash it.
 703          */
 704         bne-    ht64_bail_ok
 705 BEGIN_FTR_SECTION
 706         /* Check if PTE has the cache-inhibit bit set */
 707         andi.   r0,r31,_PAGE_NO_CACHE
 708         /* If so, bail out and refault as a 4k page */
 709         bne-    ht64_bail_ok
 710 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
 711         /* Prepare new PTE value (turn access RW into DIRTY, then
 712          * add BUSY and ACCESSED)
 713          */
 714         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 715         or      r30,r30,r31
 716         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 717         /* Write the linux PTE atomically (setting busy) */
 718         stdcx.  r30,0,r6
 719         bne-    1b
 720         isync
 721
 722         /* Step 2:
 723          *
 724          * Insert/Update the HPTE in the hash table. At this point,
 725          * r4 (access) is re-useable, we use it for the new HPTE flags
 726          */
 727
 728 BEGIN_FTR_SECTION
 729         cmpdi   r9,0                    /* check segment size */
 730         bne     3f
 731 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 732         /* Calc va and put it in r29 */
 733         rldicr  r29,r5,28,63-28
 734         rldicl  r3,r3,0,36
 735         or      r29,r3,r29
 736
 737         /* Calculate hash value for primary slot and store it in r28 */
 738         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 739         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
 740         xor     r28,r5,r0
 741         b       4f
 742
 743 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
 744         sldi    r29,r5,40               /* vsid << 40 */
 745         clrldi  r3,r3,24                /* ea & 0xffffffffff */
 746         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 747         clrldi  r5,r5,40                /* vsid & 0xffffff */
 748         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
 749         xor     r28,r28,r5
 750         or      r29,r3,r29              /* VA */
 751         xor     r28,r28,r0              /* hash */
 752
 753         /* Convert linux PTE bits into HW equivalents */
 754 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 755         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 756         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 757         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 758         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 759         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 760         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 761         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 762
 763         /* We eventually do the icache sync here (maybe inline that
 764          * code rather than call a C function...)
 765          */
 766 BEGIN_FTR_SECTION
 767         mr      r4,r30
 768         mr      r5,r7
 769         bl      .hash_page_do_lazy_icache
 770 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 771
 772         /* At this point, r3 contains new PP bits, save them in
 773          * place of "access" in the param area (sic)
 774          */
 775         std     r3,STK_PARAM(R4)(r1)
 776
 777         /* Get htab_hash_mask */
 778         ld      r4,htab_hash_mask@got(2)
 779         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 780
 781         /* Check if we may already be in the hashtable, in this case, we
 782          * go to out-of-line code to try to modify the HPTE
 783          */
 784         rldicl. r0,r31,64-12,48
 785         bne     ht64_modify_pte
 786
 787 ht64_insert_pte:
 788         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 789          * add _PAGE_HPTE_SUB0
 790          */
 791         lis     r0,_PAGE_HPTEFLAGS@h
 792         ori     r0,r0,_PAGE_HPTEFLAGS@l
 793         andc    r30,r30,r0
 794 #ifdef CONFIG_PPC_64K_PAGES
 795         oris    r30,r30,_PAGE_HPTE_SUB0@h
 796 #else
 797         ori     r30,r30,_PAGE_HASHPTE
 798 #endif
 799         /* Phyical address in r5 */
 800         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 801         sldi    r5,r5,PAGE_SHIFT
 802
 803         /* Calculate primary group hash */
 804         and     r0,r28,r27
 805         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 806
 807         /* Call ppc_md.hpte_insert */
 808         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 809         mr      r4,r29                  /* Retrieve va */
 810         li      r7,0                    /* !bolted, !secondary */
 811         li      r8,MMU_PAGE_64K
 812         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 813 _GLOBAL(ht64_call_hpte_insert1)
 814         bl      .                       /* patched by htab_finish_init() */
 815         cmpdi   0,r3,0
 816         bge     ht64_pte_insert_ok      /* Insertion successful */
 817         cmpdi   0,r3,-2                 /* Critical failure */
 818         beq-    ht64_pte_insert_failure
 819
 820         /* Now try secondary slot */
 821
 822         /* Phyical address in r5 */
 823         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 824         sldi    r5,r5,PAGE_SHIFT
 825
 826         /* Calculate secondary group hash */
 827         andc    r0,r27,r28
 828         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 829
 830         /* Call ppc_md.hpte_insert */
 831         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 832         mr      r4,r29                  /* Retrieve va */
 833         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 834         li      r8,MMU_PAGE_64K
 835         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 836 _GLOBAL(ht64_call_hpte_insert2)
 837         bl      .                       /* patched by htab_finish_init() */
 838         cmpdi   0,r3,0
 839         bge+    ht64_pte_insert_ok      /* Insertion successful */
 840         cmpdi   0,r3,-2                 /* Critical failure */
 841         beq-    ht64_pte_insert_failure
 842
 843         /* Both are full, we need to evict something */
 844         mftb    r0
 845         /* Pick a random group based on TB */
 846         andi.   r0,r0,1
 847         mr      r5,r28
 848         bne     2f
 849         not     r5,r5
 850 2:      and     r0,r5,r27
 851         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 852         /* Call ppc_md.hpte_remove */
 853 _GLOBAL(ht64_call_hpte_remove)
 854         bl      .                       /* patched by htab_finish_init() */
 855
 856         /* Try all again */
 857         b       ht64_insert_pte
 858
 859 ht64_bail_ok:
 860         li      r3,0
 861         b       ht64_bail
 862
 863 ht64_pte_insert_ok:
 864         /* Insert slot number & secondary bit in PTE */
 865         rldimi  r30,r3,12,63-15
 866
 867         /* Write out the PTE with a normal write
 868          * (maybe add eieio may be good still ?)
 869          */
 870 ht64_write_out_pte:
 871         ld      r6,STK_PARAM(R6)(r1)
 872         std     r30,0(r6)
 873         li      r3, 0
 874 ht64_bail:
 875         ld      r27,STK_REG(R27)(r1)
 876         ld      r28,STK_REG(R28)(r1)
 877         ld      r29,STK_REG(R29)(r1)
 878         ld      r30,STK_REG(R30)(r1)
 879         ld      r31,STK_REG(R31)(r1)
 880         addi    r1,r1,STACKFRAMESIZE
 881         ld      r0,16(r1)
 882         mtlr    r0
 883         blr
 884
 885 ht64_modify_pte:
 886         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 887         mr      r4,r3
 888         rlwinm  r3,r31,32-12,29,31
 889
 890         /* Secondary group ? if yes, get a inverted hash value */
 891         mr      r5,r28
 892         andi.   r0,r31,_PAGE_F_SECOND
 893         beq     1f
 894         not     r5,r5
 895 1:
 896         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 897         and     r0,r5,r27
 898         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 899         add     r3,r0,r3        /* add slot idx */
 900
 901         /* Call ppc_md.hpte_updatepp */
 902         mr      r5,r29                  /* va */
 903         li      r6,MMU_PAGE_64K
 904         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 905         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 906 _GLOBAL(ht64_call_hpte_updatepp)
 907         bl      .                       /* patched by htab_finish_init() */
 908
 909         /* if we failed because typically the HPTE wasn't really here
 910          * we try an insertion.
 911          */
 912         cmpdi   0,r3,-1
 913         beq-    ht64_insert_pte
 914
 915         /* Clear the BUSY bit and Write out the PTE */
 916         li      r0,_PAGE_BUSY
 917         andc    r30,r30,r0
 918         b       ht64_write_out_pte
 919
 920 ht64_wrong_access:
 921         /* Bail out clearing reservation */
 922         stdcx.  r31,0,r6
 923         li      r3,1
 924         b       ht64_bail
 925
 926 ht64_pte_insert_failure:
 927         /* Bail out restoring old PTE */
 928         ld      r6,STK_PARAM(R6)(r1)
 929         std     r31,0(r6)
 930         li      r3,-1
 931         b       ht64_bail
 932
 933
 934 #endif /* CONFIG_PPC_HAS_HASH_64K */
 935
 936
 937 /*****************************************************************************
 938  *                                                                           *
 939  *           Huge pages implementation is in hugetlbpage.c                   *
 940  *                                                                           *
 941  *****************************************************************************/