arch/mips/mm/tlb-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
   8  * Carsten Langgaard, carstenl@mips.com
   9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
  10  */
  11 #include <linux/cpu_pm.h>
  12 #include <linux/init.h>
  13 #include <linux/sched.h>
  14 #include <linux/smp.h>
  15 #include <linux/mm.h>
  16 #include <linux/hugetlb.h>
  17 #include <linux/module.h>
  18
  19 #include <asm/cpu.h>
  20 #include <asm/cpu-type.h>
  21 #include <asm/bootinfo.h>
  22 #include <asm/mmu_context.h>
  23 #include <asm/pgtable.h>
  24 #include <asm/tlb.h>
  25 #include <asm/tlbmisc.h>
  26
  27 extern void build_tlb_refill_handler(void);
  28
  29 /*
  30  * LOONGSON2/3 has a 4 entry itlb which is a subset of dtlb,
  31  * unfortunately, itlb is not totally transparent to software.
  32  */
  33 static inline void flush_itlb(void)
  34 {
  35         switch (current_cpu_type()) {
  36         case CPU_LOONGSON2:
  37         case CPU_LOONGSON3:
  38                 write_c0_diag(4);
  39                 break;
  40         default:
  41                 break;
  42         }
  43 }
  44
  45 static inline void flush_itlb_vm(struct vm_area_struct *vma)
  46 {
  47         if (vma->vm_flags & VM_EXEC)
  48                 flush_itlb();
  49 }
  50
  51 void local_flush_tlb_all(void)
  52 {
  53         unsigned long flags;
  54         unsigned long old_ctx;
  55         int entry, ftlbhighset;
  56
  57         local_irq_save(flags);
  58         /* Save old context and create impossible VPN2 value */
  59         old_ctx = read_c0_entryhi();
  60         write_c0_entrylo0(0);
  61         write_c0_entrylo1(0);
  62
  63         entry = read_c0_wired();
  64
  65         /* Blast 'em all away. */
  66         if (cpu_has_tlbinv) {
  67                 if (current_cpu_data.tlbsizevtlb) {
  68                         write_c0_index(0);
  69                         mtc0_tlbw_hazard();
  70                         tlbinvf();  /* invalidate VTLB */
  71                 }
  72                 ftlbhighset = current_cpu_data.tlbsizevtlb +
  73                         current_cpu_data.tlbsizeftlbsets;
  74                 for (entry = current_cpu_data.tlbsizevtlb;
  75                      entry < ftlbhighset;
  76                      entry++) {
  77                         write_c0_index(entry);
  78                         mtc0_tlbw_hazard();
  79                         tlbinvf();  /* invalidate one FTLB set */
  80                 }
  81         } else {
  82                 while (entry < current_cpu_data.tlbsize) {
  83                         /* Make sure all entries differ. */
  84                         write_c0_entryhi(UNIQUE_ENTRYHI(entry));
  85                         write_c0_index(entry);
  86                         mtc0_tlbw_hazard();
  87                         tlb_write_indexed();
  88                         entry++;
  89                 }
  90         }
  91         tlbw_use_hazard();
  92         write_c0_entryhi(old_ctx);
  93         flush_itlb();
  94         local_irq_restore(flags);
  95 }
  96 EXPORT_SYMBOL(local_flush_tlb_all);
  97
  98 /* All entries common to a mm share an asid.  To effectively flush
  99    these entries, we just bump the asid. */
 100 void local_flush_tlb_mm(struct mm_struct *mm)
 101 {
 102         int cpu;
 103
 104         preempt_disable();
 105
 106         cpu = smp_processor_id();
 107
 108         if (cpu_context(cpu, mm) != 0) {
 109                 drop_mmu_context(mm, cpu);
 110         }
 111
 112         preempt_enable();
 113 }
 114
 115 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 116         unsigned long end)
 117 {
 118         struct mm_struct *mm = vma->vm_mm;
 119         int cpu = smp_processor_id();
 120
 121         if (cpu_context(cpu, mm) != 0) {
 122                 unsigned long size, flags;
 123
 124                 local_irq_save(flags);
 125                 start = round_down(start, PAGE_SIZE << 1);
 126                 end = round_up(end, PAGE_SIZE << 1);
 127                 size = (end - start) >> (PAGE_SHIFT + 1);
 128                 if (size <= (current_cpu_data.tlbsizeftlbsets ?
 129                              current_cpu_data.tlbsize / 8 :
 130                              current_cpu_data.tlbsize / 2)) {
 131                         int oldpid = read_c0_entryhi();
 132                         int newpid = cpu_asid(cpu, mm);
 133
 134                         while (start < end) {
 135                                 int idx;
 136
 137                                 write_c0_entryhi(start | newpid);
 138                                 start += (PAGE_SIZE << 1);
 139                                 mtc0_tlbw_hazard();
 140                                 tlb_probe();
 141                                 tlb_probe_hazard();
 142                                 idx = read_c0_index();
 143                                 write_c0_entrylo0(0);
 144                                 write_c0_entrylo1(0);
 145                                 if (idx < 0)
 146                                         continue;
 147                                 /* Make sure all entries differ. */
 148                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 149                                 mtc0_tlbw_hazard();
 150                                 tlb_write_indexed();
 151                         }
 152                         tlbw_use_hazard();
 153                         write_c0_entryhi(oldpid);
 154                 } else {
 155                         drop_mmu_context(mm, cpu);
 156                 }
 157                 flush_itlb();
 158                 local_irq_restore(flags);
 159         }
 160 }
 161
 162 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 163 {
 164         unsigned long size, flags;
 165
 166         local_irq_save(flags);
 167         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 168         size = (size + 1) >> 1;
 169         if (size <= (current_cpu_data.tlbsizeftlbsets ?
 170                      current_cpu_data.tlbsize / 8 :
 171                      current_cpu_data.tlbsize / 2)) {
 172                 int pid = read_c0_entryhi();
 173
 174                 start &= (PAGE_MASK << 1);
 175                 end += ((PAGE_SIZE << 1) - 1);
 176                 end &= (PAGE_MASK << 1);
 177
 178                 while (start < end) {
 179                         int idx;
 180
 181                         write_c0_entryhi(start);
 182                         start += (PAGE_SIZE << 1);
 183                         mtc0_tlbw_hazard();
 184                         tlb_probe();
 185                         tlb_probe_hazard();
 186                         idx = read_c0_index();
 187                         write_c0_entrylo0(0);
 188                         write_c0_entrylo1(0);
 189                         if (idx < 0)
 190                                 continue;
 191                         /* Make sure all entries differ. */
 192                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 193                         mtc0_tlbw_hazard();
 194                         tlb_write_indexed();
 195                 }
 196                 tlbw_use_hazard();
 197                 write_c0_entryhi(pid);
 198         } else {
 199                 local_flush_tlb_all();
 200         }
 201         flush_itlb();
 202         local_irq_restore(flags);
 203 }
 204
 205 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 206 {
 207         int cpu = smp_processor_id();
 208
 209         if (cpu_context(cpu, vma->vm_mm) != 0) {
 210                 unsigned long flags;
 211                 int oldpid, newpid, idx;
 212
 213                 newpid = cpu_asid(cpu, vma->vm_mm);
 214                 page &= (PAGE_MASK << 1);
 215                 local_irq_save(flags);
 216                 oldpid = read_c0_entryhi();
 217                 write_c0_entryhi(page | newpid);
 218                 mtc0_tlbw_hazard();
 219                 tlb_probe();
 220                 tlb_probe_hazard();
 221                 idx = read_c0_index();
 222                 write_c0_entrylo0(0);
 223                 write_c0_entrylo1(0);
 224                 if (idx < 0)
 225                         goto finish;
 226                 /* Make sure all entries differ. */
 227                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 228                 mtc0_tlbw_hazard();
 229                 tlb_write_indexed();
 230                 tlbw_use_hazard();
 231
 232         finish:
 233                 write_c0_entryhi(oldpid);
 234                 flush_itlb_vm(vma);
 235                 local_irq_restore(flags);
 236         }
 237 }
 238
 239 /*
 240  * This one is only used for pages with the global bit set so we don't care
 241  * much about the ASID.
 242  */
 243 void local_flush_tlb_one(unsigned long page)
 244 {
 245         unsigned long flags;
 246         int oldpid, idx;
 247
 248         local_irq_save(flags);
 249         oldpid = read_c0_entryhi();
 250         page &= (PAGE_MASK << 1);
 251         write_c0_entryhi(page);
 252         mtc0_tlbw_hazard();
 253         tlb_probe();
 254         tlb_probe_hazard();
 255         idx = read_c0_index();
 256         write_c0_entrylo0(0);
 257         write_c0_entrylo1(0);
 258         if (idx >= 0) {
 259                 /* Make sure all entries differ. */
 260                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 261                 mtc0_tlbw_hazard();
 262                 tlb_write_indexed();
 263                 tlbw_use_hazard();
 264         }
 265         write_c0_entryhi(oldpid);
 266         flush_itlb();
 267         local_irq_restore(flags);
 268 }
 269
 270 /*
 271  * We will need multiple versions of update_mmu_cache(), one that just
 272  * updates the TLB with the new pte(s), and another which also checks
 273  * for the R4k "end of page" hardware bug and does the needy.
 274  */
 275 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 276 {
 277         unsigned long flags;
 278         pgd_t *pgdp;
 279         pud_t *pudp;
 280         pmd_t *pmdp;
 281         pte_t *ptep;
 282         int idx, pid;
 283
 284         /*
 285          * Handle debugger faulting in for debugee.
 286          */
 287         if (current->active_mm != vma->vm_mm)
 288                 return;
 289
 290         local_irq_save(flags);
 291
 292         pid = read_c0_entryhi() & ASID_MASK;
 293         address &= (PAGE_MASK << 1);
 294         write_c0_entryhi(address | pid);
 295         pgdp = pgd_offset(vma->vm_mm, address);
 296         mtc0_tlbw_hazard();
 297         tlb_probe();
 298         tlb_probe_hazard();
 299         pudp = pud_offset(pgdp, address);
 300         pmdp = pmd_offset(pudp, address);
 301         idx = read_c0_index();
 302 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
 303         /* this could be a huge page  */
 304         if (pmd_huge(*pmdp)) {
 305                 unsigned long lo;
 306                 write_c0_pagemask(PM_HUGE_MASK);
 307                 ptep = (pte_t *)pmdp;
 308                 lo = pte_to_entrylo(pte_val(*ptep));
 309                 write_c0_entrylo0(lo);
 310                 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
 311
 312                 mtc0_tlbw_hazard();
 313                 if (idx < 0)
 314                         tlb_write_random();
 315                 else
 316                         tlb_write_indexed();
 317                 tlbw_use_hazard();
 318                 write_c0_pagemask(PM_DEFAULT_MASK);
 319         } else
 320 #endif
 321         {
 322                 ptep = pte_offset_map(pmdp, address);
 323
 324 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
 325                 write_c0_entrylo0(ptep->pte_high);
 326                 ptep++;
 327                 write_c0_entrylo1(ptep->pte_high);
 328 #else
 329                 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
 330                 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
 331 #endif
 332                 mtc0_tlbw_hazard();
 333                 if (idx < 0)
 334                         tlb_write_random();
 335                 else
 336                         tlb_write_indexed();
 337         }
 338         tlbw_use_hazard();
 339         flush_itlb_vm(vma);
 340         local_irq_restore(flags);
 341 }
 342
 343 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 344                      unsigned long entryhi, unsigned long pagemask)
 345 {
 346         unsigned long flags;
 347         unsigned long wired;
 348         unsigned long old_pagemask;
 349         unsigned long old_ctx;
 350
 351         local_irq_save(flags);
 352         /* Save old context and create impossible VPN2 value */
 353         old_ctx = read_c0_entryhi();
 354         old_pagemask = read_c0_pagemask();
 355         wired = read_c0_wired();
 356         write_c0_wired(wired + 1);
 357         write_c0_index(wired);
 358         tlbw_use_hazard();      /* What is the hazard here? */
 359         write_c0_pagemask(pagemask);
 360         write_c0_entryhi(entryhi);
 361         write_c0_entrylo0(entrylo0);
 362         write_c0_entrylo1(entrylo1);
 363         mtc0_tlbw_hazard();
 364         tlb_write_indexed();
 365         tlbw_use_hazard();
 366
 367         write_c0_entryhi(old_ctx);
 368         tlbw_use_hazard();      /* What is the hazard here? */
 369         write_c0_pagemask(old_pagemask);
 370         local_flush_tlb_all();
 371         local_irq_restore(flags);
 372 }
 373
 374 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 375
 376 int __init has_transparent_hugepage(void)
 377 {
 378         unsigned int mask;
 379         unsigned long flags;
 380
 381         local_irq_save(flags);
 382         write_c0_pagemask(PM_HUGE_MASK);
 383         back_to_back_c0_hazard();
 384         mask = read_c0_pagemask();
 385         write_c0_pagemask(PM_DEFAULT_MASK);
 386
 387         local_irq_restore(flags);
 388
 389         return mask == PM_HUGE_MASK;
 390 }
 391
 392 #endif /* CONFIG_TRANSPARENT_HUGEPAGE  */
 393
 394 static int ntlb;
 395 static int __init set_ntlb(char *str)
 396 {
 397         get_option(&str, &ntlb);
 398         return 1;
 399 }
 400
 401 __setup("ntlb=", set_ntlb);
 402
 403 /*
 404  * Configure TLB (for init or after a CPU has been powered off).
 405  */
 406 static void r4k_tlb_configure(void)
 407 {
 408         /*
 409          * You should never change this register:
 410          *   - On R4600 1.7 the tlbp never hits for pages smaller than
 411          *     the value in the c0_pagemask register.
 412          *   - The entire mm handling assumes the c0_pagemask register to
 413          *     be set to fixed-size pages.
 414          */
 415         write_c0_pagemask(PM_DEFAULT_MASK);
 416         write_c0_wired(0);
 417         if (current_cpu_type() == CPU_R10000 ||
 418             current_cpu_type() == CPU_R12000 ||
 419             current_cpu_type() == CPU_R14000)
 420                 write_c0_framemask(0);
 421
 422         if (cpu_has_rixi) {
 423                 /*
 424                  * Enable the no read, no exec bits, and enable large virtual
 425                  * address.
 426                  */
 427                 u32 pg = PG_RIE | PG_XIE;
 428 #ifdef CONFIG_64BIT
 429                 pg |= PG_ELPA;
 430 #endif
 431                 write_c0_pagegrain(pg);
 432         }
 433
 434         /* From this point on the ARC firmware is dead.  */
 435         local_flush_tlb_all();
 436
 437         /* Did I tell you that ARC SUCKS?  */
 438 }
 439
 440 void tlb_init(void)
 441 {
 442         r4k_tlb_configure();
 443
 444         if (ntlb) {
 445                 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
 446                         int wired = current_cpu_data.tlbsize - ntlb;
 447                         write_c0_wired(wired);
 448                         write_c0_index(wired-1);
 449                         printk("Restricting TLB to %d entries\n", ntlb);
 450                 } else
 451                         printk("Ignoring invalid argument ntlb=%d\n", ntlb);
 452         }
 453
 454         build_tlb_refill_handler();
 455 }
 456
 457 static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd,
 458                                void *v)
 459 {
 460         switch (cmd) {
 461         case CPU_PM_ENTER_FAILED:
 462         case CPU_PM_EXIT:
 463                 r4k_tlb_configure();
 464                 break;
 465         }
 466
 467         return NOTIFY_OK;
 468 }
 469
 470 static struct notifier_block r4k_tlb_pm_notifier_block = {
 471         .notifier_call = r4k_tlb_pm_notifier,
 472 };
 473
 474 static int __init r4k_tlb_init_pm(void)
 475 {
 476         return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block);
 477 }
 478 arch_initcall(r4k_tlb_init_pm);