2 * srmmu.c: SRMMU specific routines for memory management.
4 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
7 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
11 #include <linux/seq_file.h>
12 #include <linux/spinlock.h>
13 #include <linux/bootmem.h>
14 #include <linux/pagemap.h>
15 #include <linux/vmalloc.h>
16 #include <linux/kdebug.h>
17 #include <linux/export.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
20 #include <linux/log2.h>
21 #include <linux/gfp.h>
25 #include <asm/mmu_context.h>
26 #include <asm/cacheflush.h>
27 #include <asm/tlbflush.h>
28 #include <asm/io-unit.h>
29 #include <asm/pgalloc.h>
30 #include <asm/pgtable.h>
31 #include <asm/bitext.h>
32 #include <asm/vaddrs.h>
33 #include <asm/cache.h>
34 #include <asm/traps.h>
35 #include <asm/oplib.h>
43 /* Now the cpu specific definitions. */
44 #include <asm/turbosparc.h>
45 #include <asm/tsunami.h>
46 #include <asm/viking.h>
47 #include <asm/swift.h>
54 enum mbus_module srmmu_modtype;
55 static unsigned int hwbug_bitmask;
59 extern struct resource sparc_iomap;
61 extern unsigned long last_valid_pfn;
63 static pgd_t *srmmu_swapper_pg_dir;
65 const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops;
66 EXPORT_SYMBOL(sparc32_cachetlb_ops);
69 const struct sparc32_cachetlb_ops *local_ops;
71 #define FLUSH_BEGIN(mm)
74 #define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) {
78 int flush_page_for_dma_global = 1;
82 ctxd_t *srmmu_ctx_table_phys;
83 static ctxd_t *srmmu_context_table;
85 int viking_mxcc_present;
86 static DEFINE_SPINLOCK(srmmu_context_spinlock);
88 static int is_hypersparc;
90 static int srmmu_cache_pagetables;
92 /* these will be initialized in srmmu_nocache_calcsize() */
93 static unsigned long srmmu_nocache_size;
94 static unsigned long srmmu_nocache_end;
96 /* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
97 #define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
99 /* The context table is a nocache user with the biggest alignment needs. */
100 #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
102 void *srmmu_nocache_pool;
103 void *srmmu_nocache_bitmap;
104 static struct bit_map srmmu_nocache_map;
106 static inline int srmmu_pmd_none(pmd_t pmd)
107 { return !(pmd_val(pmd) & 0xFFFFFFF); }
109 /* XXX should we hyper_flush_whole_icache here - Anton */
110 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
111 { set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); }
113 void pmd_set(pmd_t *pmdp, pte_t *ptep)
115 unsigned long ptp; /* Physical address, shifted right by 4 */
118 ptp = __nocache_pa((unsigned long) ptep) >> 4;
119 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
120 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
121 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
125 void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
127 unsigned long ptp; /* Physical address, shifted right by 4 */
130 ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */
131 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
132 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
133 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
137 /* Find an entry in the third-level page table.. */
138 pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address)
142 pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4);
143 return (pte_t *) pte +
144 ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
148 * size: bytes to allocate in the nocache area.
149 * align: bytes, number to align at.
150 * Returns the virtual address of the allocated area.
152 static void *__srmmu_get_nocache(int size, int align)
157 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
158 printk(KERN_ERR "Size 0x%x too small for nocache request\n",
160 size = SRMMU_NOCACHE_BITMAP_SHIFT;
162 if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) {
163 printk(KERN_ERR "Size 0x%x unaligned int nocache request\n",
165 size += SRMMU_NOCACHE_BITMAP_SHIFT - 1;
167 BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
169 offset = bit_map_string_get(&srmmu_nocache_map,
170 size >> SRMMU_NOCACHE_BITMAP_SHIFT,
171 align >> SRMMU_NOCACHE_BITMAP_SHIFT);
173 printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
174 size, (int) srmmu_nocache_size,
175 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
179 addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
183 void *srmmu_get_nocache(int size, int align)
187 tmp = __srmmu_get_nocache(size, align);
190 memset(tmp, 0, size);
195 void srmmu_free_nocache(void *addr, int size)
200 vaddr = (unsigned long)addr;
201 if (vaddr < SRMMU_NOCACHE_VADDR) {
202 printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
203 vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
206 if (vaddr + size > srmmu_nocache_end) {
207 printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
208 vaddr, srmmu_nocache_end);
211 if (!is_power_of_2(size)) {
212 printk("Size 0x%x is not a power of 2\n", size);
215 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
216 printk("Size 0x%x is too small\n", size);
219 if (vaddr & (size - 1)) {
220 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
224 offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
225 size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
227 bit_map_clear(&srmmu_nocache_map, offset, size);
230 static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
233 /* Return how much physical memory we have. */
234 static unsigned long __init probe_memory(void)
236 unsigned long total = 0;
239 for (i = 0; sp_banks[i].num_bytes; i++)
240 total += sp_banks[i].num_bytes;
246 * Reserve nocache dynamically proportionally to the amount of
247 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
249 static void __init srmmu_nocache_calcsize(void)
251 unsigned long sysmemavail = probe_memory() / 1024;
252 int srmmu_nocache_npages;
254 srmmu_nocache_npages =
255 sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
257 /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
258 // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
259 if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
260 srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
262 /* anything above 1280 blows up */
263 if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
264 srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
266 srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
267 srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
270 static void __init srmmu_nocache_init(void)
272 unsigned int bitmap_bits;
276 unsigned long paddr, vaddr;
277 unsigned long pteval;
279 bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
281 srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size,
282 SRMMU_NOCACHE_ALIGN_MAX, 0UL);
283 memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
285 srmmu_nocache_bitmap =
286 __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
287 SMP_CACHE_BYTES, 0UL);
288 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
290 srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
291 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
292 init_mm.pgd = srmmu_swapper_pg_dir;
294 srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
296 paddr = __pa((unsigned long)srmmu_nocache_pool);
297 vaddr = SRMMU_NOCACHE_VADDR;
299 while (vaddr < srmmu_nocache_end) {
300 pgd = pgd_offset_k(vaddr);
301 pmd = pmd_offset(__nocache_fix(pgd), vaddr);
302 pte = pte_offset_kernel(__nocache_fix(pmd), vaddr);
304 pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
306 if (srmmu_cache_pagetables)
307 pteval |= SRMMU_CACHE;
309 set_pte(__nocache_fix(pte), __pte(pteval));
319 pgd_t *get_pgd_fast(void)
323 pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
325 pgd_t *init = pgd_offset_k(0);
326 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
327 memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
328 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
335 * Hardware needs alignment to 256 only, but we align to whole page size
336 * to reduce fragmentation problems due to the buddy principle.
337 * XXX Provide actual fragmentation statistics in /proc.
339 * Alignments up to the page size are the same for physical and virtual
340 * addresses of the nocache area.
342 pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
347 if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
349 page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
350 if (!pgtable_page_ctor(page)) {
357 void pte_free(struct mm_struct *mm, pgtable_t pte)
361 pgtable_page_dtor(pte);
362 p = (unsigned long)page_address(pte); /* Cached address (for test) */
365 p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */
367 /* free non cached virtual address*/
368 srmmu_free_nocache(__nocache_va(p), PTE_SIZE);
371 /* context handling - a dynamically sized pool is used */
372 #define NO_CONTEXT -1
375 struct ctx_list *next;
376 struct ctx_list *prev;
377 unsigned int ctx_number;
378 struct mm_struct *ctx_mm;
381 static struct ctx_list *ctx_list_pool;
382 static struct ctx_list ctx_free;
383 static struct ctx_list ctx_used;
385 /* At boot time we determine the number of contexts */
386 static int num_contexts;
388 static inline void remove_from_ctx_list(struct ctx_list *entry)
390 entry->next->prev = entry->prev;
391 entry->prev->next = entry->next;
394 static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
397 (entry->prev = head->prev)->next = entry;
400 #define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
401 #define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
404 static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
406 struct ctx_list *ctxp;
408 ctxp = ctx_free.next;
409 if (ctxp != &ctx_free) {
410 remove_from_ctx_list(ctxp);
411 add_to_used_ctxlist(ctxp);
412 mm->context = ctxp->ctx_number;
416 ctxp = ctx_used.next;
417 if (ctxp->ctx_mm == old_mm)
419 if (ctxp == &ctx_used)
420 panic("out of mmu contexts");
421 flush_cache_mm(ctxp->ctx_mm);
422 flush_tlb_mm(ctxp->ctx_mm);
423 remove_from_ctx_list(ctxp);
424 add_to_used_ctxlist(ctxp);
425 ctxp->ctx_mm->context = NO_CONTEXT;
427 mm->context = ctxp->ctx_number;
430 static inline void free_context(int context)
432 struct ctx_list *ctx_old;
434 ctx_old = ctx_list_pool + context;
435 remove_from_ctx_list(ctx_old);
436 add_to_free_ctxlist(ctx_old);
439 static void __init sparc_context_init(int numctx)
444 size = numctx * sizeof(struct ctx_list);
445 ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL);
447 for (ctx = 0; ctx < numctx; ctx++) {
448 struct ctx_list *clist;
450 clist = (ctx_list_pool + ctx);
451 clist->ctx_number = ctx;
452 clist->ctx_mm = NULL;
454 ctx_free.next = ctx_free.prev = &ctx_free;
455 ctx_used.next = ctx_used.prev = &ctx_used;
456 for (ctx = 0; ctx < numctx; ctx++)
457 add_to_free_ctxlist(ctx_list_pool + ctx);
460 void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
461 struct task_struct *tsk)
463 if (mm->context == NO_CONTEXT) {
464 spin_lock(&srmmu_context_spinlock);
465 alloc_context(old_mm, mm);
466 spin_unlock(&srmmu_context_spinlock);
467 srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
470 if (sparc_cpu_model == sparc_leon)
474 hyper_flush_whole_icache();
476 srmmu_set_context(mm->context);
479 /* Low level IO area allocation on the SRMMU. */
480 static inline void srmmu_mapioaddr(unsigned long physaddr,
481 unsigned long virt_addr, int bus_type)
488 physaddr &= PAGE_MASK;
489 pgdp = pgd_offset_k(virt_addr);
490 pmdp = pmd_offset(pgdp, virt_addr);
491 ptep = pte_offset_kernel(pmdp, virt_addr);
492 tmp = (physaddr >> 4) | SRMMU_ET_PTE;
494 /* I need to test whether this is consistent over all
495 * sun4m's. The bus_type represents the upper 4 bits of
496 * 36-bit physical address on the I/O space lines...
498 tmp |= (bus_type << 28);
500 __flush_page_to_ram(virt_addr);
501 set_pte(ptep, __pte(tmp));
504 void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
505 unsigned long xva, unsigned int len)
509 srmmu_mapioaddr(xpa, xva, bus);
516 static inline void srmmu_unmapioaddr(unsigned long virt_addr)
522 pgdp = pgd_offset_k(virt_addr);
523 pmdp = pmd_offset(pgdp, virt_addr);
524 ptep = pte_offset_kernel(pmdp, virt_addr);
526 /* No need to flush uncacheable page. */
530 void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
534 srmmu_unmapioaddr(virt_addr);
535 virt_addr += PAGE_SIZE;
541 extern void tsunami_flush_cache_all(void);
542 extern void tsunami_flush_cache_mm(struct mm_struct *mm);
543 extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
544 extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
545 extern void tsunami_flush_page_to_ram(unsigned long page);
546 extern void tsunami_flush_page_for_dma(unsigned long page);
547 extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
548 extern void tsunami_flush_tlb_all(void);
549 extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
550 extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
551 extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
552 extern void tsunami_setup_blockops(void);
555 extern void swift_flush_cache_all(void);
556 extern void swift_flush_cache_mm(struct mm_struct *mm);
557 extern void swift_flush_cache_range(struct vm_area_struct *vma,
558 unsigned long start, unsigned long end);
559 extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
560 extern void swift_flush_page_to_ram(unsigned long page);
561 extern void swift_flush_page_for_dma(unsigned long page);
562 extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
563 extern void swift_flush_tlb_all(void);
564 extern void swift_flush_tlb_mm(struct mm_struct *mm);
565 extern void swift_flush_tlb_range(struct vm_area_struct *vma,
566 unsigned long start, unsigned long end);
567 extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
569 #if 0 /* P3: deadwood to debug precise flushes on Swift. */
570 void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
575 if ((ctx1 = vma->vm_mm->context) != -1) {
576 cctx = srmmu_get_context();
577 /* Is context # ever different from current context? P3 */
579 printk("flush ctx %02x curr %02x\n", ctx1, cctx);
580 srmmu_set_context(ctx1);
581 swift_flush_page(page);
582 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
583 "r" (page), "i" (ASI_M_FLUSH_PROBE));
584 srmmu_set_context(cctx);
586 /* Rm. prot. bits from virt. c. */
587 /* swift_flush_cache_all(); */
588 /* swift_flush_cache_page(vma, page); */
589 swift_flush_page(page);
591 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
592 "r" (page), "i" (ASI_M_FLUSH_PROBE));
593 /* same as above: srmmu_flush_tlb_page() */
600 * The following are all MBUS based SRMMU modules, and therefore could
601 * be found in a multiprocessor configuration. On the whole, these
602 * chips seems to be much more touchy about DVMA and page tables
603 * with respect to cache coherency.
607 extern void viking_flush_cache_all(void);
608 extern void viking_flush_cache_mm(struct mm_struct *mm);
609 extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
611 extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
612 extern void viking_flush_page_to_ram(unsigned long page);
613 extern void viking_flush_page_for_dma(unsigned long page);
614 extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
615 extern void viking_flush_page(unsigned long page);
616 extern void viking_mxcc_flush_page(unsigned long page);
617 extern void viking_flush_tlb_all(void);
618 extern void viking_flush_tlb_mm(struct mm_struct *mm);
619 extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
621 extern void viking_flush_tlb_page(struct vm_area_struct *vma,
623 extern void sun4dsmp_flush_tlb_all(void);
624 extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
625 extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
627 extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
631 extern void hypersparc_flush_cache_all(void);
632 extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
633 extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
634 extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
635 extern void hypersparc_flush_page_to_ram(unsigned long page);
636 extern void hypersparc_flush_page_for_dma(unsigned long page);
637 extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
638 extern void hypersparc_flush_tlb_all(void);
639 extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
640 extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
641 extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
642 extern void hypersparc_setup_blockops(void);
645 * NOTE: All of this startup code assumes the low 16mb (approx.) of
646 * kernel mappings are done with one single contiguous chunk of
647 * ram. On small ram machines (classics mainly) we only get
648 * around 8mb mapped for us.
651 static void __init early_pgtable_allocfail(char *type)
653 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
657 static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
664 while (start < end) {
665 pgdp = pgd_offset_k(start);
666 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
667 pmdp = __srmmu_get_nocache(
668 SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
670 early_pgtable_allocfail("pmd");
671 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
672 pgd_set(__nocache_fix(pgdp), pmdp);
674 pmdp = pmd_offset(__nocache_fix(pgdp), start);
675 if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
676 ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
678 early_pgtable_allocfail("pte");
679 memset(__nocache_fix(ptep), 0, PTE_SIZE);
680 pmd_set(__nocache_fix(pmdp), ptep);
682 if (start > (0xffffffffUL - PMD_SIZE))
684 start = (start + PMD_SIZE) & PMD_MASK;
688 static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
695 while (start < end) {
696 pgdp = pgd_offset_k(start);
697 if (pgd_none(*pgdp)) {
698 pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
700 early_pgtable_allocfail("pmd");
701 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
704 pmdp = pmd_offset(pgdp, start);
705 if (srmmu_pmd_none(*pmdp)) {
706 ptep = __srmmu_get_nocache(PTE_SIZE,
709 early_pgtable_allocfail("pte");
710 memset(ptep, 0, PTE_SIZE);
713 if (start > (0xffffffffUL - PMD_SIZE))
715 start = (start + PMD_SIZE) & PMD_MASK;
719 /* These flush types are not available on all chips... */
720 static inline unsigned long srmmu_probe(unsigned long vaddr)
722 unsigned long retval;
724 if (sparc_cpu_model != sparc_leon) {
727 __asm__ __volatile__("lda [%1] %2, %0\n\t" :
729 "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE));
731 retval = leon_swprobe(vaddr, 0);
737 * This is much cleaner than poking around physical address space
738 * looking at the prom's page table directly which is what most
739 * other OS's do. Yuck... this is much better.
741 static void __init srmmu_inherit_prom_mappings(unsigned long start,
744 unsigned long probed;
749 int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
751 while (start <= end) {
753 break; /* probably wrap around */
754 if (start == 0xfef00000)
755 start = KADB_DEBUGGER_BEGVM;
756 probed = srmmu_probe(start);
758 /* continue probing until we find an entry */
763 /* A red snapper, see what it really is. */
765 addr = start - PAGE_SIZE;
767 if (!(start & ~(SRMMU_REAL_PMD_MASK))) {
768 if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed)
772 if (!(start & ~(SRMMU_PGDIR_MASK))) {
773 if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed)
777 pgdp = pgd_offset_k(start);
779 *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed);
780 start += SRMMU_PGDIR_SIZE;
783 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
784 pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
785 SRMMU_PMD_TABLE_SIZE);
787 early_pgtable_allocfail("pmd");
788 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
789 pgd_set(__nocache_fix(pgdp), pmdp);
791 pmdp = pmd_offset(__nocache_fix(pgdp), start);
792 if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
793 ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
795 early_pgtable_allocfail("pte");
796 memset(__nocache_fix(ptep), 0, PTE_SIZE);
797 pmd_set(__nocache_fix(pmdp), ptep);
800 /* We bend the rule where all 16 PTPs in a pmd_t point
801 * inside the same PTE page, and we leak a perfectly
802 * good hardware PTE piece. Alternatives seem worse.
804 unsigned int x; /* Index of HW PMD in soft cluster */
806 x = (start >> PMD_SHIFT) & 15;
807 val = &pmdp->pmdv[x];
808 *(unsigned long *)__nocache_fix(val) = probed;
809 start += SRMMU_REAL_PMD_SIZE;
812 ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
813 *(pte_t *)__nocache_fix(ptep) = __pte(probed);
818 #define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
820 /* Create a third-level SRMMU 16MB page mapping. */
821 static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
823 pgd_t *pgdp = pgd_offset_k(vaddr);
824 unsigned long big_pte;
826 big_pte = KERNEL_PTE(phys_base >> 4);
827 *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
830 /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
831 static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
833 unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK);
834 unsigned long vstart = (vbase & SRMMU_PGDIR_MASK);
835 unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
836 /* Map "low" memory only */
837 const unsigned long min_vaddr = PAGE_OFFSET;
838 const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
840 if (vstart < min_vaddr || vstart >= max_vaddr)
843 if (vend > max_vaddr || vend < min_vaddr)
846 while (vstart < vend) {
847 do_large_mapping(vstart, pstart);
848 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
853 static void __init map_kernel(void)
858 do_large_mapping(PAGE_OFFSET, phys_base);
861 for (i = 0; sp_banks[i].num_bytes != 0; i++) {
862 map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
866 void (*poke_srmmu)(void) = NULL;
868 extern unsigned long bootmem_init(unsigned long *pages_avail);
870 void __init srmmu_paging_init(void)
878 unsigned long pages_avail;
880 init_mm.context = (unsigned long) NO_CONTEXT;
881 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
883 if (sparc_cpu_model == sun4d)
884 num_contexts = 65536; /* We know it is Viking */
886 /* Find the number of contexts on the srmmu. */
887 cpunode = prom_getchild(prom_root_node);
889 while (cpunode != 0) {
890 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
891 if (!strcmp(node_str, "cpu")) {
892 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
895 cpunode = prom_getsibling(cpunode);
900 prom_printf("Something wrong, can't find cpu node in paging_init.\n");
905 last_valid_pfn = bootmem_init(&pages_avail);
907 srmmu_nocache_calcsize();
908 srmmu_nocache_init();
909 srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE));
912 /* ctx table has to be physically aligned to its size */
913 srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t));
914 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
916 for (i = 0; i < num_contexts; i++)
917 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
920 srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
922 /* Stop from hanging here... */
923 local_ops->tlb_all();
929 srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
930 srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
932 srmmu_allocate_ptable_skeleton(
933 __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
934 srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
936 pgd = pgd_offset_k(PKMAP_BASE);
937 pmd = pmd_offset(pgd, PKMAP_BASE);
938 pte = pte_offset_kernel(pmd, PKMAP_BASE);
939 pkmap_page_table = pte;
944 sparc_context_init(num_contexts);
949 unsigned long zones_size[MAX_NR_ZONES];
950 unsigned long zholes_size[MAX_NR_ZONES];
951 unsigned long npages;
954 for (znum = 0; znum < MAX_NR_ZONES; znum++)
955 zones_size[znum] = zholes_size[znum] = 0;
957 npages = max_low_pfn - pfn_base;
959 zones_size[ZONE_DMA] = npages;
960 zholes_size[ZONE_DMA] = npages - pages_avail;
962 npages = highend_pfn - max_low_pfn;
963 zones_size[ZONE_HIGHMEM] = npages;
964 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
966 free_area_init_node(0, zones_size, pfn_base, zholes_size);
970 void mmu_info(struct seq_file *m)
975 "nocache total\t: %ld\n"
976 "nocache used\t: %d\n",
980 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
983 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
985 mm->context = NO_CONTEXT;
989 void destroy_context(struct mm_struct *mm)
992 if (mm->context != NO_CONTEXT) {
994 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
996 spin_lock(&srmmu_context_spinlock);
997 free_context(mm->context);
998 spin_unlock(&srmmu_context_spinlock);
999 mm->context = NO_CONTEXT;
1003 /* Init various srmmu chip types. */
1004 static void __init srmmu_is_bad(void)
1006 prom_printf("Could not determine SRMMU chip type.\n");
1010 static void __init init_vac_layout(void)
1017 unsigned long max_size = 0;
1018 unsigned long min_line_size = 0x10000000;
1021 nd = prom_getchild(prom_root_node);
1022 while ((nd = prom_getsibling(nd)) != 0) {
1023 prom_getstring(nd, "device_type", node_str, sizeof(node_str));
1024 if (!strcmp(node_str, "cpu")) {
1025 vac_line_size = prom_getint(nd, "cache-line-size");
1026 if (vac_line_size == -1) {
1027 prom_printf("can't determine cache-line-size, halting.\n");
1030 cache_lines = prom_getint(nd, "cache-nlines");
1031 if (cache_lines == -1) {
1032 prom_printf("can't determine cache-nlines, halting.\n");
1036 vac_cache_size = cache_lines * vac_line_size;
1038 if (vac_cache_size > max_size)
1039 max_size = vac_cache_size;
1040 if (vac_line_size < min_line_size)
1041 min_line_size = vac_line_size;
1042 //FIXME: cpus not contiguous!!
1044 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1052 prom_printf("No CPU nodes found, halting.\n");
1056 vac_cache_size = max_size;
1057 vac_line_size = min_line_size;
1059 printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
1060 (int)vac_cache_size, (int)vac_line_size);
1063 static void poke_hypersparc(void)
1065 volatile unsigned long clear;
1066 unsigned long mreg = srmmu_get_mmureg();
1068 hyper_flush_unconditional_combined();
1070 mreg &= ~(HYPERSPARC_CWENABLE);
1071 mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
1072 mreg |= (HYPERSPARC_CMODE);
1074 srmmu_set_mmureg(mreg);
1076 #if 0 /* XXX I think this is bad news... -DaveM */
1077 hyper_clear_all_tags();
1080 put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
1081 hyper_flush_whole_icache();
1082 clear = srmmu_get_faddr();
1083 clear = srmmu_get_fstatus();
1086 static const struct sparc32_cachetlb_ops hypersparc_ops = {
1087 .cache_all = hypersparc_flush_cache_all,
1088 .cache_mm = hypersparc_flush_cache_mm,
1089 .cache_page = hypersparc_flush_cache_page,
1090 .cache_range = hypersparc_flush_cache_range,
1091 .tlb_all = hypersparc_flush_tlb_all,
1092 .tlb_mm = hypersparc_flush_tlb_mm,
1093 .tlb_page = hypersparc_flush_tlb_page,
1094 .tlb_range = hypersparc_flush_tlb_range,
1095 .page_to_ram = hypersparc_flush_page_to_ram,
1096 .sig_insns = hypersparc_flush_sig_insns,
1097 .page_for_dma = hypersparc_flush_page_for_dma,
1100 static void __init init_hypersparc(void)
1102 srmmu_name = "ROSS HyperSparc";
1103 srmmu_modtype = HyperSparc;
1108 sparc32_cachetlb_ops = &hypersparc_ops;
1110 poke_srmmu = poke_hypersparc;
1112 hypersparc_setup_blockops();
1115 static void poke_swift(void)
1119 /* Clear any crap from the cache or else... */
1120 swift_flush_cache_all();
1122 /* Enable I & D caches */
1123 mreg = srmmu_get_mmureg();
1124 mreg |= (SWIFT_IE | SWIFT_DE);
1126 * The Swift branch folding logic is completely broken. At
1127 * trap time, if things are just right, if can mistakenly
1128 * think that a trap is coming from kernel mode when in fact
1129 * it is coming from user mode (it mis-executes the branch in
1130 * the trap code). So you see things like crashme completely
1131 * hosing your machine which is completely unacceptable. Turn
1132 * this shit off... nice job Fujitsu.
1134 mreg &= ~(SWIFT_BF);
1135 srmmu_set_mmureg(mreg);
1138 static const struct sparc32_cachetlb_ops swift_ops = {
1139 .cache_all = swift_flush_cache_all,
1140 .cache_mm = swift_flush_cache_mm,
1141 .cache_page = swift_flush_cache_page,
1142 .cache_range = swift_flush_cache_range,
1143 .tlb_all = swift_flush_tlb_all,
1144 .tlb_mm = swift_flush_tlb_mm,
1145 .tlb_page = swift_flush_tlb_page,
1146 .tlb_range = swift_flush_tlb_range,
1147 .page_to_ram = swift_flush_page_to_ram,
1148 .sig_insns = swift_flush_sig_insns,
1149 .page_for_dma = swift_flush_page_for_dma,
1152 #define SWIFT_MASKID_ADDR 0x10003018
1153 static void __init init_swift(void)
1155 unsigned long swift_rev;
1157 __asm__ __volatile__("lda [%1] %2, %0\n\t"
1158 "srl %0, 0x18, %0\n\t" :
1160 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1161 srmmu_name = "Fujitsu Swift";
1162 switch (swift_rev) {
1167 srmmu_modtype = Swift_lots_o_bugs;
1168 hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
1170 * Gee george, I wonder why Sun is so hush hush about
1171 * this hardware bug... really braindamage stuff going
1172 * on here. However I think we can find a way to avoid
1173 * all of the workaround overhead under Linux. Basically,
1174 * any page fault can cause kernel pages to become user
1175 * accessible (the mmu gets confused and clears some of
1176 * the ACC bits in kernel ptes). Aha, sounds pretty
1177 * horrible eh? But wait, after extensive testing it appears
1178 * that if you use pgd_t level large kernel pte's (like the
1179 * 4MB pages on the Pentium) the bug does not get tripped
1180 * at all. This avoids almost all of the major overhead.
1181 * Welcome to a world where your vendor tells you to,
1182 * "apply this kernel patch" instead of "sorry for the
1183 * broken hardware, send it back and we'll give you
1184 * properly functioning parts"
1189 srmmu_modtype = Swift_bad_c;
1190 hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
1192 * You see Sun allude to this hardware bug but never
1193 * admit things directly, they'll say things like,
1194 * "the Swift chip cache problems" or similar.
1198 srmmu_modtype = Swift_ok;
1202 sparc32_cachetlb_ops = &swift_ops;
1203 flush_page_for_dma_global = 0;
1206 * Are you now convinced that the Swift is one of the
1207 * biggest VLSI abortions of all time? Bravo Fujitsu!
1208 * Fujitsu, the !#?!%$'d up processor people. I bet if
1209 * you examined the microcode of the Swift you'd find
1210 * XXX's all over the place.
1212 poke_srmmu = poke_swift;
1215 static void turbosparc_flush_cache_all(void)
1217 flush_user_windows();
1218 turbosparc_idflash_clear();
1221 static void turbosparc_flush_cache_mm(struct mm_struct *mm)
1224 flush_user_windows();
1225 turbosparc_idflash_clear();
1229 static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1231 FLUSH_BEGIN(vma->vm_mm)
1232 flush_user_windows();
1233 turbosparc_idflash_clear();
1237 static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1239 FLUSH_BEGIN(vma->vm_mm)
1240 flush_user_windows();
1241 if (vma->vm_flags & VM_EXEC)
1242 turbosparc_flush_icache();
1243 turbosparc_flush_dcache();
1247 /* TurboSparc is copy-back, if we turn it on, but this does not work. */
1248 static void turbosparc_flush_page_to_ram(unsigned long page)
1250 #ifdef TURBOSPARC_WRITEBACK
1251 volatile unsigned long clear;
1253 if (srmmu_probe(page))
1254 turbosparc_flush_page_cache(page);
1255 clear = srmmu_get_fstatus();
1259 static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1263 static void turbosparc_flush_page_for_dma(unsigned long page)
1265 turbosparc_flush_dcache();
1268 static void turbosparc_flush_tlb_all(void)
1270 srmmu_flush_whole_tlb();
1273 static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
1276 srmmu_flush_whole_tlb();
1280 static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1282 FLUSH_BEGIN(vma->vm_mm)
1283 srmmu_flush_whole_tlb();
1287 static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1289 FLUSH_BEGIN(vma->vm_mm)
1290 srmmu_flush_whole_tlb();
1295 static void poke_turbosparc(void)
1297 unsigned long mreg = srmmu_get_mmureg();
1298 unsigned long ccreg;
1300 /* Clear any crap from the cache or else... */
1301 turbosparc_flush_cache_all();
1302 /* Temporarily disable I & D caches */
1303 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE);
1304 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
1305 srmmu_set_mmureg(mreg);
1307 ccreg = turbosparc_get_ccreg();
1309 #ifdef TURBOSPARC_WRITEBACK
1310 ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */
1311 ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
1312 /* Write-back D-cache, emulate VLSI
1313 * abortion number three, not number one */
1315 /* For now let's play safe, optimize later */
1316 ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
1317 /* Do DVMA snooping in Dcache, Write-thru D-cache */
1318 ccreg &= ~(TURBOSPARC_uS2);
1319 /* Emulate VLSI abortion number three, not number one */
1322 switch (ccreg & 7) {
1323 case 0: /* No SE cache */
1324 case 7: /* Test mode */
1327 ccreg |= (TURBOSPARC_SCENABLE);
1329 turbosparc_set_ccreg(ccreg);
1331 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1332 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
1333 srmmu_set_mmureg(mreg);
1336 static const struct sparc32_cachetlb_ops turbosparc_ops = {
1337 .cache_all = turbosparc_flush_cache_all,
1338 .cache_mm = turbosparc_flush_cache_mm,
1339 .cache_page = turbosparc_flush_cache_page,
1340 .cache_range = turbosparc_flush_cache_range,
1341 .tlb_all = turbosparc_flush_tlb_all,
1342 .tlb_mm = turbosparc_flush_tlb_mm,
1343 .tlb_page = turbosparc_flush_tlb_page,
1344 .tlb_range = turbosparc_flush_tlb_range,
1345 .page_to_ram = turbosparc_flush_page_to_ram,
1346 .sig_insns = turbosparc_flush_sig_insns,
1347 .page_for_dma = turbosparc_flush_page_for_dma,
1350 static void __init init_turbosparc(void)
1352 srmmu_name = "Fujitsu TurboSparc";
1353 srmmu_modtype = TurboSparc;
1354 sparc32_cachetlb_ops = &turbosparc_ops;
1355 poke_srmmu = poke_turbosparc;
1358 static void poke_tsunami(void)
1360 unsigned long mreg = srmmu_get_mmureg();
1362 tsunami_flush_icache();
1363 tsunami_flush_dcache();
1364 mreg &= ~TSUNAMI_ITD;
1365 mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
1366 srmmu_set_mmureg(mreg);
1369 static const struct sparc32_cachetlb_ops tsunami_ops = {
1370 .cache_all = tsunami_flush_cache_all,
1371 .cache_mm = tsunami_flush_cache_mm,
1372 .cache_page = tsunami_flush_cache_page,
1373 .cache_range = tsunami_flush_cache_range,
1374 .tlb_all = tsunami_flush_tlb_all,
1375 .tlb_mm = tsunami_flush_tlb_mm,
1376 .tlb_page = tsunami_flush_tlb_page,
1377 .tlb_range = tsunami_flush_tlb_range,
1378 .page_to_ram = tsunami_flush_page_to_ram,
1379 .sig_insns = tsunami_flush_sig_insns,
1380 .page_for_dma = tsunami_flush_page_for_dma,
1383 static void __init init_tsunami(void)
1386 * Tsunami's pretty sane, Sun and TI actually got it
1387 * somewhat right this time. Fujitsu should have
1388 * taken some lessons from them.
1391 srmmu_name = "TI Tsunami";
1392 srmmu_modtype = Tsunami;
1393 sparc32_cachetlb_ops = &tsunami_ops;
1394 poke_srmmu = poke_tsunami;
1396 tsunami_setup_blockops();
1399 static void poke_viking(void)
1401 unsigned long mreg = srmmu_get_mmureg();
1402 static int smp_catch;
1404 if (viking_mxcc_present) {
1405 unsigned long mxcc_control = mxcc_get_creg();
1407 mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
1408 mxcc_control &= ~(MXCC_CTL_RRC);
1409 mxcc_set_creg(mxcc_control);
1412 * We don't need memory parity checks.
1413 * XXX This is a mess, have to dig out later. ecd.
1414 viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
1417 /* We do cache ptables on MXCC. */
1418 mreg |= VIKING_TCENABLE;
1420 unsigned long bpreg;
1422 mreg &= ~(VIKING_TCENABLE);
1424 /* Must disable mixed-cmd mode here for other cpu's. */
1425 bpreg = viking_get_bpreg();
1426 bpreg &= ~(VIKING_ACTION_MIX);
1427 viking_set_bpreg(bpreg);
1429 /* Just in case PROM does something funny. */
1434 mreg |= VIKING_SPENABLE;
1435 mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
1436 mreg |= VIKING_SBENABLE;
1437 mreg &= ~(VIKING_ACENABLE);
1438 srmmu_set_mmureg(mreg);
1441 static struct sparc32_cachetlb_ops viking_ops = {
1442 .cache_all = viking_flush_cache_all,
1443 .cache_mm = viking_flush_cache_mm,
1444 .cache_page = viking_flush_cache_page,
1445 .cache_range = viking_flush_cache_range,
1446 .tlb_all = viking_flush_tlb_all,
1447 .tlb_mm = viking_flush_tlb_mm,
1448 .tlb_page = viking_flush_tlb_page,
1449 .tlb_range = viking_flush_tlb_range,
1450 .page_to_ram = viking_flush_page_to_ram,
1451 .sig_insns = viking_flush_sig_insns,
1452 .page_for_dma = viking_flush_page_for_dma,
1456 /* On sun4d the cpu broadcasts local TLB flushes, so we can just
1457 * perform the local TLB flush and all the other cpus will see it.
1458 * But, unfortunately, there is a bug in the sun4d XBUS backplane
1459 * that requires that we add some synchronization to these flushes.
1461 * The bug is that the fifo which keeps track of all the pending TLB
1462 * broadcasts in the system is an entry or two too small, so if we
1463 * have too many going at once we'll overflow that fifo and lose a TLB
1464 * flush resulting in corruption.
1466 * Our workaround is to take a global spinlock around the TLB flushes,
1467 * which guarentees we won't ever have too many pending. It's a big
1468 * hammer, but a semaphore like system to make sure we only have N TLB
1469 * flushes going at once will require SMP locking anyways so there's
1470 * no real value in trying any harder than this.
1472 static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = {
1473 .cache_all = viking_flush_cache_all,
1474 .cache_mm = viking_flush_cache_mm,
1475 .cache_page = viking_flush_cache_page,
1476 .cache_range = viking_flush_cache_range,
1477 .tlb_all = sun4dsmp_flush_tlb_all,
1478 .tlb_mm = sun4dsmp_flush_tlb_mm,
1479 .tlb_page = sun4dsmp_flush_tlb_page,
1480 .tlb_range = sun4dsmp_flush_tlb_range,
1481 .page_to_ram = viking_flush_page_to_ram,
1482 .sig_insns = viking_flush_sig_insns,
1483 .page_for_dma = viking_flush_page_for_dma,
1487 static void __init init_viking(void)
1489 unsigned long mreg = srmmu_get_mmureg();
1491 /* Ahhh, the viking. SRMMU VLSI abortion number two... */
1492 if (mreg & VIKING_MMODE) {
1493 srmmu_name = "TI Viking";
1494 viking_mxcc_present = 0;
1498 * We need this to make sure old viking takes no hits
1499 * on it's cache for dma snoops to workaround the
1500 * "load from non-cacheable memory" interrupt bug.
1501 * This is only necessary because of the new way in
1502 * which we use the IOMMU.
1504 viking_ops.page_for_dma = viking_flush_page;
1506 viking_sun4d_smp_ops.page_for_dma = viking_flush_page;
1508 flush_page_for_dma_global = 0;
1510 srmmu_name = "TI Viking/MXCC";
1511 viking_mxcc_present = 1;
1512 srmmu_cache_pagetables = 1;
1515 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1518 if (sparc_cpu_model == sun4d)
1519 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1520 &viking_sun4d_smp_ops;
1523 poke_srmmu = poke_viking;
1526 /* Probe for the srmmu chip version. */
1527 static void __init get_srmmu_type(void)
1529 unsigned long mreg, psr;
1530 unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
1532 srmmu_modtype = SRMMU_INVAL_MOD;
1535 mreg = srmmu_get_mmureg(); psr = get_psr();
1536 mod_typ = (mreg & 0xf0000000) >> 28;
1537 mod_rev = (mreg & 0x0f000000) >> 24;
1538 psr_typ = (psr >> 28) & 0xf;
1539 psr_vers = (psr >> 24) & 0xf;
1541 /* First, check for sparc-leon. */
1542 if (sparc_cpu_model == sparc_leon) {
1547 /* Second, check for HyperSparc or Cypress. */
1551 /* UP or MP Hypersparc */
1563 prom_printf("Sparc-Linux Cypress support does not longer exit.\n");
1570 /* Now Fujitsu TurboSparc. It might happen that it is
1571 * in Swift emulation mode, so we will check later...
1573 if (psr_typ == 0 && psr_vers == 5) {
1578 /* Next check for Fujitsu Swift. */
1579 if (psr_typ == 0 && psr_vers == 4) {
1583 /* Look if it is not a TurboSparc emulating Swift... */
1584 cpunode = prom_getchild(prom_root_node);
1585 while ((cpunode = prom_getsibling(cpunode)) != 0) {
1586 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1587 if (!strcmp(node_str, "cpu")) {
1588 if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
1589 prom_getintdefault(cpunode, "psr-version", 1) == 5) {
1601 /* Now the Viking family of srmmu. */
1604 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
1609 /* Finally the Tsunami. */
1610 if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
1620 /* Local cross-calls. */
1621 static void smp_flush_page_for_dma(unsigned long page)
1623 xc1((smpfunc_t) local_ops->page_for_dma, page);
1624 local_ops->page_for_dma(page);
1627 static void smp_flush_cache_all(void)
1629 xc0((smpfunc_t) local_ops->cache_all);
1630 local_ops->cache_all();
1633 static void smp_flush_tlb_all(void)
1635 xc0((smpfunc_t) local_ops->tlb_all);
1636 local_ops->tlb_all();
1639 static void smp_flush_cache_mm(struct mm_struct *mm)
1641 if (mm->context != NO_CONTEXT) {
1643 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1644 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1645 if (!cpumask_empty(&cpu_mask))
1646 xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm);
1647 local_ops->cache_mm(mm);
1651 static void smp_flush_tlb_mm(struct mm_struct *mm)
1653 if (mm->context != NO_CONTEXT) {
1655 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1656 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1657 if (!cpumask_empty(&cpu_mask)) {
1658 xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm);
1659 if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
1660 cpumask_copy(mm_cpumask(mm),
1661 cpumask_of(smp_processor_id()));
1663 local_ops->tlb_mm(mm);
1667 static void smp_flush_cache_range(struct vm_area_struct *vma,
1668 unsigned long start,
1671 struct mm_struct *mm = vma->vm_mm;
1673 if (mm->context != NO_CONTEXT) {
1675 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1676 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1677 if (!cpumask_empty(&cpu_mask))
1678 xc3((smpfunc_t) local_ops->cache_range,
1679 (unsigned long) vma, start, end);
1680 local_ops->cache_range(vma, start, end);
1684 static void smp_flush_tlb_range(struct vm_area_struct *vma,
1685 unsigned long start,
1688 struct mm_struct *mm = vma->vm_mm;
1690 if (mm->context != NO_CONTEXT) {
1692 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1693 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1694 if (!cpumask_empty(&cpu_mask))
1695 xc3((smpfunc_t) local_ops->tlb_range,
1696 (unsigned long) vma, start, end);
1697 local_ops->tlb_range(vma, start, end);
1701 static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1703 struct mm_struct *mm = vma->vm_mm;
1705 if (mm->context != NO_CONTEXT) {
1707 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1708 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1709 if (!cpumask_empty(&cpu_mask))
1710 xc2((smpfunc_t) local_ops->cache_page,
1711 (unsigned long) vma, page);
1712 local_ops->cache_page(vma, page);
1716 static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1718 struct mm_struct *mm = vma->vm_mm;
1720 if (mm->context != NO_CONTEXT) {
1722 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1723 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1724 if (!cpumask_empty(&cpu_mask))
1725 xc2((smpfunc_t) local_ops->tlb_page,
1726 (unsigned long) vma, page);
1727 local_ops->tlb_page(vma, page);
1731 static void smp_flush_page_to_ram(unsigned long page)
1733 /* Current theory is that those who call this are the one's
1734 * who have just dirtied their cache with the pages contents
1735 * in kernel space, therefore we only run this on local cpu.
1737 * XXX This experiment failed, research further... -DaveM
1740 xc1((smpfunc_t) local_ops->page_to_ram, page);
1742 local_ops->page_to_ram(page);
1745 static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1748 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1749 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1750 if (!cpumask_empty(&cpu_mask))
1751 xc2((smpfunc_t) local_ops->sig_insns,
1752 (unsigned long) mm, insn_addr);
1753 local_ops->sig_insns(mm, insn_addr);
1756 static struct sparc32_cachetlb_ops smp_cachetlb_ops = {
1757 .cache_all = smp_flush_cache_all,
1758 .cache_mm = smp_flush_cache_mm,
1759 .cache_page = smp_flush_cache_page,
1760 .cache_range = smp_flush_cache_range,
1761 .tlb_all = smp_flush_tlb_all,
1762 .tlb_mm = smp_flush_tlb_mm,
1763 .tlb_page = smp_flush_tlb_page,
1764 .tlb_range = smp_flush_tlb_range,
1765 .page_to_ram = smp_flush_page_to_ram,
1766 .sig_insns = smp_flush_sig_insns,
1767 .page_for_dma = smp_flush_page_for_dma,
1771 /* Load up routines and constants for sun4m and sun4d mmu */
1772 void __init load_mmu(void)
1774 extern void ld_mmu_iommu(void);
1775 extern void ld_mmu_iounit(void);
1781 /* El switcheroo... */
1782 local_ops = sparc32_cachetlb_ops;
1784 if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) {
1785 smp_cachetlb_ops.tlb_all = local_ops->tlb_all;
1786 smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm;
1787 smp_cachetlb_ops.tlb_range = local_ops->tlb_range;
1788 smp_cachetlb_ops.tlb_page = local_ops->tlb_page;
1791 if (poke_srmmu == poke_viking) {
1792 /* Avoid unnecessary cross calls. */
1793 smp_cachetlb_ops.cache_all = local_ops->cache_all;
1794 smp_cachetlb_ops.cache_mm = local_ops->cache_mm;
1795 smp_cachetlb_ops.cache_range = local_ops->cache_range;
1796 smp_cachetlb_ops.cache_page = local_ops->cache_page;
1798 smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram;
1799 smp_cachetlb_ops.sig_insns = local_ops->sig_insns;
1800 smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma;
1803 /* It really is const after this point. */
1804 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1808 if (sparc_cpu_model == sun4d)
1813 if (sparc_cpu_model == sun4d)
1815 else if (sparc_cpu_model == sparc_leon)