arm64: add early_ioremap support

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / mm / mmu.c

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index eeecc9c8ed6860accacc36bb5f5aa0e6c7db7e58..6b7e89569a3a9ff8518e7c6ee856603f1e9fb93b 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -125,7 +125,7 @@ early_param("cachepolicy", early_cachepolicy);
 /*
  * Adjust the PMD section entries according to the CPU in use.
  */
-static void __init init_mem_pgprot(void)
+void __init init_mem_pgprot(void)
 {
        pteval_t default_pgprot;
        int i;
@@ -203,10 +203,18 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
        do {
                next = pmd_addr_end(addr, end);
                /* try section mapping first */
-               if (((addr | next | phys) & ~SECTION_MASK) == 0)
+               if (((addr | next | phys) & ~SECTION_MASK) == 0) {
+                       pmd_t old_pmd =*pmd;
                        set_pmd(pmd, __pmd(phys | prot_sect_kernel));
-               else
+                       /*
+                        * Check for previous table entries created during
+                        * boot (__create_page_tables) and flush them.
+                        */
+                       if (!pmd_none(old_pmd))
+                               flush_tlb_all();
+               } else {
                        alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys));
+               }
                phys += next - addr;
        } while (pmd++, addr = next, addr != end);
 }
@@ -252,50 +260,22 @@ static void __init create_mapping(phys_addr_t phys, unsigned long virt,
        } while (pgd++, addr = next, addr != end);
 }
 
-#ifdef CONFIG_EARLY_PRINTK
-/*
- * Create an early I/O mapping using the pgd/pmd entries already populated
- * in head.S as this function is called too early to allocated any memory. The
- * mapping size is 2MB with 4KB pages or 64KB or 64KB pages.
- */
-void __iomem * __init early_io_map(phys_addr_t phys, unsigned long virt)
+static void __init map_mem(void)
 {
-       unsigned long size, mask;
-       bool page64k = IS_ENABLED(CONFIG_ARM64_64K_PAGES);
-       pgd_t *pgd;
-       pud_t *pud;
-       pmd_t *pmd;
-       pte_t *pte;
+       struct memblock_region *reg;
+       phys_addr_t limit;
 
        /*
-        * No early pte entries with !ARM64_64K_PAGES configuration, so using
-        * sections (pmd).
+        * Temporarily limit the memblock range. We need to do this as
+        * create_mapping requires puds, pmds and ptes to be allocated from
+        * memory addressable from the initial direct kernel mapping.
+        *
+        * The initial direct kernel mapping, located at swapper_pg_dir,
+        * gives us PGDIR_SIZE memory starting from PHYS_OFFSET (which must be
+        * aligned to 2MB as per Documentation/arm64/booting.txt).
         */
-       size = page64k ? PAGE_SIZE : SECTION_SIZE;
-       mask = ~(size - 1);
-
-       pgd = pgd_offset_k(virt);
-       pud = pud_offset(pgd, virt);
-       if (pud_none(*pud))
-               return NULL;
-       pmd = pmd_offset(pud, virt);
-
-       if (page64k) {
-               if (pmd_none(*pmd))
-                       return NULL;
-               pte = pte_offset_kernel(pmd, virt);
-               set_pte(pte, __pte((phys & mask) | PROT_DEVICE_nGnRE));
-       } else {
-               set_pmd(pmd, __pmd((phys & mask) | PROT_SECT_DEVICE_nGnRE));
-       }
-
-       return (void __iomem *)((virt & mask) + (phys & ~mask));
-}
-#endif
-
-static void __init map_mem(void)
-{
-       struct memblock_region *reg;
+       limit = PHYS_OFFSET + PGDIR_SIZE;
+       memblock_set_current_limit(limit);
 
        /* map all the memory banks */
        for_each_memblock(memory, reg) {
@@ -305,8 +285,27 @@ static void __init map_mem(void)
                if (start >= end)
                        break;
 
+#ifndef CONFIG_ARM64_64K_PAGES
+               /*
+                * For the first memory bank align the start address and
+                * current memblock limit to prevent create_mapping() from
+                * allocating pte page tables from unmapped memory.
+                * When 64K pages are enabled, the pte page table for the
+                * first PGDIR_SIZE is already present in swapper_pg_dir.
+                */
+               if (start < limit)
+                       start = ALIGN(start, PMD_SIZE);
+               if (end < limit) {
+                       limit = end & PMD_MASK;
+                       memblock_set_current_limit(limit);
+               }
+#endif
+
                create_mapping(start, __phys_to_virt(start), end - start);
        }
+
+       /* Limit no longer required. */
+       memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 }
 
 /*
@@ -317,13 +316,6 @@ void __init paging_init(void)
 {
        void *zero_page;
 
-       /*
-        * Maximum PGDIR_SIZE addressable via the initial direct kernel
-        * mapping in swapper_pg_dir.
-        */
-       memblock_set_current_limit((PHYS_OFFSET & PGDIR_MASK) + PGDIR_SIZE);
-
-       init_mem_pgprot();
        map_mem();
 
        /*
@@ -339,7 +331,6 @@ void __init paging_init(void)
        bootmem_init();
 
        empty_zero_page = virt_to_page(zero_page);
-       __flush_dcache_page(empty_zero_page);
 
        /*
         * TTBR0 is only used for the identity mapping at this stage. Make it