arm64: place initial page tables above the kernel

Currently we place swapper_pg_dir and idmap_pg_dir below the kernel
image, between PHYS_OFFSET and (PHYS_OFFSET + TEXT_OFFSET). However,
bootloaders may use portions of this memory below the kernel and we do
not parse the memory reservation list until after the MMU has been
enabled. As such we may clobber some memory a bootloader wishes to have
preserved.

To enable the use of all of this memory by bootloaders (when the
required memory reservations are communicated to the kernel) it is
necessary to move our initial page tables elsewhere. As we currently
have an effectively unbound requirement for memory at the end of the
kernel image for .bss, we can place the page tables here.

This patch moves the initial page table to the end of the kernel image,
after the BSS. As they do not consist of any initialised data they will
be stripped from the kernel Image as with the BSS. The BSS clearing
routine is updated to stop at __bss_stop rather than _end so as to not
clobber the page tables, and memory reservations made redundant by the
new organisation are removed.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <lauraa@codeaurora.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
(cherry picked from commit bd00cd5f8c8c3c282bb1e1eac6a6679a4f808091)
Signed-off-by: Mark Brown <broonie@linaro.org>
Conflicts:
	arch/arm64/mm/init.c

diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index 46bf66628b6a69f00e1c866c68e134cb32b3ff69..a6331e6a92b5cdec29ea3d31c5b529c6a8294f19 100644 (file)
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -31,6 +31,15 @@
 /* We do define AT_SYSINFO_EHDR but don't use the gate mechanism */
 #define __HAVE_ARCH_GATE_AREA          1
 
+/*
+ * The idmap and swapper page tables need some space reserved in the kernel
+ * image. The idmap only requires a pgd and a next level table to (section) map
+ * the kernel, while the swapper also maps the FDT and requires an additional
+ * table to map an early UART. See __create_page_tables for more information.
+ */
+#define SWAPPER_DIR_SIZE       (3 * PAGE_SIZE)
+#define IDMAP_DIR_SIZE         (2 * PAGE_SIZE)
+
 #ifndef __ASSEMBLY__
 
 #ifdef CONFIG_ARM64_64K_PAGES

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 30746fc16f8148e0da131b70dd90c825650f7aa0..39a8a83f1883b2388f64322683a62848eaf5f0a3 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -35,29 +35,17 @@
 #include <asm/page.h>
 #include <asm/virt.h>
 
-/*
- * swapper_pg_dir is the virtual address of the initial page table. We place
- * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has
- * 2 pages and is placed below swapper_pg_dir.
- */
 #define KERNEL_RAM_VADDR       (PAGE_OFFSET + TEXT_OFFSET)
 
 #if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000
 #error KERNEL_RAM_VADDR must start at 0xXXX80000
 #endif
 
-#define SWAPPER_DIR_SIZE       (3 * PAGE_SIZE)
-#define IDMAP_DIR_SIZE         (2 * PAGE_SIZE)
-
-       .globl  swapper_pg_dir
-       .equ    swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE
-
-       .globl  idmap_pg_dir
-       .equ    idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE
-
-       .macro  pgtbl, ttb0, ttb1, phys
-       add     \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE
-       sub     \ttb0, \ttb1, #IDMAP_DIR_SIZE
+       .macro  pgtbl, ttb0, ttb1, virt_to_phys
+       ldr     \ttb1, =swapper_pg_dir
+       ldr     \ttb0, =idmap_pg_dir
+       add     \ttb1, \ttb1, \virt_to_phys
+       add     \ttb0, \ttb0, \virt_to_phys
        .endm
 
 #ifdef CONFIG_ARM64_64K_PAGES
@@ -292,7 +280,7 @@ ENTRY(secondary_startup)
        mov     x23, x0                         // x23=current cpu_table
        cbz     x23, __error_p                  // invalid processor (x23=0)?
 
-       pgtbl   x25, x26, x24                   // x25=TTBR0, x26=TTBR1
+       pgtbl   x25, x26, x28                   // x25=TTBR0, x26=TTBR1
        ldr     x12, [x23, #CPU_INFO_SETUP]
        add     x12, x12, x28                   // __virt_to_phys
        blr     x12                             // initialise processor
@@ -406,7 +394,7 @@ ENDPROC(__calc_phys_offset)
  *   - pgd entry for fixed mappings (TTBR1)
  */
 __create_page_tables:
-       pgtbl   x25, x26, x24                   // idmap_pg_dir and swapper_pg_dir addresses
+       pgtbl   x25, x26, x28                   // idmap_pg_dir and swapper_pg_dir addresses
        mov     x27, lr
 
        /*
@@ -495,7 +483,7 @@ ENDPROC(__create_page_tables)
 __switch_data:
        .quad   __mmap_switched
        .quad   __bss_start                     // x6
-       .quad   _end                            // x7
+       .quad   __bss_stop                      // x7
        .quad   processor_id                    // x4
        .quad   __fdt_pointer                   // x5
        .quad   memstart_addr                   // x6

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 18a08e10357f2f2e2529210e78f2cb2f8afd76bc..ce2d97255ba950837c9946f032454070ee3344e3 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -90,6 +90,13 @@ SECTIONS
        _edata = .;
 
        BSS_SECTION(0, 0, 0)
+
+       . = ALIGN(PAGE_SIZE);
+       idmap_pg_dir = .;
+       . += IDMAP_DIR_SIZE;
+       swapper_pg_dir = .;
+       . += SWAPPER_DIR_SIZE;
+
        _end = .;
 
        STABS_DEBUG

diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 5c47534fe47d7cdba5eae76b2e1b303ed6462897..52806427e15d2cd7ee8cf145cfb132a34b6030ef 100644 (file)
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -136,7 +136,10 @@ void __init arm64_memblock_init(void)
 {
        u64 *reserve_map, base, size;
 
-       /* Register the kernel text, kernel data and initrd with memblock */
+       /*
+        * Register the kernel text, kernel data, initrd, and initial
+        * pagetables with memblock.
+        */
        memblock_reserve(__pa(_text), _end - _text);
 #ifdef CONFIG_BLK_DEV_INITRD
        if (phys_initrd_size) {
@@ -148,13 +151,6 @@ void __init arm64_memblock_init(void)
        }
 #endif
 
-       /*
-        * Reserve the page tables.  These are already in use,
-        * and can only be in node 0.
-        */
-       memblock_reserve(__pa(swapper_pg_dir), SWAPPER_DIR_SIZE);
-       memblock_reserve(__pa(idmap_pg_dir), IDMAP_DIR_SIZE);
-
        /* Reserve the dtb region */
        memblock_reserve(virt_to_phys(initial_boot_params),
                         be32_to_cpu(initial_boot_params->totalsize));