BACKPORT: arm64: Introduce uaccess_{disable,enable} functionality based on TTBR0_EL1

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / include / asm / uaccess.h

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 8259eded53dca542336ed5a352be5c6961e5086d..955c6e58a624efb5ce949da0dfca4d87f66c0ef7 100644 (file)
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -19,6 +19,7 @@
 #define __ASM_UACCESS_H
 
 #include <asm/alternative.h>
+#include <asm/kernel-pgtable.h>
 #include <asm/sysreg.h>
 
 #ifndef __ASSEMBLY__
@@ -129,16 +130,71 @@ static inline void set_fs(mm_segment_t fs)
 /*
  * User access enabling/disabling.
  */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+static inline void __uaccess_ttbr0_disable(void)
+{
+       unsigned long ttbr;
+
+       /* reserved_ttbr0 placed at the end of swapper_pg_dir */
+       ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
+       write_sysreg(ttbr, ttbr0_el1);
+       isb();
+}
+
+static inline void __uaccess_ttbr0_enable(void)
+{
+       unsigned long flags;
+
+       /*
+        * Disable interrupts to avoid preemption between reading the 'ttbr0'
+        * variable and the MSR. A context switch could trigger an ASID
+        * roll-over and an update of 'ttbr0'.
+        */
+       local_irq_save(flags);
+       write_sysreg(current_thread_info()->ttbr0, ttbr0_el1);
+       isb();
+       local_irq_restore(flags);
+}
+
+static inline bool uaccess_ttbr0_disable(void)
+{
+       if (!system_uses_ttbr0_pan())
+               return false;
+       __uaccess_ttbr0_disable();
+       return true;
+}
+
+static inline bool uaccess_ttbr0_enable(void)
+{
+       if (!system_uses_ttbr0_pan())
+               return false;
+       __uaccess_ttbr0_enable();
+       return true;
+}
+#else
+static inline bool uaccess_ttbr0_disable(void)
+{
+       return false;
+}
+
+static inline bool uaccess_ttbr0_enable(void)
+{
+       return false;
+}
+#endif
+
 #define __uaccess_disable(alt)                                         \
 do {                                                                   \
-       asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,                  \
-                       CONFIG_ARM64_PAN));                             \
+       if (!uaccess_ttbr0_disable())                                   \
+               asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,          \
+                               CONFIG_ARM64_PAN));                     \
 } while (0)
 
 #define __uaccess_enable(alt)                                          \
 do {                                                                   \
-       asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,                  \
-                       CONFIG_ARM64_PAN));                             \
+       if (uaccess_ttbr0_enable())                                     \
+               asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,          \
+                               CONFIG_ARM64_PAN));                     \
 } while (0)
 
 static inline void uaccess_disable(void)
@@ -369,16 +425,56 @@ extern __must_check long strnlen_user(const char __user *str, long n);
 #include <asm/assembler.h>
 
 /*
- * User access enabling/disabling macros. These are no-ops when UAO is
- * present.
+ * User access enabling/disabling macros.
+ */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+       .macro  __uaccess_ttbr0_disable, tmp1
+       mrs     \tmp1, ttbr1_el1                // swapper_pg_dir
+       add     \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir
+       msr     ttbr0_el1, \tmp1                // set reserved TTBR0_EL1
+       isb
+       .endm
+
+       .macro  __uaccess_ttbr0_enable, tmp1
+       get_thread_info \tmp1
+       ldr     \tmp1, [\tmp1, #TSK_TI_TTBR0]   // load saved TTBR0_EL1
+       msr     ttbr0_el1, \tmp1                // set the non-PAN TTBR0_EL1
+       isb
+       .endm
+
+       .macro  uaccess_ttbr0_disable, tmp1
+alternative_if_not ARM64_HAS_PAN
+       __uaccess_ttbr0_disable \tmp1
+alternative_else_nop_endif
+       .endm
+
+       .macro  uaccess_ttbr0_enable, tmp1, tmp2
+alternative_if_not ARM64_HAS_PAN
+       save_and_disable_irq \tmp2              // avoid preemption
+       __uaccess_ttbr0_enable \tmp1
+       restore_irq \tmp2
+alternative_else_nop_endif
+       .endm
+#else
+       .macro  uaccess_ttbr0_disable, tmp1
+       .endm
+
+       .macro  uaccess_ttbr0_enable, tmp1, tmp2
+       .endm
+#endif
+
+/*
+ * These macros are no-ops when UAO is present.
  */
        .macro  uaccess_disable_not_uao, tmp1
+       uaccess_ttbr0_disable \tmp1
 alternative_if ARM64_ALT_PAN_NOT_UAO
        SET_PSTATE_PAN(1)
 alternative_else_nop_endif
        .endm
 
        .macro  uaccess_enable_not_uao, tmp1, tmp2
+       uaccess_ttbr0_enable \tmp1, \tmp2
 alternative_if ARM64_ALT_PAN_NOT_UAO
        SET_PSTATE_PAN(0)
 alternative_else_nop_endif