Merge remote-tracking branch 'origin/v4.4/topic/kprobes64' into linux-linaro-lsk...

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / mm / fault.c

diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 92ddac1e8ca2f35b8f24028514bb761aed3591a4..6c16e4963b39092a30ad444f1862d39d270538ae 100644 (file)
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -41,6 +41,28 @@
 
 static const char *fault_name(unsigned int esr);
 
+#ifdef CONFIG_KPROBES
+static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
+{
+       int ret = 0;
+
+       /* kprobe_running() needs smp_processor_id() */
+       if (!user_mode(regs)) {
+               preempt_disable();
+               if (kprobe_running() && kprobe_fault_handler(regs, esr))
+                       ret = 1;
+               preempt_enable();
+       }
+
+       return ret;
+}
+#else
+static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
+{
+       return 0;
+}
+#endif
+
 /*
  * Dump out the page tables associated with 'addr' in mm 'mm'.
  */
@@ -81,6 +103,56 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
        printk("\n");
 }
 
+#ifdef CONFIG_ARM64_HW_AFDBM
+/*
+ * This function sets the access flags (dirty, accessed), as well as write
+ * permission, and only to a more permissive setting.
+ *
+ * It needs to cope with hardware update of the accessed/dirty state by other
+ * agents in the system and can safely skip the __sync_icache_dcache() call as,
+ * like set_pte_at(), the PTE is never changed from no-exec to exec here.
+ *
+ * Returns whether or not the PTE actually changed.
+ */
+int ptep_set_access_flags(struct vm_area_struct *vma,
+                         unsigned long address, pte_t *ptep,
+                         pte_t entry, int dirty)
+{
+       pteval_t old_pteval;
+       unsigned int tmp;
+
+       if (pte_same(*ptep, entry))
+               return 0;
+
+       /* only preserve the access flags and write permission */
+       pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY;
+
+       /*
+        * PTE_RDONLY is cleared by default in the asm below, so set it in
+        * back if necessary (read-only or clean PTE).
+        */
+       if (!pte_write(entry) || !pte_sw_dirty(entry))
+               pte_val(entry) |= PTE_RDONLY;
+
+       /*
+        * Setting the flags must be done atomically to avoid racing with the
+        * hardware update of the access/dirty state.
+        */
+       asm volatile("//        ptep_set_access_flags\n"
+       "       prfm    pstl1strm, %2\n"
+       "1:     ldxr    %0, %2\n"
+       "       and     %0, %0, %3              // clear PTE_RDONLY\n"
+       "       orr     %0, %0, %4              // set flags\n"
+       "       stxr    %w1, %0, %2\n"
+       "       cbnz    %w1, 1b\n"
+       : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
+       : "L" (~PTE_RDONLY), "r" (pte_val(entry)));
+
+       flush_tlb_fix_spurious_fault(vma, address);
+       return 1;
+}
+#endif
+
 /*
  * The kernel tried to access some page that wasn't present.
  */
@@ -192,6 +264,14 @@ out:
        return fault;
 }
 
+static inline int permission_fault(unsigned int esr)
+{
+       unsigned int ec       = (esr & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT;
+       unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
+
+       return (ec == ESR_ELx_EC_DABT_CUR && fsc_type == ESR_ELx_FSC_PERM);
+}
+
 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
                                   struct pt_regs *regs)
 {
@@ -201,6 +281,9 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
        unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
        unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
+       if (notify_page_fault(regs, esr))
+               return 0;
+
        tsk = current;
        mm  = tsk->mm;
 
@@ -225,12 +308,13 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
                mm_flags |= FAULT_FLAG_WRITE;
        }
 
-       /*
-        * PAN bit set implies the fault happened in kernel space, but not
-        * in the arch's user access functions.
-        */
-       if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT))
-               goto no_context;
+       if (permission_fault(esr) && (addr < USER_DS)) {
+               if (get_fs() == KERNEL_DS)
+                       die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
+
+               if (!search_exception_tables(regs->pc))
+                       die("Accessing user space memory outside uaccess.h routines", regs, esr);
+       }
 
        /*
         * As per x86, we may deadlock here. However, since the kernel only
@@ -554,6 +638,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
 
        return 0;
 }
+NOKPROBE_SYMBOL(do_debug_exception);
 
 #ifdef CONFIG_ARM64_PAN
 void cpu_enable_pan(void *__unused)
@@ -561,3 +646,16 @@ void cpu_enable_pan(void *__unused)
        config_sctlr_el1(SCTLR_EL1_SPAN, 0);
 }
 #endif /* CONFIG_ARM64_PAN */
+
+#ifdef CONFIG_ARM64_UAO
+/*
+ * Kernel threads have fs=KERNEL_DS by default, and don't need to call
+ * set_fs(), devtmpfs in particular relies on this behaviour.
+ * We need to enable the feature at runtime (instead of adding it to
+ * PSR_MODE_EL1h) as the feature may not be implemented by the cpu.
+ */
+void cpu_enable_uao(void *__unused)
+{
+       asm(SET_PSTATE_UAO(1));
+}
+#endif /* CONFIG_ARM64_UAO */