arm64: Add trampoline code for kretprobes

commit da6a91252ad98d49b49e83b76c1f032cdf6e5258 upstream.

The trampoline code is used by kretprobes to capture a return from a probed
function.  This is done by saving the registers, calling the handler, and
restoring the registers. The code then returns to the original saved caller
return address. It is necessary to do this directly instead of using a
software breakpoint because the code used in processing that breakpoint
could itself be kprobe'd and cause a problematic reentry into the debug
exception handler.

Signed-off-by: William Cohen <wcohen@redhat.com>
Signed-off-by: David A. Long <dave.long@linaro.org>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
[catalin.marinas@arm.com: removed unnecessary masking of the PSTATE bits]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h
index 79c9511612b51ae3fe75e260e6f0ce3abb3af4a7..61b49150dfa3046cb521c6fc54232d718e840460 100644 (file)
--- a/arch/arm64/include/asm/kprobes.h
+++ b/arch/arm64/include/asm/kprobes.h
@@ -56,5 +56,7 @@ int kprobe_exceptions_notify(struct notifier_block *self,
                             unsigned long val, void *data);
 int kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr);
 int kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr);
+void kretprobe_trampoline(void);
+void __kprobes *trampoline_probe_handler(struct pt_regs *regs);
 
 #endif /* _ARM_KPROBES_H */

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 087cf9a65359b5fac32a0fcb4c5fa2804ffd73bc..dd2925192405547c9dfd433680ddc9b9d7624008 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -49,6 +49,17 @@ int main(void)
   DEFINE(S_X5,                 offsetof(struct pt_regs, regs[5]));
   DEFINE(S_X6,                 offsetof(struct pt_regs, regs[6]));
   DEFINE(S_X7,                 offsetof(struct pt_regs, regs[7]));
+  DEFINE(S_X8,                 offsetof(struct pt_regs, regs[8]));
+  DEFINE(S_X10,                        offsetof(struct pt_regs, regs[10]));
+  DEFINE(S_X12,                        offsetof(struct pt_regs, regs[12]));
+  DEFINE(S_X14,                        offsetof(struct pt_regs, regs[14]));
+  DEFINE(S_X16,                        offsetof(struct pt_regs, regs[16]));
+  DEFINE(S_X18,                        offsetof(struct pt_regs, regs[18]));
+  DEFINE(S_X20,                        offsetof(struct pt_regs, regs[20]));
+  DEFINE(S_X22,                        offsetof(struct pt_regs, regs[22]));
+  DEFINE(S_X24,                        offsetof(struct pt_regs, regs[24]));
+  DEFINE(S_X26,                        offsetof(struct pt_regs, regs[26]));
+  DEFINE(S_X28,                        offsetof(struct pt_regs, regs[28]));
   DEFINE(S_LR,                 offsetof(struct pt_regs, regs[30]));
   DEFINE(S_SP,                 offsetof(struct pt_regs, sp));
 #ifdef CONFIG_COMPAT

diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile
index e184d00ebf01727c5054b55b6287c8b5063d6796..ce06312e3d349d862dbe10d7113fea9973ed1698 100644 (file)
--- a/arch/arm64/kernel/probes/Makefile
+++ b/arch/arm64/kernel/probes/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_KPROBES)          += kprobes.o decode-insn.o      \
+                                  kprobes_trampoline.o         \
                                   simulate-insn.o

diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index aff80b342382dc36e4d8867bcc1bd49d9d654b5c..5619041ac74b68431c50d3425987da20bb070000 100644 (file)
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -565,6 +565,11 @@ bool arch_within_kprobe_blacklist(unsigned long addr)
        return false;
 }
 
+void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
+{
+       return NULL;
+}
+
 int __init arch_init_kprobes(void)
 {
        return 0;

diff --git a/arch/arm64/kernel/probes/kprobes_trampoline.S b/arch/arm64/kernel/probes/kprobes_trampoline.S
new file mode 100644 (file)
index 0000000..5d6e7f1
--- /dev/null
+++ b/arch/arm64/kernel/probes/kprobes_trampoline.S
@@ -0,0 +1,81 @@
+/*
+ * trampoline entry and return code for kretprobes.
+ */
+
+#include <linux/linkage.h>
+#include <asm/asm-offsets.h>
+#include <asm/assembler.h>
+
+       .text
+
+       .macro  save_all_base_regs
+       stp x0, x1, [sp, #S_X0]
+       stp x2, x3, [sp, #S_X2]
+       stp x4, x5, [sp, #S_X4]
+       stp x6, x7, [sp, #S_X6]
+       stp x8, x9, [sp, #S_X8]
+       stp x10, x11, [sp, #S_X10]
+       stp x12, x13, [sp, #S_X12]
+       stp x14, x15, [sp, #S_X14]
+       stp x16, x17, [sp, #S_X16]
+       stp x18, x19, [sp, #S_X18]
+       stp x20, x21, [sp, #S_X20]
+       stp x22, x23, [sp, #S_X22]
+       stp x24, x25, [sp, #S_X24]
+       stp x26, x27, [sp, #S_X26]
+       stp x28, x29, [sp, #S_X28]
+       add x0, sp, #S_FRAME_SIZE
+       stp lr, x0, [sp, #S_LR]
+       /*
+        * Construct a useful saved PSTATE
+        */
+       mrs x0, nzcv
+       mrs x1, daif
+       orr x0, x0, x1
+       mrs x1, CurrentEL
+       orr x0, x0, x1
+       mrs x1, SPSel
+       orr x0, x0, x1
+       stp xzr, x0, [sp, #S_PC]
+       .endm
+
+       .macro  restore_all_base_regs
+       ldr x0, [sp, #S_PSTATE]
+       and x0, x0, #(PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT)
+       msr nzcv, x0
+       ldp x0, x1, [sp, #S_X0]
+       ldp x2, x3, [sp, #S_X2]
+       ldp x4, x5, [sp, #S_X4]
+       ldp x6, x7, [sp, #S_X6]
+       ldp x8, x9, [sp, #S_X8]
+       ldp x10, x11, [sp, #S_X10]
+       ldp x12, x13, [sp, #S_X12]
+       ldp x14, x15, [sp, #S_X14]
+       ldp x16, x17, [sp, #S_X16]
+       ldp x18, x19, [sp, #S_X18]
+       ldp x20, x21, [sp, #S_X20]
+       ldp x22, x23, [sp, #S_X22]
+       ldp x24, x25, [sp, #S_X24]
+       ldp x26, x27, [sp, #S_X26]
+       ldp x28, x29, [sp, #S_X28]
+       .endm
+
+ENTRY(kretprobe_trampoline)
+       sub sp, sp, #S_FRAME_SIZE
+
+       save_all_base_regs
+
+       mov x0, sp
+       bl trampoline_probe_handler
+       /*
+        * Replace trampoline address in lr with actual orig_ret_addr return
+        * address.
+        */
+       mov lr, x0
+
+       restore_all_base_regs
+
+       add sp, sp, #S_FRAME_SIZE
+       ret
+
+ENDPROC(kretprobe_trampoline)