arch/arm64/kernel/probes/kprobes.c

   1 /*
   2  * arch/arm64/kernel/probes/kprobes.c
   3  *
   4  * Kprobes support for ARM64
   5  *
   6  * Copyright (C) 2013 Linaro Limited.
   7  * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  */
  19 #include <linux/kernel.h>
  20 #include <linux/kprobes.h>
  21 #include <linux/module.h>
  22 #include <linux/slab.h>
  23 #include <linux/stop_machine.h>
  24 #include <linux/stringify.h>
  25 #include <asm/traps.h>
  26 #include <asm/ptrace.h>
  27 #include <asm/cacheflush.h>
  28 #include <asm/debug-monitors.h>
  29 #include <asm/system_misc.h>
  30 #include <asm/insn.h>
  31 #include <asm/uaccess.h>
  32 #include <asm/irq.h>
  33 #include <asm-generic/sections.h>
  34
  35 #include "decode-insn.h"
  36
  37 #define MIN_STACK_SIZE(addr)    min((unsigned long)MAX_STACK_SIZE,      \
  38         (unsigned long)current_thread_info() + THREAD_START_SP - (addr))
  39
  40 void jprobe_return_break(void);
  41
  42 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
  43 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  44
  45 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
  46 {
  47         /* prepare insn slot */
  48         p->ainsn.insn[0] = cpu_to_le32(p->opcode);
  49
  50         flush_icache_range((uintptr_t) (p->ainsn.insn),
  51                            (uintptr_t) (p->ainsn.insn) +
  52                            MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
  53
  54         /*
  55          * Needs restoring of return address after stepping xol.
  56          */
  57         p->ainsn.restore = (unsigned long) p->addr +
  58           sizeof(kprobe_opcode_t);
  59 }
  60
  61 int __kprobes arch_prepare_kprobe(struct kprobe *p)
  62 {
  63         unsigned long probe_addr = (unsigned long)p->addr;
  64         extern char __start_rodata[];
  65         extern char __end_rodata[];
  66
  67         if (probe_addr & 0x3)
  68                 return -EINVAL;
  69
  70         /* copy instruction */
  71         p->opcode = le32_to_cpu(*p->addr);
  72
  73         if (in_exception_text(probe_addr))
  74                 return -EINVAL;
  75         if (probe_addr >= (unsigned long) __start_rodata &&
  76             probe_addr <= (unsigned long) __end_rodata)
  77                 return -EINVAL;
  78
  79         /* decode instruction */
  80         switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) {
  81         case INSN_REJECTED:     /* insn not supported */
  82                 return -EINVAL;
  83
  84         case INSN_GOOD: /* instruction uses slot */
  85                 p->ainsn.insn = get_insn_slot();
  86                 if (!p->ainsn.insn)
  87                         return -ENOMEM;
  88                 break;
  89         };
  90
  91         /* prepare the instruction */
  92         arch_prepare_ss_slot(p);
  93
  94         return 0;
  95 }
  96
  97 static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
  98 {
  99         void *addrs[1];
 100         u32 insns[1];
 101
 102         addrs[0] = (void *)addr;
 103         insns[0] = (u32)opcode;
 104
 105         return aarch64_insn_patch_text(addrs, insns, 1);
 106 }
 107
 108 /* arm kprobe: install breakpoint in text */
 109 void __kprobes arch_arm_kprobe(struct kprobe *p)
 110 {
 111         patch_text(p->addr, BRK64_OPCODE_KPROBES);
 112 }
 113
 114 /* disarm kprobe: remove breakpoint from text */
 115 void __kprobes arch_disarm_kprobe(struct kprobe *p)
 116 {
 117         patch_text(p->addr, p->opcode);
 118 }
 119
 120 void __kprobes arch_remove_kprobe(struct kprobe *p)
 121 {
 122         if (p->ainsn.insn) {
 123                 free_insn_slot(p->ainsn.insn, 0);
 124                 p->ainsn.insn = NULL;
 125         }
 126 }
 127
 128 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
 129 {
 130         kcb->prev_kprobe.kp = kprobe_running();
 131         kcb->prev_kprobe.status = kcb->kprobe_status;
 132 }
 133
 134 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
 135 {
 136         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 137         kcb->kprobe_status = kcb->prev_kprobe.status;
 138 }
 139
 140 static void __kprobes set_current_kprobe(struct kprobe *p)
 141 {
 142         __this_cpu_write(current_kprobe, p);
 143 }
 144
 145 /*
 146  * The D-flag (Debug mask) is set (masked) upon debug exception entry.
 147  * Kprobes needs to clear (unmask) D-flag -ONLY- in case of recursive
 148  * probe i.e. when probe hit from kprobe handler context upon
 149  * executing the pre/post handlers. In this case we return with
 150  * D-flag clear so that single-stepping can be carried-out.
 151  *
 152  * Leave D-flag set in all other cases.
 153  */
 154 static void __kprobes
 155 spsr_set_debug_flag(struct pt_regs *regs, int mask)
 156 {
 157         unsigned long spsr = regs->pstate;
 158
 159         if (mask)
 160                 spsr |= PSR_D_BIT;
 161         else
 162                 spsr &= ~PSR_D_BIT;
 163
 164         regs->pstate = spsr;
 165 }
 166
 167 /*
 168  * Interrupts need to be disabled before single-step mode is set, and not
 169  * reenabled until after single-step mode ends.
 170  * Without disabling interrupt on local CPU, there is a chance of
 171  * interrupt occurrence in the period of exception return and  start of
 172  * out-of-line single-step, that result in wrongly single stepping
 173  * into the interrupt handler.
 174  */
 175 static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
 176                                                 struct pt_regs *regs)
 177 {
 178         kcb->saved_irqflag = regs->pstate;
 179         regs->pstate |= PSR_I_BIT;
 180 }
 181
 182 static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
 183                                                 struct pt_regs *regs)
 184 {
 185         if (kcb->saved_irqflag & PSR_I_BIT)
 186                 regs->pstate |= PSR_I_BIT;
 187         else
 188                 regs->pstate &= ~PSR_I_BIT;
 189 }
 190
 191 static void __kprobes
 192 set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr)
 193 {
 194         kcb->ss_ctx.ss_pending = true;
 195         kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t);
 196 }
 197
 198 static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb)
 199 {
 200         kcb->ss_ctx.ss_pending = false;
 201         kcb->ss_ctx.match_addr = 0;
 202 }
 203
 204 static void __kprobes setup_singlestep(struct kprobe *p,
 205                                        struct pt_regs *regs,
 206                                        struct kprobe_ctlblk *kcb, int reenter)
 207 {
 208         unsigned long slot;
 209
 210         if (reenter) {
 211                 save_previous_kprobe(kcb);
 212                 set_current_kprobe(p);
 213                 kcb->kprobe_status = KPROBE_REENTER;
 214         } else {
 215                 kcb->kprobe_status = KPROBE_HIT_SS;
 216         }
 217
 218         BUG_ON(!p->ainsn.insn);
 219
 220         /* prepare for single stepping */
 221         slot = (unsigned long)p->ainsn.insn;
 222
 223         set_ss_context(kcb, slot);      /* mark pending ss */
 224
 225         if (kcb->kprobe_status == KPROBE_REENTER)
 226                 spsr_set_debug_flag(regs, 0);
 227
 228         /* IRQs and single stepping do not mix well. */
 229         kprobes_save_local_irqflag(kcb, regs);
 230         kernel_enable_single_step(regs);
 231         instruction_pointer_set(regs, slot);
 232 }
 233
 234 static int __kprobes reenter_kprobe(struct kprobe *p,
 235                                     struct pt_regs *regs,
 236                                     struct kprobe_ctlblk *kcb)
 237 {
 238         switch (kcb->kprobe_status) {
 239         case KPROBE_HIT_SSDONE:
 240         case KPROBE_HIT_ACTIVE:
 241                 kprobes_inc_nmissed_count(p);
 242                 setup_singlestep(p, regs, kcb, 1);
 243                 break;
 244         case KPROBE_HIT_SS:
 245         case KPROBE_REENTER:
 246                 pr_warn("Unrecoverable kprobe detected at %p.\n", p->addr);
 247                 dump_kprobe(p);
 248                 BUG();
 249                 break;
 250         default:
 251                 WARN_ON(1);
 252                 return 0;
 253         }
 254
 255         return 1;
 256 }
 257
 258 static void __kprobes
 259 post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
 260 {
 261         struct kprobe *cur = kprobe_running();
 262
 263         if (!cur)
 264                 return;
 265
 266         /* return addr restore if non-branching insn */
 267         if (cur->ainsn.restore != 0)
 268                 instruction_pointer_set(regs, cur->ainsn.restore);
 269
 270         /* restore back original saved kprobe variables and continue */
 271         if (kcb->kprobe_status == KPROBE_REENTER) {
 272                 restore_previous_kprobe(kcb);
 273                 return;
 274         }
 275         /* call post handler */
 276         kcb->kprobe_status = KPROBE_HIT_SSDONE;
 277         if (cur->post_handler)  {
 278                 /* post_handler can hit breakpoint and single step
 279                  * again, so we enable D-flag for recursive exception.
 280                  */
 281                 cur->post_handler(cur, regs, 0);
 282         }
 283
 284         reset_current_kprobe();
 285 }
 286
 287 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
 288 {
 289         struct kprobe *cur = kprobe_running();
 290         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 291
 292         switch (kcb->kprobe_status) {
 293         case KPROBE_HIT_SS:
 294         case KPROBE_REENTER:
 295                 /*
 296                  * We are here because the instruction being single
 297                  * stepped caused a page fault. We reset the current
 298                  * kprobe and the ip points back to the probe address
 299                  * and allow the page fault handler to continue as a
 300                  * normal page fault.
 301                  */
 302                 instruction_pointer_set(regs, (unsigned long) cur->addr);
 303                 if (!instruction_pointer(regs))
 304                         BUG();
 305
 306                 kernel_disable_single_step();
 307                 if (kcb->kprobe_status == KPROBE_REENTER)
 308                         spsr_set_debug_flag(regs, 1);
 309
 310                 if (kcb->kprobe_status == KPROBE_REENTER)
 311                         restore_previous_kprobe(kcb);
 312                 else
 313                         reset_current_kprobe();
 314
 315                 break;
 316         case KPROBE_HIT_ACTIVE:
 317         case KPROBE_HIT_SSDONE:
 318                 /*
 319                  * We increment the nmissed count for accounting,
 320                  * we can also use npre/npostfault count for accounting
 321                  * these specific fault cases.
 322                  */
 323                 kprobes_inc_nmissed_count(cur);
 324
 325                 /*
 326                  * We come here because instructions in the pre/post
 327                  * handler caused the page_fault, this could happen
 328                  * if handler tries to access user space by
 329                  * copy_from_user(), get_user() etc. Let the
 330                  * user-specified handler try to fix it first.
 331                  */
 332                 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr))
 333                         return 1;
 334
 335                 /*
 336                  * In case the user-specified fault handler returned
 337                  * zero, try to fix up.
 338                  */
 339                 if (fixup_exception(regs))
 340                         return 1;
 341         }
 342         return 0;
 343 }
 344
 345 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
 346                                        unsigned long val, void *data)
 347 {
 348         return NOTIFY_DONE;
 349 }
 350
 351 static void __kprobes kprobe_handler(struct pt_regs *regs)
 352 {
 353         struct kprobe *p, *cur_kprobe;
 354         struct kprobe_ctlblk *kcb;
 355         unsigned long addr = instruction_pointer(regs);
 356
 357         kcb = get_kprobe_ctlblk();
 358         cur_kprobe = kprobe_running();
 359
 360         p = get_kprobe((kprobe_opcode_t *) addr);
 361
 362         if (p) {
 363                 if (cur_kprobe) {
 364                         if (reenter_kprobe(p, regs, kcb))
 365                                 return;
 366                 } else {
 367                         /* Probe hit */
 368                         set_current_kprobe(p);
 369                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 370
 371                         /*
 372                          * If we have no pre-handler or it returned 0, we
 373                          * continue with normal processing.  If we have a
 374                          * pre-handler and it returned non-zero, it prepped
 375                          * for calling the break_handler below on re-entry,
 376                          * so get out doing nothing more here.
 377                          *
 378                          * pre_handler can hit a breakpoint and can step thru
 379                          * before return, keep PSTATE D-flag enabled until
 380                          * pre_handler return back.
 381                          */
 382                         if (!p->pre_handler || !p->pre_handler(p, regs)) {
 383                                 setup_singlestep(p, regs, kcb, 0);
 384                                 return;
 385                         }
 386                 }
 387         } else if ((le32_to_cpu(*(kprobe_opcode_t *) addr) ==
 388             BRK64_OPCODE_KPROBES) && cur_kprobe) {
 389                 /* We probably hit a jprobe.  Call its break handler. */
 390                 if (cur_kprobe->break_handler  &&
 391                      cur_kprobe->break_handler(cur_kprobe, regs)) {
 392                         setup_singlestep(cur_kprobe, regs, kcb, 0);
 393                         return;
 394                 }
 395         }
 396         /*
 397          * The breakpoint instruction was removed right
 398          * after we hit it.  Another cpu has removed
 399          * either a probepoint or a debugger breakpoint
 400          * at this address.  In either case, no further
 401          * handling of this interrupt is appropriate.
 402          * Return back to original instruction, and continue.
 403          */
 404 }
 405
 406 static int __kprobes
 407 kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr)
 408 {
 409         if ((kcb->ss_ctx.ss_pending)
 410             && (kcb->ss_ctx.match_addr == addr)) {
 411                 clear_ss_context(kcb);  /* clear pending ss */
 412                 return DBG_HOOK_HANDLED;
 413         }
 414         /* not ours, kprobes should ignore it */
 415         return DBG_HOOK_ERROR;
 416 }
 417
 418 int __kprobes
 419 kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr)
 420 {
 421         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 422         int retval;
 423
 424         /* return error if this is not our step */
 425         retval = kprobe_ss_hit(kcb, instruction_pointer(regs));
 426
 427         if (retval == DBG_HOOK_HANDLED) {
 428                 kprobes_restore_local_irqflag(kcb, regs);
 429                 kernel_disable_single_step();
 430
 431                 if (kcb->kprobe_status == KPROBE_REENTER)
 432                         spsr_set_debug_flag(regs, 1);
 433
 434                 post_kprobe_handler(kcb, regs);
 435         }
 436
 437         return retval;
 438 }
 439
 440 int __kprobes
 441 kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr)
 442 {
 443         kprobe_handler(regs);
 444         return DBG_HOOK_HANDLED;
 445 }
 446
 447 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
 448 {
 449         struct jprobe *jp = container_of(p, struct jprobe, kp);
 450         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 451         long stack_ptr = kernel_stack_pointer(regs);
 452
 453         kcb->jprobe_saved_regs = *regs;
 454         /*
 455          * As Linus pointed out, gcc assumes that the callee
 456          * owns the argument space and could overwrite it, e.g.
 457          * tailcall optimization. So, to be absolutely safe
 458          * we also save and restore enough stack bytes to cover
 459          * the argument area.
 460          */
 461         memcpy(kcb->jprobes_stack, (void *)stack_ptr,
 462                MIN_STACK_SIZE(stack_ptr));
 463
 464         instruction_pointer_set(regs, (unsigned long) jp->entry);
 465         preempt_disable();
 466         pause_graph_tracing();
 467         return 1;
 468 }
 469
 470 void __kprobes jprobe_return(void)
 471 {
 472         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 473
 474         /*
 475          * Jprobe handler return by entering break exception,
 476          * encoded same as kprobe, but with following conditions
 477          * -a magic number in x0 to identify from rest of other kprobes.
 478          * -restore stack addr to original saved pt_regs
 479          */
 480         asm volatile ("ldr x0, [%0]\n\t"
 481                       "mov sp, x0\n\t"
 482                       ".globl jprobe_return_break\n\t"
 483                       "jprobe_return_break:\n\t"
 484                       "brk %1\n\t"
 485                       :
 486                       : "r"(&kcb->jprobe_saved_regs.sp),
 487                       "I"(BRK64_ESR_KPROBES)
 488                       : "memory");
 489 }
 490
 491 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
 492 {
 493         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 494         long stack_addr = kcb->jprobe_saved_regs.sp;
 495         long orig_sp = kernel_stack_pointer(regs);
 496         struct jprobe *jp = container_of(p, struct jprobe, kp);
 497
 498         if (instruction_pointer(regs) != (u64) jprobe_return_break)
 499                 return 0;
 500
 501         if (orig_sp != stack_addr) {
 502                 struct pt_regs *saved_regs =
 503                     (struct pt_regs *)kcb->jprobe_saved_regs.sp;
 504                 pr_err("current sp %lx does not match saved sp %lx\n",
 505                        orig_sp, stack_addr);
 506                 pr_err("Saved registers for jprobe %p\n", jp);
 507                 show_regs(saved_regs);
 508                 pr_err("Current registers\n");
 509                 show_regs(regs);
 510                 BUG();
 511         }
 512         unpause_graph_tracing();
 513         *regs = kcb->jprobe_saved_regs;
 514         memcpy((void *)stack_addr, kcb->jprobes_stack,
 515                MIN_STACK_SIZE(stack_addr));
 516         preempt_enable_no_resched();
 517         return 1;
 518 }
 519
 520 bool arch_within_kprobe_blacklist(unsigned long addr)
 521 {
 522         extern char __idmap_text_start[], __idmap_text_end[];
 523
 524         if ((addr >= (unsigned long)__kprobes_text_start &&
 525             addr < (unsigned long)__kprobes_text_end) ||
 526             (addr >= (unsigned long)__entry_text_start &&
 527             addr < (unsigned long)__entry_text_end) ||
 528             (addr >= (unsigned long)__idmap_text_start &&
 529             addr < (unsigned long)__idmap_text_end) ||
 530             !!search_exception_tables(addr))
 531                 return true;
 532
 533
 534         return false;
 535 }
 536
 537 int __init arch_init_kprobes(void)
 538 {
 539         return 0;
 540 }