arch/arm64/kernel/probes/kprobes.c

   1 /*
   2  * arch/arm64/kernel/probes/kprobes.c
   3  *
   4  * Kprobes support for ARM64
   5  *
   6  * Copyright (C) 2013 Linaro Limited.
   7  * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  */
  19 #include <linux/kernel.h>
  20 #include <linux/kprobes.h>
  21 #include <linux/module.h>
  22 #include <linux/slab.h>
  23 #include <linux/stop_machine.h>
  24 #include <linux/stringify.h>
  25 #include <asm/traps.h>
  26 #include <asm/ptrace.h>
  27 #include <asm/cacheflush.h>
  28 #include <asm/debug-monitors.h>
  29 #include <asm/system_misc.h>
  30 #include <asm/insn.h>
  31 #include <asm/uaccess.h>
  32 #include <asm/irq.h>
  33 #include <asm-generic/sections.h>
  34
  35 #include "decode-insn.h"
  36
  37 #define MIN_STACK_SIZE(addr)    min((unsigned long)MAX_STACK_SIZE,      \
  38         (unsigned long)current_thread_info() + THREAD_START_SP - (addr))
  39
  40 void jprobe_return_break(void);
  41
  42 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
  43 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  44
  45 static void __kprobes
  46 post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
  47
  48 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
  49 {
  50         /* prepare insn slot */
  51         p->ainsn.insn[0] = cpu_to_le32(p->opcode);
  52
  53         flush_icache_range((uintptr_t) (p->ainsn.insn),
  54                            (uintptr_t) (p->ainsn.insn) +
  55                            MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
  56
  57         /*
  58          * Needs restoring of return address after stepping xol.
  59          */
  60         p->ainsn.restore = (unsigned long) p->addr +
  61           sizeof(kprobe_opcode_t);
  62 }
  63
  64 static void __kprobes arch_prepare_simulate(struct kprobe *p)
  65 {
  66         /* This instructions is not executed xol. No need to adjust the PC */
  67         p->ainsn.restore = 0;
  68 }
  69
  70 static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
  71 {
  72         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
  73
  74         if (p->ainsn.handler)
  75                 p->ainsn.handler((u32)p->opcode, (long)p->addr, regs);
  76
  77         /* single step simulated, now go for post processing */
  78         post_kprobe_handler(kcb, regs);
  79 }
  80
  81 int __kprobes arch_prepare_kprobe(struct kprobe *p)
  82 {
  83         unsigned long probe_addr = (unsigned long)p->addr;
  84         extern char __start_rodata[];
  85         extern char __end_rodata[];
  86
  87         if (probe_addr & 0x3)
  88                 return -EINVAL;
  89
  90         /* copy instruction */
  91         p->opcode = le32_to_cpu(*p->addr);
  92
  93         if (in_exception_text(probe_addr))
  94                 return -EINVAL;
  95         if (probe_addr >= (unsigned long) __start_rodata &&
  96             probe_addr <= (unsigned long) __end_rodata)
  97                 return -EINVAL;
  98
  99         /* decode instruction */
 100         switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) {
 101         case INSN_REJECTED:     /* insn not supported */
 102                 return -EINVAL;
 103
 104         case INSN_GOOD_NO_SLOT: /* insn need simulation */
 105                 p->ainsn.insn = NULL;
 106                 break;
 107
 108         case INSN_GOOD: /* instruction uses slot */
 109                 p->ainsn.insn = get_insn_slot();
 110                 if (!p->ainsn.insn)
 111                         return -ENOMEM;
 112                 break;
 113         };
 114
 115         /* prepare the instruction */
 116         if (p->ainsn.insn)
 117                 arch_prepare_ss_slot(p);
 118         else
 119                 arch_prepare_simulate(p);
 120
 121         return 0;
 122 }
 123
 124 static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
 125 {
 126         void *addrs[1];
 127         u32 insns[1];
 128
 129         addrs[0] = (void *)addr;
 130         insns[0] = (u32)opcode;
 131
 132         return aarch64_insn_patch_text(addrs, insns, 1);
 133 }
 134
 135 /* arm kprobe: install breakpoint in text */
 136 void __kprobes arch_arm_kprobe(struct kprobe *p)
 137 {
 138         patch_text(p->addr, BRK64_OPCODE_KPROBES);
 139 }
 140
 141 /* disarm kprobe: remove breakpoint from text */
 142 void __kprobes arch_disarm_kprobe(struct kprobe *p)
 143 {
 144         patch_text(p->addr, p->opcode);
 145 }
 146
 147 void __kprobes arch_remove_kprobe(struct kprobe *p)
 148 {
 149         if (p->ainsn.insn) {
 150                 free_insn_slot(p->ainsn.insn, 0);
 151                 p->ainsn.insn = NULL;
 152         }
 153 }
 154
 155 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
 156 {
 157         kcb->prev_kprobe.kp = kprobe_running();
 158         kcb->prev_kprobe.status = kcb->kprobe_status;
 159 }
 160
 161 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
 162 {
 163         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 164         kcb->kprobe_status = kcb->prev_kprobe.status;
 165 }
 166
 167 static void __kprobes set_current_kprobe(struct kprobe *p)
 168 {
 169         __this_cpu_write(current_kprobe, p);
 170 }
 171
 172 /*
 173  * The D-flag (Debug mask) is set (masked) upon debug exception entry.
 174  * Kprobes needs to clear (unmask) D-flag -ONLY- in case of recursive
 175  * probe i.e. when probe hit from kprobe handler context upon
 176  * executing the pre/post handlers. In this case we return with
 177  * D-flag clear so that single-stepping can be carried-out.
 178  *
 179  * Leave D-flag set in all other cases.
 180  */
 181 static void __kprobes
 182 spsr_set_debug_flag(struct pt_regs *regs, int mask)
 183 {
 184         unsigned long spsr = regs->pstate;
 185
 186         if (mask)
 187                 spsr |= PSR_D_BIT;
 188         else
 189                 spsr &= ~PSR_D_BIT;
 190
 191         regs->pstate = spsr;
 192 }
 193
 194 /*
 195  * Interrupts need to be disabled before single-step mode is set, and not
 196  * reenabled until after single-step mode ends.
 197  * Without disabling interrupt on local CPU, there is a chance of
 198  * interrupt occurrence in the period of exception return and  start of
 199  * out-of-line single-step, that result in wrongly single stepping
 200  * into the interrupt handler.
 201  */
 202 static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
 203                                                 struct pt_regs *regs)
 204 {
 205         kcb->saved_irqflag = regs->pstate;
 206         regs->pstate |= PSR_I_BIT;
 207 }
 208
 209 static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
 210                                                 struct pt_regs *regs)
 211 {
 212         if (kcb->saved_irqflag & PSR_I_BIT)
 213                 regs->pstate |= PSR_I_BIT;
 214         else
 215                 regs->pstate &= ~PSR_I_BIT;
 216 }
 217
 218 static void __kprobes
 219 set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr)
 220 {
 221         kcb->ss_ctx.ss_pending = true;
 222         kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t);
 223 }
 224
 225 static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb)
 226 {
 227         kcb->ss_ctx.ss_pending = false;
 228         kcb->ss_ctx.match_addr = 0;
 229 }
 230
 231 static void __kprobes setup_singlestep(struct kprobe *p,
 232                                        struct pt_regs *regs,
 233                                        struct kprobe_ctlblk *kcb, int reenter)
 234 {
 235         unsigned long slot;
 236
 237         if (reenter) {
 238                 save_previous_kprobe(kcb);
 239                 set_current_kprobe(p);
 240                 kcb->kprobe_status = KPROBE_REENTER;
 241         } else {
 242                 kcb->kprobe_status = KPROBE_HIT_SS;
 243         }
 244
 245
 246         if (p->ainsn.insn) {
 247                 /* prepare for single stepping */
 248                 slot = (unsigned long)p->ainsn.insn;
 249
 250                 set_ss_context(kcb, slot);      /* mark pending ss */
 251
 252                 if (kcb->kprobe_status == KPROBE_REENTER)
 253                         spsr_set_debug_flag(regs, 0);
 254
 255                 /* IRQs and single stepping do not mix well. */
 256                 kprobes_save_local_irqflag(kcb, regs);
 257                 kernel_enable_single_step(regs);
 258                 instruction_pointer_set(regs, slot);
 259         } else {
 260                 /* insn simulation */
 261                 arch_simulate_insn(p, regs);
 262         }
 263 }
 264
 265 static int __kprobes reenter_kprobe(struct kprobe *p,
 266                                     struct pt_regs *regs,
 267                                     struct kprobe_ctlblk *kcb)
 268 {
 269         switch (kcb->kprobe_status) {
 270         case KPROBE_HIT_SSDONE:
 271         case KPROBE_HIT_ACTIVE:
 272                 kprobes_inc_nmissed_count(p);
 273                 setup_singlestep(p, regs, kcb, 1);
 274                 break;
 275         case KPROBE_HIT_SS:
 276         case KPROBE_REENTER:
 277                 pr_warn("Unrecoverable kprobe detected at %p.\n", p->addr);
 278                 dump_kprobe(p);
 279                 BUG();
 280                 break;
 281         default:
 282                 WARN_ON(1);
 283                 return 0;
 284         }
 285
 286         return 1;
 287 }
 288
 289 static void __kprobes
 290 post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
 291 {
 292         struct kprobe *cur = kprobe_running();
 293
 294         if (!cur)
 295                 return;
 296
 297         /* return addr restore if non-branching insn */
 298         if (cur->ainsn.restore != 0)
 299                 instruction_pointer_set(regs, cur->ainsn.restore);
 300
 301         /* restore back original saved kprobe variables and continue */
 302         if (kcb->kprobe_status == KPROBE_REENTER) {
 303                 restore_previous_kprobe(kcb);
 304                 return;
 305         }
 306         /* call post handler */
 307         kcb->kprobe_status = KPROBE_HIT_SSDONE;
 308         if (cur->post_handler)  {
 309                 /* post_handler can hit breakpoint and single step
 310                  * again, so we enable D-flag for recursive exception.
 311                  */
 312                 cur->post_handler(cur, regs, 0);
 313         }
 314
 315         reset_current_kprobe();
 316 }
 317
 318 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
 319 {
 320         struct kprobe *cur = kprobe_running();
 321         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 322
 323         switch (kcb->kprobe_status) {
 324         case KPROBE_HIT_SS:
 325         case KPROBE_REENTER:
 326                 /*
 327                  * We are here because the instruction being single
 328                  * stepped caused a page fault. We reset the current
 329                  * kprobe and the ip points back to the probe address
 330                  * and allow the page fault handler to continue as a
 331                  * normal page fault.
 332                  */
 333                 instruction_pointer_set(regs, (unsigned long) cur->addr);
 334                 if (!instruction_pointer(regs))
 335                         BUG();
 336
 337                 kernel_disable_single_step();
 338                 if (kcb->kprobe_status == KPROBE_REENTER)
 339                         spsr_set_debug_flag(regs, 1);
 340
 341                 if (kcb->kprobe_status == KPROBE_REENTER)
 342                         restore_previous_kprobe(kcb);
 343                 else
 344                         reset_current_kprobe();
 345
 346                 break;
 347         case KPROBE_HIT_ACTIVE:
 348         case KPROBE_HIT_SSDONE:
 349                 /*
 350                  * We increment the nmissed count for accounting,
 351                  * we can also use npre/npostfault count for accounting
 352                  * these specific fault cases.
 353                  */
 354                 kprobes_inc_nmissed_count(cur);
 355
 356                 /*
 357                  * We come here because instructions in the pre/post
 358                  * handler caused the page_fault, this could happen
 359                  * if handler tries to access user space by
 360                  * copy_from_user(), get_user() etc. Let the
 361                  * user-specified handler try to fix it first.
 362                  */
 363                 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr))
 364                         return 1;
 365
 366                 /*
 367                  * In case the user-specified fault handler returned
 368                  * zero, try to fix up.
 369                  */
 370                 if (fixup_exception(regs))
 371                         return 1;
 372         }
 373         return 0;
 374 }
 375
 376 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
 377                                        unsigned long val, void *data)
 378 {
 379         return NOTIFY_DONE;
 380 }
 381
 382 static void __kprobes kprobe_handler(struct pt_regs *regs)
 383 {
 384         struct kprobe *p, *cur_kprobe;
 385         struct kprobe_ctlblk *kcb;
 386         unsigned long addr = instruction_pointer(regs);
 387
 388         kcb = get_kprobe_ctlblk();
 389         cur_kprobe = kprobe_running();
 390
 391         p = get_kprobe((kprobe_opcode_t *) addr);
 392
 393         if (p) {
 394                 if (cur_kprobe) {
 395                         if (reenter_kprobe(p, regs, kcb))
 396                                 return;
 397                 } else {
 398                         /* Probe hit */
 399                         set_current_kprobe(p);
 400                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 401
 402                         /*
 403                          * If we have no pre-handler or it returned 0, we
 404                          * continue with normal processing.  If we have a
 405                          * pre-handler and it returned non-zero, it prepped
 406                          * for calling the break_handler below on re-entry,
 407                          * so get out doing nothing more here.
 408                          *
 409                          * pre_handler can hit a breakpoint and can step thru
 410                          * before return, keep PSTATE D-flag enabled until
 411                          * pre_handler return back.
 412                          */
 413                         if (!p->pre_handler || !p->pre_handler(p, regs)) {
 414                                 setup_singlestep(p, regs, kcb, 0);
 415                                 return;
 416                         }
 417                 }
 418         } else if ((le32_to_cpu(*(kprobe_opcode_t *) addr) ==
 419             BRK64_OPCODE_KPROBES) && cur_kprobe) {
 420                 /* We probably hit a jprobe.  Call its break handler. */
 421                 if (cur_kprobe->break_handler  &&
 422                      cur_kprobe->break_handler(cur_kprobe, regs)) {
 423                         setup_singlestep(cur_kprobe, regs, kcb, 0);
 424                         return;
 425                 }
 426         }
 427         /*
 428          * The breakpoint instruction was removed right
 429          * after we hit it.  Another cpu has removed
 430          * either a probepoint or a debugger breakpoint
 431          * at this address.  In either case, no further
 432          * handling of this interrupt is appropriate.
 433          * Return back to original instruction, and continue.
 434          */
 435 }
 436
 437 static int __kprobes
 438 kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr)
 439 {
 440         if ((kcb->ss_ctx.ss_pending)
 441             && (kcb->ss_ctx.match_addr == addr)) {
 442                 clear_ss_context(kcb);  /* clear pending ss */
 443                 return DBG_HOOK_HANDLED;
 444         }
 445         /* not ours, kprobes should ignore it */
 446         return DBG_HOOK_ERROR;
 447 }
 448
 449 int __kprobes
 450 kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr)
 451 {
 452         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 453         int retval;
 454
 455         /* return error if this is not our step */
 456         retval = kprobe_ss_hit(kcb, instruction_pointer(regs));
 457
 458         if (retval == DBG_HOOK_HANDLED) {
 459                 kprobes_restore_local_irqflag(kcb, regs);
 460                 kernel_disable_single_step();
 461
 462                 if (kcb->kprobe_status == KPROBE_REENTER)
 463                         spsr_set_debug_flag(regs, 1);
 464
 465                 post_kprobe_handler(kcb, regs);
 466         }
 467
 468         return retval;
 469 }
 470
 471 int __kprobes
 472 kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr)
 473 {
 474         kprobe_handler(regs);
 475         return DBG_HOOK_HANDLED;
 476 }
 477
 478 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
 479 {
 480         struct jprobe *jp = container_of(p, struct jprobe, kp);
 481         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 482         long stack_ptr = kernel_stack_pointer(regs);
 483
 484         kcb->jprobe_saved_regs = *regs;
 485         /*
 486          * As Linus pointed out, gcc assumes that the callee
 487          * owns the argument space and could overwrite it, e.g.
 488          * tailcall optimization. So, to be absolutely safe
 489          * we also save and restore enough stack bytes to cover
 490          * the argument area.
 491          */
 492         memcpy(kcb->jprobes_stack, (void *)stack_ptr,
 493                MIN_STACK_SIZE(stack_ptr));
 494
 495         instruction_pointer_set(regs, (unsigned long) jp->entry);
 496         preempt_disable();
 497         pause_graph_tracing();
 498         return 1;
 499 }
 500
 501 void __kprobes jprobe_return(void)
 502 {
 503         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 504
 505         /*
 506          * Jprobe handler return by entering break exception,
 507          * encoded same as kprobe, but with following conditions
 508          * -a magic number in x0 to identify from rest of other kprobes.
 509          * -restore stack addr to original saved pt_regs
 510          */
 511         asm volatile ("ldr x0, [%0]\n\t"
 512                       "mov sp, x0\n\t"
 513                       ".globl jprobe_return_break\n\t"
 514                       "jprobe_return_break:\n\t"
 515                       "brk %1\n\t"
 516                       :
 517                       : "r"(&kcb->jprobe_saved_regs.sp),
 518                       "I"(BRK64_ESR_KPROBES)
 519                       : "memory");
 520 }
 521
 522 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
 523 {
 524         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 525         long stack_addr = kcb->jprobe_saved_regs.sp;
 526         long orig_sp = kernel_stack_pointer(regs);
 527         struct jprobe *jp = container_of(p, struct jprobe, kp);
 528
 529         if (instruction_pointer(regs) != (u64) jprobe_return_break)
 530                 return 0;
 531
 532         if (orig_sp != stack_addr) {
 533                 struct pt_regs *saved_regs =
 534                     (struct pt_regs *)kcb->jprobe_saved_regs.sp;
 535                 pr_err("current sp %lx does not match saved sp %lx\n",
 536                        orig_sp, stack_addr);
 537                 pr_err("Saved registers for jprobe %p\n", jp);
 538                 show_regs(saved_regs);
 539                 pr_err("Current registers\n");
 540                 show_regs(regs);
 541                 BUG();
 542         }
 543         unpause_graph_tracing();
 544         *regs = kcb->jprobe_saved_regs;
 545         memcpy((void *)stack_addr, kcb->jprobes_stack,
 546                MIN_STACK_SIZE(stack_addr));
 547         preempt_enable_no_resched();
 548         return 1;
 549 }
 550
 551 bool arch_within_kprobe_blacklist(unsigned long addr)
 552 {
 553         extern char __idmap_text_start[], __idmap_text_end[];
 554
 555         if ((addr >= (unsigned long)__kprobes_text_start &&
 556             addr < (unsigned long)__kprobes_text_end) ||
 557             (addr >= (unsigned long)__entry_text_start &&
 558             addr < (unsigned long)__entry_text_end) ||
 559             (addr >= (unsigned long)__idmap_text_start &&
 560             addr < (unsigned long)__idmap_text_end) ||
 561             !!search_exception_tables(addr))
 562                 return true;
 563
 564
 565         return false;
 566 }
 567
 568 void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
 569 {
 570         return NULL;
 571 }
 572
 573 int __init arch_init_kprobes(void)
 574 {
 575         return 0;
 576 }