arch/arm64/kernel/insn.c

   1 /*
   2  * Copyright (C) 2013 Huawei Ltd.
   3  * Author: Jiang Liu <liuj97@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 #include <linux/bitops.h>
  18 #include <linux/compiler.h>
  19 #include <linux/kernel.h>
  20 #include <linux/smp.h>
  21 #include <linux/stop_machine.h>
  22 #include <linux/uaccess.h>
  23 #include <asm/cacheflush.h>
  24 #include <asm/insn.h>
  25
  26 static int aarch64_insn_encoding_class[] = {
  27         AARCH64_INSN_CLS_UNKNOWN,
  28         AARCH64_INSN_CLS_UNKNOWN,
  29         AARCH64_INSN_CLS_UNKNOWN,
  30         AARCH64_INSN_CLS_UNKNOWN,
  31         AARCH64_INSN_CLS_LDST,
  32         AARCH64_INSN_CLS_DP_REG,
  33         AARCH64_INSN_CLS_LDST,
  34         AARCH64_INSN_CLS_DP_FPSIMD,
  35         AARCH64_INSN_CLS_DP_IMM,
  36         AARCH64_INSN_CLS_DP_IMM,
  37         AARCH64_INSN_CLS_BR_SYS,
  38         AARCH64_INSN_CLS_BR_SYS,
  39         AARCH64_INSN_CLS_LDST,
  40         AARCH64_INSN_CLS_DP_REG,
  41         AARCH64_INSN_CLS_LDST,
  42         AARCH64_INSN_CLS_DP_FPSIMD,
  43 };
  44
  45 enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
  46 {
  47         return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
  48 }
  49
  50 /* NOP is an alias of HINT */
  51 bool __kprobes aarch64_insn_is_nop(u32 insn)
  52 {
  53         if (!aarch64_insn_is_hint(insn))
  54                 return false;
  55
  56         switch (insn & 0xFE0) {
  57         case AARCH64_INSN_HINT_YIELD:
  58         case AARCH64_INSN_HINT_WFE:
  59         case AARCH64_INSN_HINT_WFI:
  60         case AARCH64_INSN_HINT_SEV:
  61         case AARCH64_INSN_HINT_SEVL:
  62                 return false;
  63         default:
  64                 return true;
  65         }
  66 }
  67
  68 /*
  69  * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
  70  * little-endian.
  71  */
  72 int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
  73 {
  74         int ret;
  75         u32 val;
  76
  77         ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE);
  78         if (!ret)
  79                 *insnp = le32_to_cpu(val);
  80
  81         return ret;
  82 }
  83
  84 int __kprobes aarch64_insn_write(void *addr, u32 insn)
  85 {
  86         insn = cpu_to_le32(insn);
  87         return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
  88 }
  89
  90 static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
  91 {
  92         if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
  93                 return false;
  94
  95         return  aarch64_insn_is_b(insn) ||
  96                 aarch64_insn_is_bl(insn) ||
  97                 aarch64_insn_is_svc(insn) ||
  98                 aarch64_insn_is_hvc(insn) ||
  99                 aarch64_insn_is_smc(insn) ||
 100                 aarch64_insn_is_brk(insn) ||
 101                 aarch64_insn_is_nop(insn);
 102 }
 103
 104 /*
 105  * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
 106  * Section B2.6.5 "Concurrent modification and execution of instructions":
 107  * Concurrent modification and execution of instructions can lead to the
 108  * resulting instruction performing any behavior that can be achieved by
 109  * executing any sequence of instructions that can be executed from the
 110  * same Exception level, except where the instruction before modification
 111  * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
 112  * or SMC instruction.
 113  */
 114 bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
 115 {
 116         return __aarch64_insn_hotpatch_safe(old_insn) &&
 117                __aarch64_insn_hotpatch_safe(new_insn);
 118 }
 119
 120 int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
 121 {
 122         u32 *tp = addr;
 123         int ret;
 124
 125         /* A64 instructions must be word aligned */
 126         if ((uintptr_t)tp & 0x3)
 127                 return -EINVAL;
 128
 129         ret = aarch64_insn_write(tp, insn);
 130         if (ret == 0)
 131                 flush_icache_range((uintptr_t)tp,
 132                                    (uintptr_t)tp + AARCH64_INSN_SIZE);
 133
 134         return ret;
 135 }
 136
 137 struct aarch64_insn_patch {
 138         void            **text_addrs;
 139         u32             *new_insns;
 140         int             insn_cnt;
 141         atomic_t        cpu_count;
 142 };
 143
 144 static int __kprobes aarch64_insn_patch_text_cb(void *arg)
 145 {
 146         int i, ret = 0;
 147         struct aarch64_insn_patch *pp = arg;
 148
 149         /* The first CPU becomes master */
 150         if (atomic_inc_return(&pp->cpu_count) == 1) {
 151                 for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
 152                         ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
 153                                                              pp->new_insns[i]);
 154                 /*
 155                  * aarch64_insn_patch_text_nosync() calls flush_icache_range(),
 156                  * which ends with "dsb; isb" pair guaranteeing global
 157                  * visibility.
 158                  */
 159                 atomic_set(&pp->cpu_count, -1);
 160         } else {
 161                 while (atomic_read(&pp->cpu_count) != -1)
 162                         cpu_relax();
 163                 isb();
 164         }
 165
 166         return ret;
 167 }
 168
 169 int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
 170 {
 171         struct aarch64_insn_patch patch = {
 172                 .text_addrs = addrs,
 173                 .new_insns = insns,
 174                 .insn_cnt = cnt,
 175                 .cpu_count = ATOMIC_INIT(0),
 176         };
 177
 178         if (cnt <= 0)
 179                 return -EINVAL;
 180
 181         return stop_machine(aarch64_insn_patch_text_cb, &patch,
 182                             cpu_online_mask);
 183 }
 184
 185 int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
 186 {
 187         int ret;
 188         u32 insn;
 189
 190         /* Unsafe to patch multiple instructions without synchronizaiton */
 191         if (cnt == 1) {
 192                 ret = aarch64_insn_read(addrs[0], &insn);
 193                 if (ret)
 194                         return ret;
 195
 196                 if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
 197                         /*
 198                          * ARMv8 architecture doesn't guarantee all CPUs see
 199                          * the new instruction after returning from function
 200                          * aarch64_insn_patch_text_nosync(). So send IPIs to
 201                          * all other CPUs to achieve instruction
 202                          * synchronization.
 203                          */
 204                         ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
 205                         kick_all_cpus_sync();
 206                         return ret;
 207                 }
 208         }
 209
 210         return aarch64_insn_patch_text_sync(addrs, insns, cnt);
 211 }
 212
 213 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 214                                   u32 insn, u64 imm)
 215 {
 216         u32 immlo, immhi, lomask, himask, mask;
 217         int shift;
 218
 219         switch (type) {
 220         case AARCH64_INSN_IMM_ADR:
 221                 lomask = 0x3;
 222                 himask = 0x7ffff;
 223                 immlo = imm & lomask;
 224                 imm >>= 2;
 225                 immhi = imm & himask;
 226                 imm = (immlo << 24) | (immhi);
 227                 mask = (lomask << 24) | (himask);
 228                 shift = 5;
 229                 break;
 230         case AARCH64_INSN_IMM_26:
 231                 mask = BIT(26) - 1;
 232                 shift = 0;
 233                 break;
 234         case AARCH64_INSN_IMM_19:
 235                 mask = BIT(19) - 1;
 236                 shift = 5;
 237                 break;
 238         case AARCH64_INSN_IMM_16:
 239                 mask = BIT(16) - 1;
 240                 shift = 5;
 241                 break;
 242         case AARCH64_INSN_IMM_14:
 243                 mask = BIT(14) - 1;
 244                 shift = 5;
 245                 break;
 246         case AARCH64_INSN_IMM_12:
 247                 mask = BIT(12) - 1;
 248                 shift = 10;
 249                 break;
 250         case AARCH64_INSN_IMM_9:
 251                 mask = BIT(9) - 1;
 252                 shift = 12;
 253                 break;
 254         default:
 255                 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
 256                         type);
 257                 return 0;
 258         }
 259
 260         /* Update the immediate field. */
 261         insn &= ~(mask << shift);
 262         insn |= (imm & mask) << shift;
 263
 264         return insn;
 265 }
 266
 267 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
 268                                           enum aarch64_insn_branch_type type)
 269 {
 270         u32 insn;
 271         long offset;
 272
 273         /*
 274          * PC: A 64-bit Program Counter holding the address of the current
 275          * instruction. A64 instructions must be word-aligned.
 276          */
 277         BUG_ON((pc & 0x3) || (addr & 0x3));
 278
 279         /*
 280          * B/BL support [-128M, 128M) offset
 281          * ARM64 virtual address arrangement guarantees all kernel and module
 282          * texts are within +/-128M.
 283          */
 284         offset = ((long)addr - (long)pc);
 285         BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
 286
 287         if (type == AARCH64_INSN_BRANCH_LINK)
 288                 insn = aarch64_insn_get_bl_value();
 289         else
 290                 insn = aarch64_insn_get_b_value();
 291
 292         return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
 293                                              offset >> 2);
 294 }
 295
 296 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
 297 {
 298         return aarch64_insn_get_hint_value() | op;
 299 }
 300
 301 u32 __kprobes aarch64_insn_gen_nop(void)
 302 {
 303         return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
 304 }