arch/arm64/kvm/sys_regs.c

   1 /*
   2  * Copyright (C) 2012,2013 - ARM Ltd
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * Derived from arch/arm/kvm/coproc.c:
   6  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
   7  * Authors: Rusty Russell <rusty@rustcorp.com.au>
   8  *          Christoffer Dall <c.dall@virtualopensystems.com>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License, version 2, as
  12  * published by the Free Software Foundation.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21  */
  22
  23 #include <linux/mm.h>
  24 #include <linux/kvm_host.h>
  25 #include <linux/uaccess.h>
  26 #include <asm/kvm_arm.h>
  27 #include <asm/kvm_host.h>
  28 #include <asm/kvm_emulate.h>
  29 #include <asm/kvm_coproc.h>
  30 #include <asm/kvm_mmu.h>
  31 #include <asm/cacheflush.h>
  32 #include <asm/cputype.h>
  33 #include <asm/debug-monitors.h>
  34 #include <trace/events/kvm.h>
  35
  36 #include "sys_regs.h"
  37
  38 /*
  39  * All of this file is extremly similar to the ARM coproc.c, but the
  40  * types are different. My gut feeling is that it should be pretty
  41  * easy to merge, but that would be an ABI breakage -- again. VFP
  42  * would also need to be abstracted.
  43  *
  44  * For AArch32, we only take care of what is being trapped. Anything
  45  * that has to do with init and userspace access has to go via the
  46  * 64bit interface.
  47  */
  48
  49 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
  50 static u32 cache_levels;
  51
  52 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
  53 #define CSSELR_MAX 12
  54
  55 /* Which cache CCSIDR represents depends on CSSELR value. */
  56 static u32 get_ccsidr(u32 csselr)
  57 {
  58         u32 ccsidr;
  59
  60         /* Make sure noone else changes CSSELR during this! */
  61         local_irq_disable();
  62         /* Put value into CSSELR */
  63         asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
  64         isb();
  65         /* Read result out of CCSIDR */
  66         asm volatile("mrs %0, ccsidr_el1" : "=r" (ccsidr));
  67         local_irq_enable();
  68
  69         return ccsidr;
  70 }
  71
  72 static void do_dc_cisw(u32 val)
  73 {
  74         asm volatile("dc cisw, %x0" : : "r" (val));
  75         dsb(ish);
  76 }
  77
  78 static void do_dc_csw(u32 val)
  79 {
  80         asm volatile("dc csw, %x0" : : "r" (val));
  81         dsb(ish);
  82 }
  83
  84 /* See note at ARM ARM B1.14.4 */
  85 static bool access_dcsw(struct kvm_vcpu *vcpu,
  86                         const struct sys_reg_params *p,
  87                         const struct sys_reg_desc *r)
  88 {
  89         unsigned long val;
  90         int cpu;
  91
  92         if (!p->is_write)
  93                 return read_from_write_only(vcpu, p);
  94
  95         cpu = get_cpu();
  96
  97         cpumask_setall(&vcpu->arch.require_dcache_flush);
  98         cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
  99
 100         /* If we were already preempted, take the long way around */
 101         if (cpu != vcpu->arch.last_pcpu) {
 102                 flush_cache_all();
 103                 goto done;
 104         }
 105
 106         val = *vcpu_reg(vcpu, p->Rt);
 107
 108         switch (p->CRm) {
 109         case 6:                 /* Upgrade DCISW to DCCISW, as per HCR.SWIO */
 110         case 14:                /* DCCISW */
 111                 do_dc_cisw(val);
 112                 break;
 113
 114         case 10:                /* DCCSW */
 115                 do_dc_csw(val);
 116                 break;
 117         }
 118
 119 done:
 120         put_cpu();
 121
 122         return true;
 123 }
 124
 125 /*
 126  * Generic accessor for VM registers. Only called as long as HCR_TVM
 127  * is set.
 128  */
 129 static bool access_vm_reg(struct kvm_vcpu *vcpu,
 130                           const struct sys_reg_params *p,
 131                           const struct sys_reg_desc *r)
 132 {
 133         unsigned long val;
 134
 135         BUG_ON(!p->is_write);
 136
 137         val = *vcpu_reg(vcpu, p->Rt);
 138         if (!p->is_aarch32) {
 139                 vcpu_sys_reg(vcpu, r->reg) = val;
 140         } else {
 141                 if (!p->is_32bit)
 142                         vcpu_cp15_64_high(vcpu, r->reg) = val >> 32;
 143                 vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL;
 144         }
 145
 146         return true;
 147 }
 148
 149 /*
 150  * SCTLR_EL1 accessor. Only called as long as HCR_TVM is set.  If the
 151  * guest enables the MMU, we stop trapping the VM sys_regs and leave
 152  * it in complete control of the caches.
 153  */
 154 static bool access_sctlr(struct kvm_vcpu *vcpu,
 155                          const struct sys_reg_params *p,
 156                          const struct sys_reg_desc *r)
 157 {
 158         access_vm_reg(vcpu, p, r);
 159
 160         if (vcpu_has_cache_enabled(vcpu)) {     /* MMU+Caches enabled? */
 161                 vcpu->arch.hcr_el2 &= ~HCR_TVM;
 162                 stage2_flush_vm(vcpu->kvm);
 163         }
 164
 165         return true;
 166 }
 167
 168 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
 169                         const struct sys_reg_params *p,
 170                         const struct sys_reg_desc *r)
 171 {
 172         if (p->is_write)
 173                 return ignore_write(vcpu, p);
 174         else
 175                 return read_zero(vcpu, p);
 176 }
 177
 178 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
 179                            const struct sys_reg_params *p,
 180                            const struct sys_reg_desc *r)
 181 {
 182         if (p->is_write) {
 183                 return ignore_write(vcpu, p);
 184         } else {
 185                 *vcpu_reg(vcpu, p->Rt) = (1 << 3);
 186                 return true;
 187         }
 188 }
 189
 190 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
 191                                    const struct sys_reg_params *p,
 192                                    const struct sys_reg_desc *r)
 193 {
 194         if (p->is_write) {
 195                 return ignore_write(vcpu, p);
 196         } else {
 197                 u32 val;
 198                 asm volatile("mrs %0, dbgauthstatus_el1" : "=r" (val));
 199                 *vcpu_reg(vcpu, p->Rt) = val;
 200                 return true;
 201         }
 202 }
 203
 204 /*
 205  * We want to avoid world-switching all the DBG registers all the
 206  * time:
 207  *
 208  * - If we've touched any debug register, it is likely that we're
 209  *   going to touch more of them. It then makes sense to disable the
 210  *   traps and start doing the save/restore dance
 211  * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
 212  *   then mandatory to save/restore the registers, as the guest
 213  *   depends on them.
 214  *
 215  * For this, we use a DIRTY bit, indicating the guest has modified the
 216  * debug registers, used as follow:
 217  *
 218  * On guest entry:
 219  * - If the dirty bit is set (because we're coming back from trapping),
 220  *   disable the traps, save host registers, restore guest registers.
 221  * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
 222  *   set the dirty bit, disable the traps, save host registers,
 223  *   restore guest registers.
 224  * - Otherwise, enable the traps
 225  *
 226  * On guest exit:
 227  * - If the dirty bit is set, save guest registers, restore host
 228  *   registers and clear the dirty bit. This ensure that the host can
 229  *   now use the debug registers.
 230  */
 231 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
 232                             const struct sys_reg_params *p,
 233                             const struct sys_reg_desc *r)
 234 {
 235         if (p->is_write) {
 236                 vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
 237                 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
 238         } else {
 239                 *vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
 240         }
 241
 242         return true;
 243 }
 244
 245 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 246 {
 247         u64 amair;
 248
 249         asm volatile("mrs %0, amair_el1\n" : "=r" (amair));
 250         vcpu_sys_reg(vcpu, AMAIR_EL1) = amair;
 251 }
 252
 253 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 254 {
 255         /*
 256          * Simply map the vcpu_id into the Aff0 field of the MPIDR.
 257          */
 258         vcpu_sys_reg(vcpu, MPIDR_EL1) = (1UL << 31) | (vcpu->vcpu_id & 0xff);
 259 }
 260
 261 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 262 #define DBG_BCR_BVR_WCR_WVR_EL1(n)                                      \
 263         /* DBGBVRn_EL1 */                                               \
 264         { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100),     \
 265           trap_debug_regs, reset_val, (DBGBVR0_EL1 + (n)), 0 },         \
 266         /* DBGBCRn_EL1 */                                               \
 267         { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101),     \
 268           trap_debug_regs, reset_val, (DBGBCR0_EL1 + (n)), 0 },         \
 269         /* DBGWVRn_EL1 */                                               \
 270         { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110),     \
 271           trap_debug_regs, reset_val, (DBGWVR0_EL1 + (n)), 0 },         \
 272         /* DBGWCRn_EL1 */                                               \
 273         { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111),     \
 274           trap_debug_regs, reset_val, (DBGWCR0_EL1 + (n)), 0 }
 275
 276 /*
 277  * Architected system registers.
 278  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
 279  *
 280  * We could trap ID_DFR0 and tell the guest we don't support performance
 281  * monitoring.  Unfortunately the patch to make the kernel check ID_DFR0 was
 282  * NAKed, so it will read the PMCR anyway.
 283  *
 284  * Therefore we tell the guest we have 0 counters.  Unfortunately, we
 285  * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
 286  * all PM registers, which doesn't crash the guest kernel at least.
 287  *
 288  * Debug handling: We do trap most, if not all debug related system
 289  * registers. The implementation is good enough to ensure that a guest
 290  * can use these with minimal performance degradation. The drawback is
 291  * that we don't implement any of the external debug, none of the
 292  * OSlock protocol. This should be revisited if we ever encounter a
 293  * more demanding guest...
 294  */
 295 static const struct sys_reg_desc sys_reg_descs[] = {
 296         /* DC ISW */
 297         { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010),
 298           access_dcsw },
 299         /* DC CSW */
 300         { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010),
 301           access_dcsw },
 302         /* DC CISW */
 303         { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010),
 304           access_dcsw },
 305
 306         DBG_BCR_BVR_WCR_WVR_EL1(0),
 307         DBG_BCR_BVR_WCR_WVR_EL1(1),
 308         /* MDCCINT_EL1 */
 309         { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
 310           trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
 311         /* MDSCR_EL1 */
 312         { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
 313           trap_debug_regs, reset_val, MDSCR_EL1, 0 },
 314         DBG_BCR_BVR_WCR_WVR_EL1(2),
 315         DBG_BCR_BVR_WCR_WVR_EL1(3),
 316         DBG_BCR_BVR_WCR_WVR_EL1(4),
 317         DBG_BCR_BVR_WCR_WVR_EL1(5),
 318         DBG_BCR_BVR_WCR_WVR_EL1(6),
 319         DBG_BCR_BVR_WCR_WVR_EL1(7),
 320         DBG_BCR_BVR_WCR_WVR_EL1(8),
 321         DBG_BCR_BVR_WCR_WVR_EL1(9),
 322         DBG_BCR_BVR_WCR_WVR_EL1(10),
 323         DBG_BCR_BVR_WCR_WVR_EL1(11),
 324         DBG_BCR_BVR_WCR_WVR_EL1(12),
 325         DBG_BCR_BVR_WCR_WVR_EL1(13),
 326         DBG_BCR_BVR_WCR_WVR_EL1(14),
 327         DBG_BCR_BVR_WCR_WVR_EL1(15),
 328
 329         /* MDRAR_EL1 */
 330         { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
 331           trap_raz_wi },
 332         /* OSLAR_EL1 */
 333         { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100),
 334           trap_raz_wi },
 335         /* OSLSR_EL1 */
 336         { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100),
 337           trap_oslsr_el1 },
 338         /* OSDLR_EL1 */
 339         { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100),
 340           trap_raz_wi },
 341         /* DBGPRCR_EL1 */
 342         { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100),
 343           trap_raz_wi },
 344         /* DBGCLAIMSET_EL1 */
 345         { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110),
 346           trap_raz_wi },
 347         /* DBGCLAIMCLR_EL1 */
 348         { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110),
 349           trap_raz_wi },
 350         /* DBGAUTHSTATUS_EL1 */
 351         { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),
 352           trap_dbgauthstatus_el1 },
 353
 354         /* TEECR32_EL1 */
 355         { Op0(0b10), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
 356           NULL, reset_val, TEECR32_EL1, 0 },
 357         /* TEEHBR32_EL1 */
 358         { Op0(0b10), Op1(0b010), CRn(0b0001), CRm(0b0000), Op2(0b000),
 359           NULL, reset_val, TEEHBR32_EL1, 0 },
 360
 361         /* MDCCSR_EL1 */
 362         { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),
 363           trap_raz_wi },
 364         /* DBGDTR_EL0 */
 365         { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000),
 366           trap_raz_wi },
 367         /* DBGDTR[TR]X_EL0 */
 368         { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000),
 369           trap_raz_wi },
 370
 371         /* DBGVCR32_EL2 */
 372         { Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000),
 373           NULL, reset_val, DBGVCR32_EL2, 0 },
 374
 375         /* MPIDR_EL1 */
 376         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101),
 377           NULL, reset_mpidr, MPIDR_EL1 },
 378         /* SCTLR_EL1 */
 379         { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
 380           access_sctlr, reset_val, SCTLR_EL1, 0x00C50078 },
 381         /* CPACR_EL1 */
 382         { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
 383           NULL, reset_val, CPACR_EL1, 0 },
 384         /* TTBR0_EL1 */
 385         { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),
 386           access_vm_reg, reset_unknown, TTBR0_EL1 },
 387         /* TTBR1_EL1 */
 388         { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001),
 389           access_vm_reg, reset_unknown, TTBR1_EL1 },
 390         /* TCR_EL1 */
 391         { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),
 392           access_vm_reg, reset_val, TCR_EL1, 0 },
 393
 394         /* AFSR0_EL1 */
 395         { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),
 396           access_vm_reg, reset_unknown, AFSR0_EL1 },
 397         /* AFSR1_EL1 */
 398         { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001),
 399           access_vm_reg, reset_unknown, AFSR1_EL1 },
 400         /* ESR_EL1 */
 401         { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000),
 402           access_vm_reg, reset_unknown, ESR_EL1 },
 403         /* FAR_EL1 */
 404         { Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000),
 405           access_vm_reg, reset_unknown, FAR_EL1 },
 406         /* PAR_EL1 */
 407         { Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000),
 408           NULL, reset_unknown, PAR_EL1 },
 409
 410         /* PMINTENSET_EL1 */
 411         { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
 412           trap_raz_wi },
 413         /* PMINTENCLR_EL1 */
 414         { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
 415           trap_raz_wi },
 416
 417         /* MAIR_EL1 */
 418         { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
 419           access_vm_reg, reset_unknown, MAIR_EL1 },
 420         /* AMAIR_EL1 */
 421         { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000),
 422           access_vm_reg, reset_amair_el1, AMAIR_EL1 },
 423
 424         /* VBAR_EL1 */
 425         { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
 426           NULL, reset_val, VBAR_EL1, 0 },
 427         /* CONTEXTIDR_EL1 */
 428         { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001),
 429           access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
 430         /* TPIDR_EL1 */
 431         { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100),
 432           NULL, reset_unknown, TPIDR_EL1 },
 433
 434         /* CNTKCTL_EL1 */
 435         { Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000),
 436           NULL, reset_val, CNTKCTL_EL1, 0},
 437
 438         /* CSSELR_EL1 */
 439         { Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
 440           NULL, reset_unknown, CSSELR_EL1 },
 441
 442         /* PMCR_EL0 */
 443         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
 444           trap_raz_wi },
 445         /* PMCNTENSET_EL0 */
 446         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
 447           trap_raz_wi },
 448         /* PMCNTENCLR_EL0 */
 449         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
 450           trap_raz_wi },
 451         /* PMOVSCLR_EL0 */
 452         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
 453           trap_raz_wi },
 454         /* PMSWINC_EL0 */
 455         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
 456           trap_raz_wi },
 457         /* PMSELR_EL0 */
 458         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
 459           trap_raz_wi },
 460         /* PMCEID0_EL0 */
 461         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
 462           trap_raz_wi },
 463         /* PMCEID1_EL0 */
 464         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
 465           trap_raz_wi },
 466         /* PMCCNTR_EL0 */
 467         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
 468           trap_raz_wi },
 469         /* PMXEVTYPER_EL0 */
 470         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
 471           trap_raz_wi },
 472         /* PMXEVCNTR_EL0 */
 473         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
 474           trap_raz_wi },
 475         /* PMUSERENR_EL0 */
 476         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
 477           trap_raz_wi },
 478         /* PMOVSSET_EL0 */
 479         { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
 480           trap_raz_wi },
 481
 482         /* TPIDR_EL0 */
 483         { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
 484           NULL, reset_unknown, TPIDR_EL0 },
 485         /* TPIDRRO_EL0 */
 486         { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
 487           NULL, reset_unknown, TPIDRRO_EL0 },
 488
 489         /* DACR32_EL2 */
 490         { Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
 491           NULL, reset_unknown, DACR32_EL2 },
 492         /* IFSR32_EL2 */
 493         { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001),
 494           NULL, reset_unknown, IFSR32_EL2 },
 495         /* FPEXC32_EL2 */
 496         { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000),
 497           NULL, reset_val, FPEXC32_EL2, 0x70 },
 498 };
 499
 500 static bool trap_dbgidr(struct kvm_vcpu *vcpu,
 501                         const struct sys_reg_params *p,
 502                         const struct sys_reg_desc *r)
 503 {
 504         if (p->is_write) {
 505                 return ignore_write(vcpu, p);
 506         } else {
 507                 u64 dfr = read_cpuid(ID_AA64DFR0_EL1);
 508                 u64 pfr = read_cpuid(ID_AA64PFR0_EL1);
 509                 u32 el3 = !!((pfr >> 12) & 0xf);
 510
 511                 *vcpu_reg(vcpu, p->Rt) = ((((dfr >> 20) & 0xf) << 28) |
 512                                           (((dfr >> 12) & 0xf) << 24) |
 513                                           (((dfr >> 28) & 0xf) << 20) |
 514                                           (6 << 16) | (el3 << 14) | (el3 << 12));
 515                 return true;
 516         }
 517 }
 518
 519 static bool trap_debug32(struct kvm_vcpu *vcpu,
 520                          const struct sys_reg_params *p,
 521                          const struct sys_reg_desc *r)
 522 {
 523         if (p->is_write) {
 524                 vcpu_cp14(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
 525                 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
 526         } else {
 527                 *vcpu_reg(vcpu, p->Rt) = vcpu_cp14(vcpu, r->reg);
 528         }
 529
 530         return true;
 531 }
 532
 533 #define DBG_BCR_BVR_WCR_WVR(n)                                  \
 534         /* DBGBVRn */                                           \
 535         { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_debug32,    \
 536           NULL, (cp14_DBGBVR0 + (n) * 2) },                     \
 537         /* DBGBCRn */                                           \
 538         { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_debug32,    \
 539           NULL, (cp14_DBGBCR0 + (n) * 2) },                     \
 540         /* DBGWVRn */                                           \
 541         { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_debug32,    \
 542           NULL, (cp14_DBGWVR0 + (n) * 2) },                     \
 543         /* DBGWCRn */                                           \
 544         { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_debug32,    \
 545           NULL, (cp14_DBGWCR0 + (n) * 2) }
 546
 547 #define DBGBXVR(n)                                              \
 548         { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_debug32,    \
 549           NULL, cp14_DBGBXVR0 + n * 2 }
 550
 551 /*
 552  * Trapped cp14 registers. We generally ignore most of the external
 553  * debug, on the principle that they don't really make sense to a
 554  * guest. Revisit this one day, whould this principle change.
 555  */
 556 static const struct sys_reg_desc cp14_regs[] = {
 557         /* DBGIDR */
 558         { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
 559         /* DBGDTRRXext */
 560         { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
 561
 562         DBG_BCR_BVR_WCR_WVR(0),
 563         /* DBGDSCRint */
 564         { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
 565         DBG_BCR_BVR_WCR_WVR(1),
 566         /* DBGDCCINT */
 567         { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 },
 568         /* DBGDSCRext */
 569         { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 },
 570         DBG_BCR_BVR_WCR_WVR(2),
 571         /* DBGDTR[RT]Xint */
 572         { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
 573         /* DBGDTR[RT]Xext */
 574         { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
 575         DBG_BCR_BVR_WCR_WVR(3),
 576         DBG_BCR_BVR_WCR_WVR(4),
 577         DBG_BCR_BVR_WCR_WVR(5),
 578         /* DBGWFAR */
 579         { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
 580         /* DBGOSECCR */
 581         { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
 582         DBG_BCR_BVR_WCR_WVR(6),
 583         /* DBGVCR */
 584         { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 },
 585         DBG_BCR_BVR_WCR_WVR(7),
 586         DBG_BCR_BVR_WCR_WVR(8),
 587         DBG_BCR_BVR_WCR_WVR(9),
 588         DBG_BCR_BVR_WCR_WVR(10),
 589         DBG_BCR_BVR_WCR_WVR(11),
 590         DBG_BCR_BVR_WCR_WVR(12),
 591         DBG_BCR_BVR_WCR_WVR(13),
 592         DBG_BCR_BVR_WCR_WVR(14),
 593         DBG_BCR_BVR_WCR_WVR(15),
 594
 595         /* DBGDRAR (32bit) */
 596         { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
 597
 598         DBGBXVR(0),
 599         /* DBGOSLAR */
 600         { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
 601         DBGBXVR(1),
 602         /* DBGOSLSR */
 603         { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
 604         DBGBXVR(2),
 605         DBGBXVR(3),
 606         /* DBGOSDLR */
 607         { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
 608         DBGBXVR(4),
 609         /* DBGPRCR */
 610         { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
 611         DBGBXVR(5),
 612         DBGBXVR(6),
 613         DBGBXVR(7),
 614         DBGBXVR(8),
 615         DBGBXVR(9),
 616         DBGBXVR(10),
 617         DBGBXVR(11),
 618         DBGBXVR(12),
 619         DBGBXVR(13),
 620         DBGBXVR(14),
 621         DBGBXVR(15),
 622
 623         /* DBGDSAR (32bit) */
 624         { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
 625
 626         /* DBGDEVID2 */
 627         { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
 628         /* DBGDEVID1 */
 629         { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
 630         /* DBGDEVID */
 631         { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
 632         /* DBGCLAIMSET */
 633         { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
 634         /* DBGCLAIMCLR */
 635         { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
 636         /* DBGAUTHSTATUS */
 637         { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
 638 };
 639
 640 /* Trapped cp14 64bit registers */
 641 static const struct sys_reg_desc cp14_64_regs[] = {
 642         /* DBGDRAR (64bit) */
 643         { Op1( 0), CRm( 1), .access = trap_raz_wi },
 644
 645         /* DBGDSAR (64bit) */
 646         { Op1( 0), CRm( 2), .access = trap_raz_wi },
 647 };
 648
 649 /*
 650  * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
 651  * depending on the way they are accessed (as a 32bit or a 64bit
 652  * register).
 653  */
 654 static const struct sys_reg_desc cp15_regs[] = {
 655         { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR },
 656         { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
 657         { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
 658         { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
 659         { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
 660         { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
 661         { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
 662         { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
 663         { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
 664         { Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
 665         { Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
 666
 667         /*
 668          * DC{C,I,CI}SW operations:
 669          */
 670         { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
 671         { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
 672         { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
 673
 674         /* PMU */
 675         { Op1( 0), CRn( 9), CRm(12), Op2( 0), trap_raz_wi },
 676         { Op1( 0), CRn( 9), CRm(12), Op2( 1), trap_raz_wi },
 677         { Op1( 0), CRn( 9), CRm(12), Op2( 2), trap_raz_wi },
 678         { Op1( 0), CRn( 9), CRm(12), Op2( 3), trap_raz_wi },
 679         { Op1( 0), CRn( 9), CRm(12), Op2( 5), trap_raz_wi },
 680         { Op1( 0), CRn( 9), CRm(12), Op2( 6), trap_raz_wi },
 681         { Op1( 0), CRn( 9), CRm(12), Op2( 7), trap_raz_wi },
 682         { Op1( 0), CRn( 9), CRm(13), Op2( 0), trap_raz_wi },
 683         { Op1( 0), CRn( 9), CRm(13), Op2( 1), trap_raz_wi },
 684         { Op1( 0), CRn( 9), CRm(13), Op2( 2), trap_raz_wi },
 685         { Op1( 0), CRn( 9), CRm(14), Op2( 0), trap_raz_wi },
 686         { Op1( 0), CRn( 9), CRm(14), Op2( 1), trap_raz_wi },
 687         { Op1( 0), CRn( 9), CRm(14), Op2( 2), trap_raz_wi },
 688
 689         { Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
 690         { Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
 691         { Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
 692         { Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
 693         { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
 694 };
 695
 696 static const struct sys_reg_desc cp15_64_regs[] = {
 697         { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
 698         { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
 699 };
 700
 701 /* Target specific emulation tables */
 702 static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
 703
 704 void kvm_register_target_sys_reg_table(unsigned int target,
 705                                        struct kvm_sys_reg_target_table *table)
 706 {
 707         target_tables[target] = table;
 708 }
 709
 710 /* Get specific register table for this target. */
 711 static const struct sys_reg_desc *get_target_table(unsigned target,
 712                                                    bool mode_is_64,
 713                                                    size_t *num)
 714 {
 715         struct kvm_sys_reg_target_table *table;
 716
 717         table = target_tables[target];
 718         if (mode_is_64) {
 719                 *num = table->table64.num;
 720                 return table->table64.table;
 721         } else {
 722                 *num = table->table32.num;
 723                 return table->table32.table;
 724         }
 725 }
 726
 727 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
 728                                          const struct sys_reg_desc table[],
 729                                          unsigned int num)
 730 {
 731         unsigned int i;
 732
 733         for (i = 0; i < num; i++) {
 734                 const struct sys_reg_desc *r = &table[i];
 735
 736                 if (params->Op0 != r->Op0)
 737                         continue;
 738                 if (params->Op1 != r->Op1)
 739                         continue;
 740                 if (params->CRn != r->CRn)
 741                         continue;
 742                 if (params->CRm != r->CRm)
 743                         continue;
 744                 if (params->Op2 != r->Op2)
 745                         continue;
 746
 747                 return r;
 748         }
 749         return NULL;
 750 }
 751
 752 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
 753 {
 754         kvm_inject_undefined(vcpu);
 755         return 1;
 756 }
 757
 758 /*
 759  * emulate_cp --  tries to match a sys_reg access in a handling table, and
 760  *                call the corresponding trap handler.
 761  *
 762  * @params: pointer to the descriptor of the access
 763  * @table: array of trap descriptors
 764  * @num: size of the trap descriptor array
 765  *
 766  * Return 0 if the access has been handled, and -1 if not.
 767  */
 768 static int emulate_cp(struct kvm_vcpu *vcpu,
 769                       const struct sys_reg_params *params,
 770                       const struct sys_reg_desc *table,
 771                       size_t num)
 772 {
 773         const struct sys_reg_desc *r;
 774
 775         if (!table)
 776                 return -1;      /* Not handled */
 777
 778         r = find_reg(params, table, num);
 779
 780         if (r) {
 781                 /*
 782                  * Not having an accessor means that we have
 783                  * configured a trap that we don't know how to
 784                  * handle. This certainly qualifies as a gross bug
 785                  * that should be fixed right away.
 786                  */
 787                 BUG_ON(!r->access);
 788
 789                 if (likely(r->access(vcpu, params, r))) {
 790                         /* Skip instruction, since it was emulated */
 791                         kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 792                 }
 793
 794                 /* Handled */
 795                 return 0;
 796         }
 797
 798         /* Not handled */
 799         return -1;
 800 }
 801
 802 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
 803                                 struct sys_reg_params *params)
 804 {
 805         u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
 806         int cp;
 807
 808         switch(hsr_ec) {
 809         case ESR_EL2_EC_CP15_32:
 810         case ESR_EL2_EC_CP15_64:
 811                 cp = 15;
 812                 break;
 813         case ESR_EL2_EC_CP14_MR:
 814         case ESR_EL2_EC_CP14_64:
 815                 cp = 14;
 816                 break;
 817         default:
 818                 WARN_ON((cp = -1));
 819         }
 820
 821         kvm_err("Unsupported guest CP%d access at: %08lx\n",
 822                 cp, *vcpu_pc(vcpu));
 823         print_sys_reg_instr(params);
 824         kvm_inject_undefined(vcpu);
 825 }
 826
 827 /**
 828  * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP15 access
 829  * @vcpu: The VCPU pointer
 830  * @run:  The kvm_run struct
 831  */
 832 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
 833                             const struct sys_reg_desc *global,
 834                             size_t nr_global,
 835                             const struct sys_reg_desc *target_specific,
 836                             size_t nr_specific)
 837 {
 838         struct sys_reg_params params;
 839         u32 hsr = kvm_vcpu_get_hsr(vcpu);
 840         int Rt2 = (hsr >> 10) & 0xf;
 841
 842         params.is_aarch32 = true;
 843         params.is_32bit = false;
 844         params.CRm = (hsr >> 1) & 0xf;
 845         params.Rt = (hsr >> 5) & 0xf;
 846         params.is_write = ((hsr & 1) == 0);
 847
 848         params.Op0 = 0;
 849         params.Op1 = (hsr >> 16) & 0xf;
 850         params.Op2 = 0;
 851         params.CRn = 0;
 852
 853         /*
 854          * Massive hack here. Store Rt2 in the top 32bits so we only
 855          * have one register to deal with. As we use the same trap
 856          * backends between AArch32 and AArch64, we get away with it.
 857          */
 858         if (params.is_write) {
 859                 u64 val = *vcpu_reg(vcpu, params.Rt);
 860                 val &= 0xffffffff;
 861                 val |= *vcpu_reg(vcpu, Rt2) << 32;
 862                 *vcpu_reg(vcpu, params.Rt) = val;
 863         }
 864
 865         if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
 866                 goto out;
 867         if (!emulate_cp(vcpu, &params, global, nr_global))
 868                 goto out;
 869
 870         unhandled_cp_access(vcpu, &params);
 871
 872 out:
 873         /* Do the opposite hack for the read side */
 874         if (!params.is_write) {
 875                 u64 val = *vcpu_reg(vcpu, params.Rt);
 876                 val >>= 32;
 877                 *vcpu_reg(vcpu, Rt2) = val;
 878         }
 879
 880         return 1;
 881 }
 882
 883 /**
 884  * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access
 885  * @vcpu: The VCPU pointer
 886  * @run:  The kvm_run struct
 887  */
 888 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
 889                             const struct sys_reg_desc *global,
 890                             size_t nr_global,
 891                             const struct sys_reg_desc *target_specific,
 892                             size_t nr_specific)
 893 {
 894         struct sys_reg_params params;
 895         u32 hsr = kvm_vcpu_get_hsr(vcpu);
 896
 897         params.is_aarch32 = true;
 898         params.is_32bit = true;
 899         params.CRm = (hsr >> 1) & 0xf;
 900         params.Rt  = (hsr >> 5) & 0xf;
 901         params.is_write = ((hsr & 1) == 0);
 902         params.CRn = (hsr >> 10) & 0xf;
 903         params.Op0 = 0;
 904         params.Op1 = (hsr >> 14) & 0x7;
 905         params.Op2 = (hsr >> 17) & 0x7;
 906
 907         if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
 908                 return 1;
 909         if (!emulate_cp(vcpu, &params, global, nr_global))
 910                 return 1;
 911
 912         unhandled_cp_access(vcpu, &params);
 913         return 1;
 914 }
 915
 916 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
 917 {
 918         const struct sys_reg_desc *target_specific;
 919         size_t num;
 920
 921         target_specific = get_target_table(vcpu->arch.target, false, &num);
 922         return kvm_handle_cp_64(vcpu,
 923                                 cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
 924                                 target_specific, num);
 925 }
 926
 927 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
 928 {
 929         const struct sys_reg_desc *target_specific;
 930         size_t num;
 931
 932         target_specific = get_target_table(vcpu->arch.target, false, &num);
 933         return kvm_handle_cp_32(vcpu,
 934                                 cp15_regs, ARRAY_SIZE(cp15_regs),
 935                                 target_specific, num);
 936 }
 937
 938 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
 939 {
 940         return kvm_handle_cp_64(vcpu,
 941                                 cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
 942                                 NULL, 0);
 943 }
 944
 945 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
 946 {
 947         return kvm_handle_cp_32(vcpu,
 948                                 cp14_regs, ARRAY_SIZE(cp14_regs),
 949                                 NULL, 0);
 950 }
 951
 952 static int emulate_sys_reg(struct kvm_vcpu *vcpu,
 953                            const struct sys_reg_params *params)
 954 {
 955         size_t num;
 956         const struct sys_reg_desc *table, *r;
 957
 958         table = get_target_table(vcpu->arch.target, true, &num);
 959
 960         /* Search target-specific then generic table. */
 961         r = find_reg(params, table, num);
 962         if (!r)
 963                 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
 964
 965         if (likely(r)) {
 966                 /*
 967                  * Not having an accessor means that we have
 968                  * configured a trap that we don't know how to
 969                  * handle. This certainly qualifies as a gross bug
 970                  * that should be fixed right away.
 971                  */
 972                 BUG_ON(!r->access);
 973
 974                 if (likely(r->access(vcpu, params, r))) {
 975                         /* Skip instruction, since it was emulated */
 976                         kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 977                         return 1;
 978                 }
 979                 /* If access function fails, it should complain. */
 980         } else {
 981                 kvm_err("Unsupported guest sys_reg access at: %lx\n",
 982                         *vcpu_pc(vcpu));
 983                 print_sys_reg_instr(params);
 984         }
 985         kvm_inject_undefined(vcpu);
 986         return 1;
 987 }
 988
 989 static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
 990                               const struct sys_reg_desc *table, size_t num)
 991 {
 992         unsigned long i;
 993
 994         for (i = 0; i < num; i++)
 995                 if (table[i].reset)
 996                         table[i].reset(vcpu, &table[i]);
 997 }
 998
 999 /**
1000  * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
1001  * @vcpu: The VCPU pointer
1002  * @run:  The kvm_run struct
1003  */
1004 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
1005 {
1006         struct sys_reg_params params;
1007         unsigned long esr = kvm_vcpu_get_hsr(vcpu);
1008
1009         params.is_aarch32 = false;
1010         params.is_32bit = false;
1011         params.Op0 = (esr >> 20) & 3;
1012         params.Op1 = (esr >> 14) & 0x7;
1013         params.CRn = (esr >> 10) & 0xf;
1014         params.CRm = (esr >> 1) & 0xf;
1015         params.Op2 = (esr >> 17) & 0x7;
1016         params.Rt = (esr >> 5) & 0x1f;
1017         params.is_write = !(esr & 1);
1018
1019         return emulate_sys_reg(vcpu, &params);
1020 }
1021
1022 /******************************************************************************
1023  * Userspace API
1024  *****************************************************************************/
1025
1026 static bool index_to_params(u64 id, struct sys_reg_params *params)
1027 {
1028         switch (id & KVM_REG_SIZE_MASK) {
1029         case KVM_REG_SIZE_U64:
1030                 /* Any unused index bits means it's not valid. */
1031                 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
1032                               | KVM_REG_ARM_COPROC_MASK
1033                               | KVM_REG_ARM64_SYSREG_OP0_MASK
1034                               | KVM_REG_ARM64_SYSREG_OP1_MASK
1035                               | KVM_REG_ARM64_SYSREG_CRN_MASK
1036                               | KVM_REG_ARM64_SYSREG_CRM_MASK
1037                               | KVM_REG_ARM64_SYSREG_OP2_MASK))
1038                         return false;
1039                 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
1040                                >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
1041                 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
1042                                >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
1043                 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
1044                                >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
1045                 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
1046                                >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
1047                 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
1048                                >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
1049                 return true;
1050         default:
1051                 return false;
1052         }
1053 }
1054
1055 /* Decode an index value, and find the sys_reg_desc entry. */
1056 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
1057                                                     u64 id)
1058 {
1059         size_t num;
1060         const struct sys_reg_desc *table, *r;
1061         struct sys_reg_params params;
1062
1063         /* We only do sys_reg for now. */
1064         if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
1065                 return NULL;
1066
1067         if (!index_to_params(id, &params))
1068                 return NULL;
1069
1070         table = get_target_table(vcpu->arch.target, true, &num);
1071         r = find_reg(&params, table, num);
1072         if (!r)
1073                 r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1074
1075         /* Not saved in the sys_reg array? */
1076         if (r && !r->reg)
1077                 r = NULL;
1078
1079         return r;
1080 }
1081
1082 /*
1083  * These are the invariant sys_reg registers: we let the guest see the
1084  * host versions of these, so they're part of the guest state.
1085  *
1086  * A future CPU may provide a mechanism to present different values to
1087  * the guest, or a future kvm may trap them.
1088  */
1089
1090 #define FUNCTION_INVARIANT(reg)                                         \
1091         static void get_##reg(struct kvm_vcpu *v,                       \
1092                               const struct sys_reg_desc *r)             \
1093         {                                                               \
1094                 u64 val;                                                \
1095                                                                         \
1096                 asm volatile("mrs %0, " __stringify(reg) "\n"           \
1097                              : "=r" (val));                             \
1098                 ((struct sys_reg_desc *)r)->val = val;                  \
1099         }
1100
1101 FUNCTION_INVARIANT(midr_el1)
1102 FUNCTION_INVARIANT(ctr_el0)
1103 FUNCTION_INVARIANT(revidr_el1)
1104 FUNCTION_INVARIANT(id_pfr0_el1)
1105 FUNCTION_INVARIANT(id_pfr1_el1)
1106 FUNCTION_INVARIANT(id_dfr0_el1)
1107 FUNCTION_INVARIANT(id_afr0_el1)
1108 FUNCTION_INVARIANT(id_mmfr0_el1)
1109 FUNCTION_INVARIANT(id_mmfr1_el1)
1110 FUNCTION_INVARIANT(id_mmfr2_el1)
1111 FUNCTION_INVARIANT(id_mmfr3_el1)
1112 FUNCTION_INVARIANT(id_isar0_el1)
1113 FUNCTION_INVARIANT(id_isar1_el1)
1114 FUNCTION_INVARIANT(id_isar2_el1)
1115 FUNCTION_INVARIANT(id_isar3_el1)
1116 FUNCTION_INVARIANT(id_isar4_el1)
1117 FUNCTION_INVARIANT(id_isar5_el1)
1118 FUNCTION_INVARIANT(clidr_el1)
1119 FUNCTION_INVARIANT(aidr_el1)
1120
1121 /* ->val is filled in by kvm_sys_reg_table_init() */
1122 static struct sys_reg_desc invariant_sys_regs[] = {
1123         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000),
1124           NULL, get_midr_el1 },
1125         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110),
1126           NULL, get_revidr_el1 },
1127         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000),
1128           NULL, get_id_pfr0_el1 },
1129         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001),
1130           NULL, get_id_pfr1_el1 },
1131         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010),
1132           NULL, get_id_dfr0_el1 },
1133         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011),
1134           NULL, get_id_afr0_el1 },
1135         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100),
1136           NULL, get_id_mmfr0_el1 },
1137         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101),
1138           NULL, get_id_mmfr1_el1 },
1139         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110),
1140           NULL, get_id_mmfr2_el1 },
1141         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111),
1142           NULL, get_id_mmfr3_el1 },
1143         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
1144           NULL, get_id_isar0_el1 },
1145         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001),
1146           NULL, get_id_isar1_el1 },
1147         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
1148           NULL, get_id_isar2_el1 },
1149         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011),
1150           NULL, get_id_isar3_el1 },
1151         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100),
1152           NULL, get_id_isar4_el1 },
1153         { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101),
1154           NULL, get_id_isar5_el1 },
1155         { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001),
1156           NULL, get_clidr_el1 },
1157         { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111),
1158           NULL, get_aidr_el1 },
1159         { Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001),
1160           NULL, get_ctr_el0 },
1161 };
1162
1163 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
1164 {
1165         if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
1166                 return -EFAULT;
1167         return 0;
1168 }
1169
1170 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
1171 {
1172         if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
1173                 return -EFAULT;
1174         return 0;
1175 }
1176
1177 static int get_invariant_sys_reg(u64 id, void __user *uaddr)
1178 {
1179         struct sys_reg_params params;
1180         const struct sys_reg_desc *r;
1181
1182         if (!index_to_params(id, &params))
1183                 return -ENOENT;
1184
1185         r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
1186         if (!r)
1187                 return -ENOENT;
1188
1189         return reg_to_user(uaddr, &r->val, id);
1190 }
1191
1192 static int set_invariant_sys_reg(u64 id, void __user *uaddr)
1193 {
1194         struct sys_reg_params params;
1195         const struct sys_reg_desc *r;
1196         int err;
1197         u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
1198
1199         if (!index_to_params(id, &params))
1200                 return -ENOENT;
1201         r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
1202         if (!r)
1203                 return -ENOENT;
1204
1205         err = reg_from_user(&val, uaddr, id);
1206         if (err)
1207                 return err;
1208
1209         /* This is what we mean by invariant: you can't change it. */
1210         if (r->val != val)
1211                 return -EINVAL;
1212
1213         return 0;
1214 }
1215
1216 static bool is_valid_cache(u32 val)
1217 {
1218         u32 level, ctype;
1219
1220         if (val >= CSSELR_MAX)
1221                 return -ENOENT;
1222
1223         /* Bottom bit is Instruction or Data bit.  Next 3 bits are level. */
1224         level = (val >> 1);
1225         ctype = (cache_levels >> (level * 3)) & 7;
1226
1227         switch (ctype) {
1228         case 0: /* No cache */
1229                 return false;
1230         case 1: /* Instruction cache only */
1231                 return (val & 1);
1232         case 2: /* Data cache only */
1233         case 4: /* Unified cache */
1234                 return !(val & 1);
1235         case 3: /* Separate instruction and data caches */
1236                 return true;
1237         default: /* Reserved: we can't know instruction or data. */
1238                 return false;
1239         }
1240 }
1241
1242 static int demux_c15_get(u64 id, void __user *uaddr)
1243 {
1244         u32 val;
1245         u32 __user *uval = uaddr;
1246
1247         /* Fail if we have unknown bits set. */
1248         if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
1249                    | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
1250                 return -ENOENT;
1251
1252         switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
1253         case KVM_REG_ARM_DEMUX_ID_CCSIDR:
1254                 if (KVM_REG_SIZE(id) != 4)
1255                         return -ENOENT;
1256                 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
1257                         >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
1258                 if (!is_valid_cache(val))
1259                         return -ENOENT;
1260
1261                 return put_user(get_ccsidr(val), uval);
1262         default:
1263                 return -ENOENT;
1264         }
1265 }
1266
1267 static int demux_c15_set(u64 id, void __user *uaddr)
1268 {
1269         u32 val, newval;
1270         u32 __user *uval = uaddr;
1271
1272         /* Fail if we have unknown bits set. */
1273         if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
1274                    | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
1275                 return -ENOENT;
1276
1277         switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
1278         case KVM_REG_ARM_DEMUX_ID_CCSIDR:
1279                 if (KVM_REG_SIZE(id) != 4)
1280                         return -ENOENT;
1281                 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
1282                         >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
1283                 if (!is_valid_cache(val))
1284                         return -ENOENT;
1285
1286                 if (get_user(newval, uval))
1287                         return -EFAULT;
1288
1289                 /* This is also invariant: you can't change it. */
1290                 if (newval != get_ccsidr(val))
1291                         return -EINVAL;
1292                 return 0;
1293         default:
1294                 return -ENOENT;
1295         }
1296 }
1297
1298 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
1299 {
1300         const struct sys_reg_desc *r;
1301         void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1302
1303         if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
1304                 return demux_c15_get(reg->id, uaddr);
1305
1306         if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
1307                 return -ENOENT;
1308
1309         r = index_to_sys_reg_desc(vcpu, reg->id);
1310         if (!r)
1311                 return get_invariant_sys_reg(reg->id, uaddr);
1312
1313         return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id);
1314 }
1315
1316 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
1317 {
1318         const struct sys_reg_desc *r;
1319         void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1320
1321         if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
1322                 return demux_c15_set(reg->id, uaddr);
1323
1324         if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
1325                 return -ENOENT;
1326
1327         r = index_to_sys_reg_desc(vcpu, reg->id);
1328         if (!r)
1329                 return set_invariant_sys_reg(reg->id, uaddr);
1330
1331         return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
1332 }
1333
1334 static unsigned int num_demux_regs(void)
1335 {
1336         unsigned int i, count = 0;
1337
1338         for (i = 0; i < CSSELR_MAX; i++)
1339                 if (is_valid_cache(i))
1340                         count++;
1341
1342         return count;
1343 }
1344
1345 static int write_demux_regids(u64 __user *uindices)
1346 {
1347         u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
1348         unsigned int i;
1349
1350         val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
1351         for (i = 0; i < CSSELR_MAX; i++) {
1352                 if (!is_valid_cache(i))
1353                         continue;
1354                 if (put_user(val | i, uindices))
1355                         return -EFAULT;
1356                 uindices++;
1357         }
1358         return 0;
1359 }
1360
1361 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
1362 {
1363         return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
1364                 KVM_REG_ARM64_SYSREG |
1365                 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
1366                 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
1367                 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
1368                 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
1369                 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
1370 }
1371
1372 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
1373 {
1374         if (!*uind)
1375                 return true;
1376
1377         if (put_user(sys_reg_to_index(reg), *uind))
1378                 return false;
1379
1380         (*uind)++;
1381         return true;
1382 }
1383
1384 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
1385 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
1386 {
1387         const struct sys_reg_desc *i1, *i2, *end1, *end2;
1388         unsigned int total = 0;
1389         size_t num;
1390
1391         /* We check for duplicates here, to allow arch-specific overrides. */
1392         i1 = get_target_table(vcpu->arch.target, true, &num);
1393         end1 = i1 + num;
1394         i2 = sys_reg_descs;
1395         end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
1396
1397         BUG_ON(i1 == end1 || i2 == end2);
1398
1399         /* Walk carefully, as both tables may refer to the same register. */
1400         while (i1 || i2) {
1401                 int cmp = cmp_sys_reg(i1, i2);
1402                 /* target-specific overrides generic entry. */
1403                 if (cmp <= 0) {
1404                         /* Ignore registers we trap but don't save. */
1405                         if (i1->reg) {
1406                                 if (!copy_reg_to_user(i1, &uind))
1407                                         return -EFAULT;
1408                                 total++;
1409                         }
1410                 } else {
1411                         /* Ignore registers we trap but don't save. */
1412                         if (i2->reg) {
1413                                 if (!copy_reg_to_user(i2, &uind))
1414                                         return -EFAULT;
1415                                 total++;
1416                         }
1417                 }
1418
1419                 if (cmp <= 0 && ++i1 == end1)
1420                         i1 = NULL;
1421                 if (cmp >= 0 && ++i2 == end2)
1422                         i2 = NULL;
1423         }
1424         return total;
1425 }
1426
1427 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
1428 {
1429         return ARRAY_SIZE(invariant_sys_regs)
1430                 + num_demux_regs()
1431                 + walk_sys_regs(vcpu, (u64 __user *)NULL);
1432 }
1433
1434 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
1435 {
1436         unsigned int i;
1437         int err;
1438
1439         /* Then give them all the invariant registers' indices. */
1440         for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
1441                 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
1442                         return -EFAULT;
1443                 uindices++;
1444         }
1445
1446         err = walk_sys_regs(vcpu, uindices);
1447         if (err < 0)
1448                 return err;
1449         uindices += err;
1450
1451         return write_demux_regids(uindices);
1452 }
1453
1454 static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
1455 {
1456         unsigned int i;
1457
1458         for (i = 1; i < n; i++) {
1459                 if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
1460                         kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
1461                         return 1;
1462                 }
1463         }
1464
1465         return 0;
1466 }
1467
1468 void kvm_sys_reg_table_init(void)
1469 {
1470         unsigned int i;
1471         struct sys_reg_desc clidr;
1472
1473         /* Make sure tables are unique and in order. */
1474         BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
1475         BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
1476         BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
1477         BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
1478         BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
1479         BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
1480
1481         /* We abuse the reset function to overwrite the table itself. */
1482         for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
1483                 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
1484
1485         /*
1486          * CLIDR format is awkward, so clean it up.  See ARM B4.1.20:
1487          *
1488          *   If software reads the Cache Type fields from Ctype1
1489          *   upwards, once it has seen a value of 0b000, no caches
1490          *   exist at further-out levels of the hierarchy. So, for
1491          *   example, if Ctype3 is the first Cache Type field with a
1492          *   value of 0b000, the values of Ctype4 to Ctype7 must be
1493          *   ignored.
1494          */
1495         get_clidr_el1(NULL, &clidr); /* Ugly... */
1496         cache_levels = clidr.val;
1497         for (i = 0; i < 7; i++)
1498                 if (((cache_levels >> (i*3)) & 7) == 0)
1499                         break;
1500         /* Clear all higher bits. */
1501         cache_levels &= (1 << (i*3))-1;
1502 }
1503
1504 /**
1505  * kvm_reset_sys_regs - sets system registers to reset value
1506  * @vcpu: The VCPU pointer
1507  *
1508  * This function finds the right table above and sets the registers on the
1509  * virtual CPU struct to their architecturally defined reset values.
1510  */
1511 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
1512 {
1513         size_t num;
1514         const struct sys_reg_desc *table;
1515
1516         /* Catch someone adding a register without putting in reset entry. */
1517         memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));
1518
1519         /* Generic chip reset first (so target could override). */
1520         reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1521
1522         table = get_target_table(vcpu->arch.target, true, &num);
1523         reset_sys_reg_descs(vcpu, table, num);
1524
1525         for (num = 1; num < NR_SYS_REGS; num++)
1526                 if (vcpu_sys_reg(vcpu, num) == 0x4242424242424242)
1527                         panic("Didn't reset vcpu_sys_reg(%zi)", num);
1528 }