virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24
  25 #include <clocksource/arm_arch_timer.h>
  26 #include <asm/arch_timer.h>
  27
  28 #include <kvm/arm_vgic.h>
  29 #include <kvm/arm_arch_timer.h>
  30
  31 #include "trace.h"
  32
  33 static struct timecounter *timecounter;
  34 static struct workqueue_struct *wqueue;
  35 static unsigned int host_vtimer_irq;
  36
  37 static cycle_t kvm_phys_timer_read(void)
  38 {
  39         return timecounter->cc->read(timecounter->cc);
  40 }
  41
  42 static bool timer_is_armed(struct arch_timer_cpu *timer)
  43 {
  44         return timer->armed;
  45 }
  46
  47 /* timer_arm: as in "arm the timer", not as in ARM the company */
  48 static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
  49 {
  50         timer->armed = true;
  51         hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
  52                       HRTIMER_MODE_ABS);
  53 }
  54
  55 static void timer_disarm(struct arch_timer_cpu *timer)
  56 {
  57         if (timer_is_armed(timer)) {
  58                 hrtimer_cancel(&timer->timer);
  59                 cancel_work_sync(&timer->expired);
  60                 timer->armed = false;
  61         }
  62 }
  63
  64 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  65 {
  66         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  67
  68         /*
  69          * We disable the timer in the world switch and let it be
  70          * handled by kvm_timer_sync_hwstate(). Getting a timer
  71          * interrupt at this point is a sure sign of some major
  72          * breakage.
  73          */
  74         pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
  75         return IRQ_HANDLED;
  76 }
  77
  78 /*
  79  * Work function for handling the backup timer that we schedule when a vcpu is
  80  * no longer running, but had a timer programmed to fire in the future.
  81  */
  82 static void kvm_timer_inject_irq_work(struct work_struct *work)
  83 {
  84         struct kvm_vcpu *vcpu;
  85
  86         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
  87         vcpu->arch.timer_cpu.armed = false;
  88
  89         /*
  90          * If the vcpu is blocked we want to wake it up so that it will see
  91          * the timer has expired when entering the guest.
  92          */
  93         kvm_vcpu_kick(vcpu);
  94 }
  95
  96 static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
  97 {
  98         struct arch_timer_cpu *timer;
  99         timer = container_of(hrt, struct arch_timer_cpu, timer);
 100         queue_work(wqueue, &timer->expired);
 101         return HRTIMER_NORESTART;
 102 }
 103
 104 static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu)
 105 {
 106         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 107
 108         return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 109                 (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE);
 110 }
 111
 112 bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
 113 {
 114         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 115         cycle_t cval, now;
 116
 117         if (!kvm_timer_irq_can_fire(vcpu))
 118                 return false;
 119
 120         cval = timer->cntv_cval;
 121         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 122
 123         return cval <= now;
 124 }
 125
 126 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level)
 127 {
 128         int ret;
 129         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 130
 131         BUG_ON(!vgic_initialized(vcpu->kvm));
 132
 133         timer->irq.level = new_level;
 134         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->map->virt_irq,
 135                                    timer->irq.level);
 136         ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id,
 137                                          timer->map,
 138                                          timer->irq.level);
 139         WARN_ON(ret);
 140 }
 141
 142 /*
 143  * Check if there was a change in the timer state (should we raise or lower
 144  * the line level to the GIC).
 145  */
 146 static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
 147 {
 148         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 149
 150         /*
 151          * If userspace modified the timer registers via SET_ONE_REG before
 152          * the vgic was initialized, we mustn't set the timer->irq.level value
 153          * because the guest would never see the interrupt.  Instead wait
 154          * until we call this function from kvm_timer_flush_hwstate.
 155          */
 156         if (!vgic_initialized(vcpu->kvm))
 157             return;
 158
 159         if (kvm_timer_should_fire(vcpu) != timer->irq.level)
 160                 kvm_timer_update_irq(vcpu, !timer->irq.level);
 161 }
 162
 163 /*
 164  * Schedule the background timer before calling kvm_vcpu_block, so that this
 165  * thread is removed from its waitqueue and made runnable when there's a timer
 166  * interrupt to handle.
 167  */
 168 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 169 {
 170         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 171         u64 ns;
 172         cycle_t cval, now;
 173
 174         BUG_ON(timer_is_armed(timer));
 175
 176         /*
 177          * No need to schedule a background timer if the guest timer has
 178          * already expired, because kvm_vcpu_block will return before putting
 179          * the thread to sleep.
 180          */
 181         if (kvm_timer_should_fire(vcpu))
 182                 return;
 183
 184         /*
 185          * If the timer is not capable of raising interrupts (disabled or
 186          * masked), then there's no more work for us to do.
 187          */
 188         if (!kvm_timer_irq_can_fire(vcpu))
 189                 return;
 190
 191         /*  The timer has not yet expired, schedule a background timer */
 192         cval = timer->cntv_cval;
 193         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 194
 195         ns = cyclecounter_cyc2ns(timecounter->cc,
 196                                  cval - now,
 197                                  timecounter->mask,
 198                                  &timecounter->frac);
 199         timer_arm(timer, ns);
 200 }
 201
 202 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 203 {
 204         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 205         timer_disarm(timer);
 206 }
 207
 208 /**
 209  * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
 210  * @vcpu: The vcpu pointer
 211  *
 212  * Check if the virtual timer has expired while we were running in the host,
 213  * and inject an interrupt if that was the case.
 214  */
 215 void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
 216 {
 217         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 218         bool phys_active;
 219         int ret;
 220
 221         kvm_timer_update_state(vcpu);
 222
 223         /*
 224          * If we enter the guest with the virtual input level to the VGIC
 225          * asserted, then we have already told the VGIC what we need to, and
 226          * we don't need to exit from the guest until the guest deactivates
 227          * the already injected interrupt, so therefore we should set the
 228          * hardware active state to prevent unnecessary exits from the guest.
 229          *
 230          * Conversely, if the virtual input level is deasserted, then always
 231          * clear the hardware active state to ensure that hardware interrupts
 232          * from the timer triggers a guest exit.
 233          */
 234         if (timer->irq.level)
 235                 phys_active = true;
 236         else
 237                 phys_active = false;
 238
 239         ret = irq_set_irqchip_state(timer->map->irq,
 240                                     IRQCHIP_STATE_ACTIVE,
 241                                     phys_active);
 242         WARN_ON(ret);
 243 }
 244
 245 /**
 246  * kvm_timer_sync_hwstate - sync timer state from cpu
 247  * @vcpu: The vcpu pointer
 248  *
 249  * Check if the virtual timer has expired while we were running in the guest,
 250  * and inject an interrupt if that was the case.
 251  */
 252 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 253 {
 254         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 255
 256         BUG_ON(timer_is_armed(timer));
 257
 258         /*
 259          * The guest could have modified the timer registers or the timer
 260          * could have expired, update the timer state.
 261          */
 262         kvm_timer_update_state(vcpu);
 263 }
 264
 265 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
 266                          const struct kvm_irq_level *irq)
 267 {
 268         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 269         struct irq_phys_map *map;
 270
 271         /*
 272          * The vcpu timer irq number cannot be determined in
 273          * kvm_timer_vcpu_init() because it is called much before
 274          * kvm_vcpu_set_target(). To handle this, we determine
 275          * vcpu timer irq number when the vcpu is reset.
 276          */
 277         timer->irq.irq = irq->irq;
 278
 279         /*
 280          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 281          * and to 0 for ARMv7.  We provide an implementation that always
 282          * resets the timer to be disabled and unmasked and is compliant with
 283          * the ARMv7 architecture.
 284          */
 285         timer->cntv_ctl = 0;
 286         kvm_timer_update_state(vcpu);
 287
 288         /*
 289          * Tell the VGIC that the virtual interrupt is tied to a
 290          * physical interrupt. We do that once per VCPU.
 291          */
 292         map = kvm_vgic_map_phys_irq(vcpu, irq->irq, host_vtimer_irq);
 293         if (WARN_ON(IS_ERR(map)))
 294                 return PTR_ERR(map);
 295
 296         timer->map = map;
 297         return 0;
 298 }
 299
 300 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 301 {
 302         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 303
 304         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 305         hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 306         timer->timer.function = kvm_timer_expire;
 307 }
 308
 309 static void kvm_timer_init_interrupt(void *info)
 310 {
 311         enable_percpu_irq(host_vtimer_irq, 0);
 312 }
 313
 314 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 315 {
 316         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 317
 318         switch (regid) {
 319         case KVM_REG_ARM_TIMER_CTL:
 320                 timer->cntv_ctl = value;
 321                 break;
 322         case KVM_REG_ARM_TIMER_CNT:
 323                 vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value;
 324                 break;
 325         case KVM_REG_ARM_TIMER_CVAL:
 326                 timer->cntv_cval = value;
 327                 break;
 328         default:
 329                 return -1;
 330         }
 331
 332         kvm_timer_update_state(vcpu);
 333         return 0;
 334 }
 335
 336 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 337 {
 338         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 339
 340         switch (regid) {
 341         case KVM_REG_ARM_TIMER_CTL:
 342                 return timer->cntv_ctl;
 343         case KVM_REG_ARM_TIMER_CNT:
 344                 return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 345         case KVM_REG_ARM_TIMER_CVAL:
 346                 return timer->cntv_cval;
 347         }
 348         return (u64)-1;
 349 }
 350
 351 static int kvm_timer_cpu_notify(struct notifier_block *self,
 352                                 unsigned long action, void *cpu)
 353 {
 354         switch (action) {
 355         case CPU_STARTING:
 356         case CPU_STARTING_FROZEN:
 357                 kvm_timer_init_interrupt(NULL);
 358                 break;
 359         case CPU_DYING:
 360         case CPU_DYING_FROZEN:
 361                 disable_percpu_irq(host_vtimer_irq);
 362                 break;
 363         }
 364
 365         return NOTIFY_OK;
 366 }
 367
 368 static struct notifier_block kvm_timer_cpu_nb = {
 369         .notifier_call = kvm_timer_cpu_notify,
 370 };
 371
 372 static const struct of_device_id arch_timer_of_match[] = {
 373         { .compatible   = "arm,armv7-timer",    },
 374         { .compatible   = "arm,armv8-timer",    },
 375         {},
 376 };
 377
 378 int kvm_timer_hyp_init(void)
 379 {
 380         struct device_node *np;
 381         unsigned int ppi;
 382         int err;
 383
 384         timecounter = arch_timer_get_timecounter();
 385         if (!timecounter)
 386                 return -ENODEV;
 387
 388         np = of_find_matching_node(NULL, arch_timer_of_match);
 389         if (!np) {
 390                 kvm_err("kvm_arch_timer: can't find DT node\n");
 391                 return -ENODEV;
 392         }
 393
 394         ppi = irq_of_parse_and_map(np, 2);
 395         if (!ppi) {
 396                 kvm_err("kvm_arch_timer: no virtual timer interrupt\n");
 397                 err = -EINVAL;
 398                 goto out;
 399         }
 400
 401         err = request_percpu_irq(ppi, kvm_arch_timer_handler,
 402                                  "kvm guest timer", kvm_get_running_vcpus());
 403         if (err) {
 404                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 405                         ppi, err);
 406                 goto out;
 407         }
 408
 409         host_vtimer_irq = ppi;
 410
 411         err = __register_cpu_notifier(&kvm_timer_cpu_nb);
 412         if (err) {
 413                 kvm_err("Cannot register timer CPU notifier\n");
 414                 goto out_free;
 415         }
 416
 417         wqueue = create_singlethread_workqueue("kvm_arch_timer");
 418         if (!wqueue) {
 419                 err = -ENOMEM;
 420                 goto out_free;
 421         }
 422
 423         kvm_info("%s IRQ%d\n", np->name, ppi);
 424         on_each_cpu(kvm_timer_init_interrupt, NULL, 1);
 425
 426         goto out;
 427 out_free:
 428         free_percpu_irq(ppi, kvm_get_running_vcpus());
 429 out:
 430         of_node_put(np);
 431         return err;
 432 }
 433
 434 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 435 {
 436         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 437
 438         timer_disarm(timer);
 439         if (timer->map)
 440                 kvm_vgic_unmap_phys_irq(vcpu, timer->map);
 441 }
 442
 443 void kvm_timer_enable(struct kvm *kvm)
 444 {
 445         if (kvm->arch.timer.enabled)
 446                 return;
 447
 448         /*
 449          * There is a potential race here between VCPUs starting for the first
 450          * time, which may be enabling the timer multiple times.  That doesn't
 451          * hurt though, because we're just setting a variable to the same
 452          * variable that it already was.  The important thing is that all
 453          * VCPUs have the enabled variable set, before entering the guest, if
 454          * the arch timers are enabled.
 455          */
 456         if (timecounter && wqueue)
 457                 kvm->arch.timer.enabled = 1;
 458 }
 459
 460 void kvm_timer_init(struct kvm *kvm)
 461 {
 462         kvm->arch.timer.cntvoff = kvm_phys_timer_read();
 463 }