virt/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include "iodev.h"
  20
  21 #include <linux/kvm_host.h>
  22 #include <linux/kvm.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/percpu.h>
  26 #include <linux/mm.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/vmalloc.h>
  29 #include <linux/reboot.h>
  30 #include <linux/debugfs.h>
  31 #include <linux/highmem.h>
  32 #include <linux/file.h>
  33 #include <linux/syscore_ops.h>
  34 #include <linux/cpu.h>
  35 #include <linux/sched.h>
  36 #include <linux/cpumask.h>
  37 #include <linux/smp.h>
  38 #include <linux/anon_inodes.h>
  39 #include <linux/profile.h>
  40 #include <linux/kvm_para.h>
  41 #include <linux/pagemap.h>
  42 #include <linux/mman.h>
  43 #include <linux/swap.h>
  44 #include <linux/bitops.h>
  45 #include <linux/spinlock.h>
  46 #include <linux/compat.h>
  47 #include <linux/srcu.h>
  48 #include <linux/hugetlb.h>
  49 #include <linux/slab.h>
  50 #include <linux/sort.h>
  51 #include <linux/bsearch.h>
  52
  53 #include <asm/processor.h>
  54 #include <asm/io.h>
  55 #include <asm/uaccess.h>
  56 #include <asm/pgtable.h>
  57
  58 #include "coalesced_mmio.h"
  59 #include "async_pf.h"
  60
  61 #define CREATE_TRACE_POINTS
  62 #include <trace/events/kvm.h>
  63
  64 MODULE_AUTHOR("Qumranet");
  65 MODULE_LICENSE("GPL");
  66
  67 /*
  68  * Ordering of locks:
  69  *
  70  *              kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  71  */
  72
  73 DEFINE_SPINLOCK(kvm_lock);
  74 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
  75 LIST_HEAD(vm_list);
  76
  77 static cpumask_var_t cpus_hardware_enabled;
  78 static int kvm_usage_count = 0;
  79 static atomic_t hardware_enable_failed;
  80
  81 struct kmem_cache *kvm_vcpu_cache;
  82 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  83
  84 static __read_mostly struct preempt_ops kvm_preempt_ops;
  85
  86 struct dentry *kvm_debugfs_dir;
  87
  88 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  89                            unsigned long arg);
  90 #ifdef CONFIG_COMPAT
  91 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
  92                                   unsigned long arg);
  93 #endif
  94 static int hardware_enable_all(void);
  95 static void hardware_disable_all(void);
  96
  97 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
  98 static void update_memslots(struct kvm_memslots *slots,
  99                             struct kvm_memory_slot *new, u64 last_generation);
 100
 101 static void kvm_release_pfn_dirty(pfn_t pfn);
 102 static void mark_page_dirty_in_slot(struct kvm *kvm,
 103                                     struct kvm_memory_slot *memslot, gfn_t gfn);
 104
 105 __visible bool kvm_rebooting;
 106 EXPORT_SYMBOL_GPL(kvm_rebooting);
 107
 108 static bool largepages_enabled = true;
 109
 110 bool kvm_is_mmio_pfn(pfn_t pfn)
 111 {
 112         if (pfn_valid(pfn))
 113                 return PageReserved(pfn_to_page(pfn));
 114
 115         return true;
 116 }
 117
 118 /*
 119  * Switches to specified vcpu, until a matching vcpu_put()
 120  */
 121 int vcpu_load(struct kvm_vcpu *vcpu)
 122 {
 123         int cpu;
 124
 125         if (mutex_lock_killable(&vcpu->mutex))
 126                 return -EINTR;
 127         if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
 128                 /* The thread running this VCPU changed. */
 129                 struct pid *oldpid = vcpu->pid;
 130                 struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
 131                 rcu_assign_pointer(vcpu->pid, newpid);
 132                 if (oldpid)
 133                         synchronize_rcu();
 134                 put_pid(oldpid);
 135         }
 136         cpu = get_cpu();
 137         preempt_notifier_register(&vcpu->preempt_notifier);
 138         kvm_arch_vcpu_load(vcpu, cpu);
 139         put_cpu();
 140         return 0;
 141 }
 142
 143 void vcpu_put(struct kvm_vcpu *vcpu)
 144 {
 145         preempt_disable();
 146         kvm_arch_vcpu_put(vcpu);
 147         preempt_notifier_unregister(&vcpu->preempt_notifier);
 148         preempt_enable();
 149         mutex_unlock(&vcpu->mutex);
 150 }
 151
 152 static void ack_flush(void *_completed)
 153 {
 154 }
 155
 156 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
 157 {
 158         int i, cpu, me;
 159         cpumask_var_t cpus;
 160         bool called = true;
 161         struct kvm_vcpu *vcpu;
 162
 163         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
 164
 165         me = get_cpu();
 166         kvm_for_each_vcpu(i, vcpu, kvm) {
 167                 kvm_make_request(req, vcpu);
 168                 cpu = vcpu->cpu;
 169
 170                 /* Set ->requests bit before we read ->mode */
 171                 smp_mb();
 172
 173                 if (cpus != NULL && cpu != -1 && cpu != me &&
 174                       kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
 175                         cpumask_set_cpu(cpu, cpus);
 176         }
 177         if (unlikely(cpus == NULL))
 178                 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
 179         else if (!cpumask_empty(cpus))
 180                 smp_call_function_many(cpus, ack_flush, NULL, 1);
 181         else
 182                 called = false;
 183         put_cpu();
 184         free_cpumask_var(cpus);
 185         return called;
 186 }
 187
 188 void kvm_flush_remote_tlbs(struct kvm *kvm)
 189 {
 190         long dirty_count = kvm->tlbs_dirty;
 191
 192         smp_mb();
 193         if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
 194                 ++kvm->stat.remote_tlb_flush;
 195         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 196 }
 197
 198 void kvm_reload_remote_mmus(struct kvm *kvm)
 199 {
 200         make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
 201 }
 202
 203 void kvm_make_mclock_inprogress_request(struct kvm *kvm)
 204 {
 205         make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
 206 }
 207
 208 void kvm_make_scan_ioapic_request(struct kvm *kvm)
 209 {
 210         make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
 211 }
 212
 213 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 214 {
 215         struct page *page;
 216         int r;
 217
 218         mutex_init(&vcpu->mutex);
 219         vcpu->cpu = -1;
 220         vcpu->kvm = kvm;
 221         vcpu->vcpu_id = id;
 222         vcpu->pid = NULL;
 223         init_waitqueue_head(&vcpu->wq);
 224         kvm_async_pf_vcpu_init(vcpu);
 225
 226         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 227         if (!page) {
 228                 r = -ENOMEM;
 229                 goto fail;
 230         }
 231         vcpu->run = page_address(page);
 232
 233         kvm_vcpu_set_in_spin_loop(vcpu, false);
 234         kvm_vcpu_set_dy_eligible(vcpu, false);
 235         vcpu->preempted = false;
 236
 237         r = kvm_arch_vcpu_init(vcpu);
 238         if (r < 0)
 239                 goto fail_free_run;
 240         return 0;
 241
 242 fail_free_run:
 243         free_page((unsigned long)vcpu->run);
 244 fail:
 245         return r;
 246 }
 247 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 248
 249 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 250 {
 251         put_pid(vcpu->pid);
 252         kvm_arch_vcpu_uninit(vcpu);
 253         free_page((unsigned long)vcpu->run);
 254 }
 255 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 256
 257 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 258 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 259 {
 260         return container_of(mn, struct kvm, mmu_notifier);
 261 }
 262
 263 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 264                                              struct mm_struct *mm,
 265                                              unsigned long address)
 266 {
 267         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 268         int need_tlb_flush, idx;
 269
 270         /*
 271          * When ->invalidate_page runs, the linux pte has been zapped
 272          * already but the page is still allocated until
 273          * ->invalidate_page returns. So if we increase the sequence
 274          * here the kvm page fault will notice if the spte can't be
 275          * established because the page is going to be freed. If
 276          * instead the kvm page fault establishes the spte before
 277          * ->invalidate_page runs, kvm_unmap_hva will release it
 278          * before returning.
 279          *
 280          * The sequence increase only need to be seen at spin_unlock
 281          * time, and not at spin_lock time.
 282          *
 283          * Increasing the sequence after the spin_unlock would be
 284          * unsafe because the kvm page fault could then establish the
 285          * pte after kvm_unmap_hva returned, without noticing the page
 286          * is going to be freed.
 287          */
 288         idx = srcu_read_lock(&kvm->srcu);
 289         spin_lock(&kvm->mmu_lock);
 290
 291         kvm->mmu_notifier_seq++;
 292         need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
 293         /* we've to flush the tlb before the pages can be freed */
 294         if (need_tlb_flush)
 295                 kvm_flush_remote_tlbs(kvm);
 296
 297         spin_unlock(&kvm->mmu_lock);
 298         srcu_read_unlock(&kvm->srcu, idx);
 299 }
 300
 301 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 302                                         struct mm_struct *mm,
 303                                         unsigned long address,
 304                                         pte_t pte)
 305 {
 306         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 307         int idx;
 308
 309         idx = srcu_read_lock(&kvm->srcu);
 310         spin_lock(&kvm->mmu_lock);
 311         kvm->mmu_notifier_seq++;
 312         kvm_set_spte_hva(kvm, address, pte);
 313         spin_unlock(&kvm->mmu_lock);
 314         srcu_read_unlock(&kvm->srcu, idx);
 315 }
 316
 317 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 318                                                     struct mm_struct *mm,
 319                                                     unsigned long start,
 320                                                     unsigned long end)
 321 {
 322         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 323         int need_tlb_flush = 0, idx;
 324
 325         idx = srcu_read_lock(&kvm->srcu);
 326         spin_lock(&kvm->mmu_lock);
 327         /*
 328          * The count increase must become visible at unlock time as no
 329          * spte can be established without taking the mmu_lock and
 330          * count is also read inside the mmu_lock critical section.
 331          */
 332         kvm->mmu_notifier_count++;
 333         need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
 334         need_tlb_flush |= kvm->tlbs_dirty;
 335         /* we've to flush the tlb before the pages can be freed */
 336         if (need_tlb_flush)
 337                 kvm_flush_remote_tlbs(kvm);
 338
 339         spin_unlock(&kvm->mmu_lock);
 340         srcu_read_unlock(&kvm->srcu, idx);
 341 }
 342
 343 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 344                                                   struct mm_struct *mm,
 345                                                   unsigned long start,
 346                                                   unsigned long end)
 347 {
 348         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 349
 350         spin_lock(&kvm->mmu_lock);
 351         /*
 352          * This sequence increase will notify the kvm page fault that
 353          * the page that is going to be mapped in the spte could have
 354          * been freed.
 355          */
 356         kvm->mmu_notifier_seq++;
 357         smp_wmb();
 358         /*
 359          * The above sequence increase must be visible before the
 360          * below count decrease, which is ensured by the smp_wmb above
 361          * in conjunction with the smp_rmb in mmu_notifier_retry().
 362          */
 363         kvm->mmu_notifier_count--;
 364         spin_unlock(&kvm->mmu_lock);
 365
 366         BUG_ON(kvm->mmu_notifier_count < 0);
 367 }
 368
 369 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 370                                               struct mm_struct *mm,
 371                                               unsigned long address)
 372 {
 373         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 374         int young, idx;
 375
 376         idx = srcu_read_lock(&kvm->srcu);
 377         spin_lock(&kvm->mmu_lock);
 378
 379         young = kvm_age_hva(kvm, address);
 380         if (young)
 381                 kvm_flush_remote_tlbs(kvm);
 382
 383         spin_unlock(&kvm->mmu_lock);
 384         srcu_read_unlock(&kvm->srcu, idx);
 385
 386         return young;
 387 }
 388
 389 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
 390                                        struct mm_struct *mm,
 391                                        unsigned long address)
 392 {
 393         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 394         int young, idx;
 395
 396         idx = srcu_read_lock(&kvm->srcu);
 397         spin_lock(&kvm->mmu_lock);
 398         young = kvm_test_age_hva(kvm, address);
 399         spin_unlock(&kvm->mmu_lock);
 400         srcu_read_unlock(&kvm->srcu, idx);
 401
 402         return young;
 403 }
 404
 405 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 406                                      struct mm_struct *mm)
 407 {
 408         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 409         int idx;
 410
 411         idx = srcu_read_lock(&kvm->srcu);
 412         kvm_arch_flush_shadow_all(kvm);
 413         srcu_read_unlock(&kvm->srcu, idx);
 414 }
 415
 416 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 417         .invalidate_page        = kvm_mmu_notifier_invalidate_page,
 418         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
 419         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
 420         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
 421         .test_young             = kvm_mmu_notifier_test_young,
 422         .change_pte             = kvm_mmu_notifier_change_pte,
 423         .release                = kvm_mmu_notifier_release,
 424 };
 425
 426 static int kvm_init_mmu_notifier(struct kvm *kvm)
 427 {
 428         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 429         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 430 }
 431
 432 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
 433
 434 static int kvm_init_mmu_notifier(struct kvm *kvm)
 435 {
 436         return 0;
 437 }
 438
 439 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 440
 441 static void kvm_init_memslots_id(struct kvm *kvm)
 442 {
 443         int i;
 444         struct kvm_memslots *slots = kvm->memslots;
 445
 446         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
 447                 slots->id_to_index[i] = slots->memslots[i].id = i;
 448 }
 449
 450 static struct kvm *kvm_create_vm(unsigned long type)
 451 {
 452         int r, i;
 453         struct kvm *kvm = kvm_arch_alloc_vm();
 454
 455         if (!kvm)
 456                 return ERR_PTR(-ENOMEM);
 457
 458         r = kvm_arch_init_vm(kvm, type);
 459         if (r)
 460                 goto out_err_no_disable;
 461
 462         r = hardware_enable_all();
 463         if (r)
 464                 goto out_err_no_disable;
 465
 466 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 467         INIT_HLIST_HEAD(&kvm->mask_notifier_list);
 468 #endif
 469 #ifdef CONFIG_HAVE_KVM_IRQFD
 470         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
 471 #endif
 472
 473         BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
 474
 475         r = -ENOMEM;
 476         kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 477         if (!kvm->memslots)
 478                 goto out_err_no_srcu;
 479         kvm_init_memslots_id(kvm);
 480         if (init_srcu_struct(&kvm->srcu))
 481                 goto out_err_no_srcu;
 482         if (init_srcu_struct(&kvm->irq_srcu))
 483                 goto out_err_no_irq_srcu;
 484         for (i = 0; i < KVM_NR_BUSES; i++) {
 485                 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
 486                                         GFP_KERNEL);
 487                 if (!kvm->buses[i])
 488                         goto out_err;
 489         }
 490
 491         spin_lock_init(&kvm->mmu_lock);
 492         kvm->mm = current->mm;
 493         atomic_inc(&kvm->mm->mm_count);
 494         kvm_eventfd_init(kvm);
 495         mutex_init(&kvm->lock);
 496         mutex_init(&kvm->irq_lock);
 497         mutex_init(&kvm->slots_lock);
 498         atomic_set(&kvm->users_count, 1);
 499         INIT_LIST_HEAD(&kvm->devices);
 500
 501         r = kvm_init_mmu_notifier(kvm);
 502         if (r)
 503                 goto out_err;
 504
 505         spin_lock(&kvm_lock);
 506         list_add(&kvm->vm_list, &vm_list);
 507         spin_unlock(&kvm_lock);
 508
 509         return kvm;
 510
 511 out_err:
 512         cleanup_srcu_struct(&kvm->irq_srcu);
 513 out_err_no_irq_srcu:
 514         cleanup_srcu_struct(&kvm->srcu);
 515 out_err_no_srcu:
 516         hardware_disable_all();
 517 out_err_no_disable:
 518         for (i = 0; i < KVM_NR_BUSES; i++)
 519                 kfree(kvm->buses[i]);
 520         kfree(kvm->memslots);
 521         kvm_arch_free_vm(kvm);
 522         return ERR_PTR(r);
 523 }
 524
 525 /*
 526  * Avoid using vmalloc for a small buffer.
 527  * Should not be used when the size is statically known.
 528  */
 529 void *kvm_kvzalloc(unsigned long size)
 530 {
 531         if (size > PAGE_SIZE)
 532                 return vzalloc(size);
 533         else
 534                 return kzalloc(size, GFP_KERNEL);
 535 }
 536
 537 void kvm_kvfree(const void *addr)
 538 {
 539         if (is_vmalloc_addr(addr))
 540                 vfree(addr);
 541         else
 542                 kfree(addr);
 543 }
 544
 545 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 546 {
 547         if (!memslot->dirty_bitmap)
 548                 return;
 549
 550         kvm_kvfree(memslot->dirty_bitmap);
 551         memslot->dirty_bitmap = NULL;
 552 }
 553
 554 /*
 555  * Free any memory in @free but not in @dont.
 556  */
 557 static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free,
 558                                   struct kvm_memory_slot *dont)
 559 {
 560         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 561                 kvm_destroy_dirty_bitmap(free);
 562
 563         kvm_arch_free_memslot(kvm, free, dont);
 564
 565         free->npages = 0;
 566 }
 567
 568 static void kvm_free_physmem(struct kvm *kvm)
 569 {
 570         struct kvm_memslots *slots = kvm->memslots;
 571         struct kvm_memory_slot *memslot;
 572
 573         kvm_for_each_memslot(memslot, slots)
 574                 kvm_free_physmem_slot(kvm, memslot, NULL);
 575
 576         kfree(kvm->memslots);
 577 }
 578
 579 static void kvm_destroy_devices(struct kvm *kvm)
 580 {
 581         struct list_head *node, *tmp;
 582
 583         list_for_each_safe(node, tmp, &kvm->devices) {
 584                 struct kvm_device *dev =
 585                         list_entry(node, struct kvm_device, vm_node);
 586
 587                 list_del(node);
 588                 dev->ops->destroy(dev);
 589         }
 590 }
 591
 592 static void kvm_destroy_vm(struct kvm *kvm)
 593 {
 594         int i;
 595         struct mm_struct *mm = kvm->mm;
 596
 597         kvm_arch_sync_events(kvm);
 598         spin_lock(&kvm_lock);
 599         list_del(&kvm->vm_list);
 600         spin_unlock(&kvm_lock);
 601         kvm_free_irq_routing(kvm);
 602         for (i = 0; i < KVM_NR_BUSES; i++)
 603                 kvm_io_bus_destroy(kvm->buses[i]);
 604         kvm_coalesced_mmio_free(kvm);
 605 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 606         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 607 #else
 608         kvm_arch_flush_shadow_all(kvm);
 609 #endif
 610         kvm_arch_destroy_vm(kvm);
 611         kvm_destroy_devices(kvm);
 612         kvm_free_physmem(kvm);
 613         cleanup_srcu_struct(&kvm->irq_srcu);
 614         cleanup_srcu_struct(&kvm->srcu);
 615         kvm_arch_free_vm(kvm);
 616         hardware_disable_all();
 617         mmdrop(mm);
 618 }
 619
 620 void kvm_get_kvm(struct kvm *kvm)
 621 {
 622         atomic_inc(&kvm->users_count);
 623 }
 624 EXPORT_SYMBOL_GPL(kvm_get_kvm);
 625
 626 void kvm_put_kvm(struct kvm *kvm)
 627 {
 628         if (atomic_dec_and_test(&kvm->users_count))
 629                 kvm_destroy_vm(kvm);
 630 }
 631 EXPORT_SYMBOL_GPL(kvm_put_kvm);
 632
 633
 634 static int kvm_vm_release(struct inode *inode, struct file *filp)
 635 {
 636         struct kvm *kvm = filp->private_data;
 637
 638         kvm_irqfd_release(kvm);
 639
 640         kvm_put_kvm(kvm);
 641         return 0;
 642 }
 643
 644 /*
 645  * Allocation size is twice as large as the actual dirty bitmap size.
 646  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
 647  */
 648 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
 649 {
 650 #ifndef CONFIG_S390
 651         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
 652
 653         memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
 654         if (!memslot->dirty_bitmap)
 655                 return -ENOMEM;
 656
 657 #endif /* !CONFIG_S390 */
 658         return 0;
 659 }
 660
 661 static int cmp_memslot(const void *slot1, const void *slot2)
 662 {
 663         struct kvm_memory_slot *s1, *s2;
 664
 665         s1 = (struct kvm_memory_slot *)slot1;
 666         s2 = (struct kvm_memory_slot *)slot2;
 667
 668         if (s1->npages < s2->npages)
 669                 return 1;
 670         if (s1->npages > s2->npages)
 671                 return -1;
 672
 673         return 0;
 674 }
 675
 676 /*
 677  * Sort the memslots base on its size, so the larger slots
 678  * will get better fit.
 679  */
 680 static void sort_memslots(struct kvm_memslots *slots)
 681 {
 682         int i;
 683
 684         sort(slots->memslots, KVM_MEM_SLOTS_NUM,
 685               sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
 686
 687         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
 688                 slots->id_to_index[slots->memslots[i].id] = i;
 689 }
 690
 691 static void update_memslots(struct kvm_memslots *slots,
 692                             struct kvm_memory_slot *new,
 693                             u64 last_generation)
 694 {
 695         if (new) {
 696                 int id = new->id;
 697                 struct kvm_memory_slot *old = id_to_memslot(slots, id);
 698                 unsigned long npages = old->npages;
 699
 700                 *old = *new;
 701                 if (new->npages != npages)
 702                         sort_memslots(slots);
 703         }
 704
 705         slots->generation = last_generation + 1;
 706 }
 707
 708 static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
 709 {
 710         u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
 711
 712 #ifdef __KVM_HAVE_READONLY_MEM
 713         valid_flags |= KVM_MEM_READONLY;
 714 #endif
 715
 716         if (mem->flags & ~valid_flags)
 717                 return -EINVAL;
 718
 719         return 0;
 720 }
 721
 722 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
 723                 struct kvm_memslots *slots, struct kvm_memory_slot *new)
 724 {
 725         struct kvm_memslots *old_memslots = kvm->memslots;
 726
 727         update_memslots(slots, new, kvm->memslots->generation);
 728         rcu_assign_pointer(kvm->memslots, slots);
 729         synchronize_srcu_expedited(&kvm->srcu);
 730
 731         kvm_arch_memslots_updated(kvm);
 732
 733         return old_memslots;
 734 }
 735
 736 /*
 737  * Allocate some memory and give it an address in the guest physical address
 738  * space.
 739  *
 740  * Discontiguous memory is allowed, mostly for framebuffers.
 741  *
 742  * Must be called holding mmap_sem for write.
 743  */
 744 int __kvm_set_memory_region(struct kvm *kvm,
 745                             struct kvm_userspace_memory_region *mem)
 746 {
 747         int r;
 748         gfn_t base_gfn;
 749         unsigned long npages;
 750         struct kvm_memory_slot *slot;
 751         struct kvm_memory_slot old, new;
 752         struct kvm_memslots *slots = NULL, *old_memslots;
 753         enum kvm_mr_change change;
 754
 755         r = check_memory_region_flags(mem);
 756         if (r)
 757                 goto out;
 758
 759         r = -EINVAL;
 760         /* General sanity checks */
 761         if (mem->memory_size & (PAGE_SIZE - 1))
 762                 goto out;
 763         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 764                 goto out;
 765         /* We can read the guest memory with __xxx_user() later on. */
 766         if ((mem->slot < KVM_USER_MEM_SLOTS) &&
 767             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
 768              !access_ok(VERIFY_WRITE,
 769                         (void __user *)(unsigned long)mem->userspace_addr,
 770                         mem->memory_size)))
 771                 goto out;
 772         if (mem->slot >= KVM_MEM_SLOTS_NUM)
 773                 goto out;
 774         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 775                 goto out;
 776
 777         slot = id_to_memslot(kvm->memslots, mem->slot);
 778         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 779         npages = mem->memory_size >> PAGE_SHIFT;
 780
 781         if (npages > KVM_MEM_MAX_NR_PAGES)
 782                 goto out;
 783
 784         if (!npages)
 785                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 786
 787         new = old = *slot;
 788
 789         new.id = mem->slot;
 790         new.base_gfn = base_gfn;
 791         new.npages = npages;
 792         new.flags = mem->flags;
 793
 794         if (npages) {
 795                 if (!old.npages)
 796                         change = KVM_MR_CREATE;
 797                 else { /* Modify an existing slot. */
 798                         if ((mem->userspace_addr != old.userspace_addr) ||
 799                             (npages != old.npages) ||
 800                             ((new.flags ^ old.flags) & KVM_MEM_READONLY))
 801                                 goto out;
 802
 803                         if (base_gfn != old.base_gfn)
 804                                 change = KVM_MR_MOVE;
 805                         else if (new.flags != old.flags)
 806                                 change = KVM_MR_FLAGS_ONLY;
 807                         else { /* Nothing to change. */
 808                                 r = 0;
 809                                 goto out;
 810                         }
 811                 }
 812         } else if (old.npages) {
 813                 change = KVM_MR_DELETE;
 814         } else /* Modify a non-existent slot: disallowed. */
 815                 goto out;
 816
 817         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
 818                 /* Check for overlaps */
 819                 r = -EEXIST;
 820                 kvm_for_each_memslot(slot, kvm->memslots) {
 821                         if ((slot->id >= KVM_USER_MEM_SLOTS) ||
 822                             (slot->id == mem->slot))
 823                                 continue;
 824                         if (!((base_gfn + npages <= slot->base_gfn) ||
 825                               (base_gfn >= slot->base_gfn + slot->npages)))
 826                                 goto out;
 827                 }
 828         }
 829
 830         /* Free page dirty bitmap if unneeded */
 831         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 832                 new.dirty_bitmap = NULL;
 833
 834         r = -ENOMEM;
 835         if (change == KVM_MR_CREATE) {
 836                 new.userspace_addr = mem->userspace_addr;
 837
 838                 if (kvm_arch_create_memslot(kvm, &new, npages))
 839                         goto out_free;
 840         }
 841
 842         /* Allocate page dirty bitmap if needed */
 843         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 844                 if (kvm_create_dirty_bitmap(&new) < 0)
 845                         goto out_free;
 846         }
 847
 848         if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
 849                 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
 850                                 GFP_KERNEL);
 851                 if (!slots)
 852                         goto out_free;
 853                 slot = id_to_memslot(slots, mem->slot);
 854                 slot->flags |= KVM_MEMSLOT_INVALID;
 855
 856                 old_memslots = install_new_memslots(kvm, slots, NULL);
 857
 858                 /* slot was deleted or moved, clear iommu mapping */
 859                 kvm_iommu_unmap_pages(kvm, &old);
 860                 /* From this point no new shadow pages pointing to a deleted,
 861                  * or moved, memslot will be created.
 862                  *
 863                  * validation of sp->gfn happens in:
 864                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
 865                  *      - kvm_is_visible_gfn (mmu_check_roots)
 866                  */
 867                 kvm_arch_flush_shadow_memslot(kvm, slot);
 868                 slots = old_memslots;
 869         }
 870
 871         r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
 872         if (r)
 873                 goto out_slots;
 874
 875         r = -ENOMEM;
 876         /*
 877          * We can re-use the old_memslots from above, the only difference
 878          * from the currently installed memslots is the invalid flag.  This
 879          * will get overwritten by update_memslots anyway.
 880          */
 881         if (!slots) {
 882                 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
 883                                 GFP_KERNEL);
 884                 if (!slots)
 885                         goto out_free;
 886         }
 887
 888         /* actual memory is freed via old in kvm_free_physmem_slot below */
 889         if (change == KVM_MR_DELETE) {
 890                 new.dirty_bitmap = NULL;
 891                 memset(&new.arch, 0, sizeof(new.arch));
 892         }
 893
 894         old_memslots = install_new_memslots(kvm, slots, &new);
 895
 896         kvm_arch_commit_memory_region(kvm, mem, &old, change);
 897
 898         kvm_free_physmem_slot(kvm, &old, &new);
 899         kfree(old_memslots);
 900
 901         /*
 902          * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
 903          * un-mapped and re-mapped if their base changes.  Since base change
 904          * unmapping is handled above with slot deletion, mapping alone is
 905          * needed here.  Anything else the iommu might care about for existing
 906          * slots (size changes, userspace addr changes and read-only flag
 907          * changes) is disallowed above, so any other attribute changes getting
 908          * here can be skipped.
 909          */
 910         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
 911                 r = kvm_iommu_map_pages(kvm, &new);
 912                 return r;
 913         }
 914
 915         return 0;
 916
 917 out_slots:
 918         kfree(slots);
 919 out_free:
 920         kvm_free_physmem_slot(kvm, &new, &old);
 921 out:
 922         return r;
 923 }
 924 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 925
 926 int kvm_set_memory_region(struct kvm *kvm,
 927                           struct kvm_userspace_memory_region *mem)
 928 {
 929         int r;
 930
 931         mutex_lock(&kvm->slots_lock);
 932         r = __kvm_set_memory_region(kvm, mem);
 933         mutex_unlock(&kvm->slots_lock);
 934         return r;
 935 }
 936 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 937
 938 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 939                                           struct kvm_userspace_memory_region *mem)
 940 {
 941         if (mem->slot >= KVM_USER_MEM_SLOTS)
 942                 return -EINVAL;
 943         return kvm_set_memory_region(kvm, mem);
 944 }
 945
 946 int kvm_get_dirty_log(struct kvm *kvm,
 947                         struct kvm_dirty_log *log, int *is_dirty)
 948 {
 949         struct kvm_memory_slot *memslot;
 950         int r, i;
 951         unsigned long n;
 952         unsigned long any = 0;
 953
 954         r = -EINVAL;
 955         if (log->slot >= KVM_USER_MEM_SLOTS)
 956                 goto out;
 957
 958         memslot = id_to_memslot(kvm->memslots, log->slot);
 959         r = -ENOENT;
 960         if (!memslot->dirty_bitmap)
 961                 goto out;
 962
 963         n = kvm_dirty_bitmap_bytes(memslot);
 964
 965         for (i = 0; !any && i < n/sizeof(long); ++i)
 966                 any = memslot->dirty_bitmap[i];
 967
 968         r = -EFAULT;
 969         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 970                 goto out;
 971
 972         if (any)
 973                 *is_dirty = 1;
 974
 975         r = 0;
 976 out:
 977         return r;
 978 }
 979
 980 bool kvm_largepages_enabled(void)
 981 {
 982         return largepages_enabled;
 983 }
 984
 985 void kvm_disable_largepages(void)
 986 {
 987         largepages_enabled = false;
 988 }
 989 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
 990
 991 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 992 {
 993         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
 994 }
 995 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 996
 997 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 998 {
 999         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1000
1001         if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1002               memslot->flags & KVM_MEMSLOT_INVALID)
1003                 return 0;
1004
1005         return 1;
1006 }
1007 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1008
1009 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1010 {
1011         struct vm_area_struct *vma;
1012         unsigned long addr, size;
1013
1014         size = PAGE_SIZE;
1015
1016         addr = gfn_to_hva(kvm, gfn);
1017         if (kvm_is_error_hva(addr))
1018                 return PAGE_SIZE;
1019
1020         down_read(&current->mm->mmap_sem);
1021         vma = find_vma(current->mm, addr);
1022         if (!vma)
1023                 goto out;
1024
1025         size = vma_kernel_pagesize(vma);
1026
1027 out:
1028         up_read(&current->mm->mmap_sem);
1029
1030         return size;
1031 }
1032
1033 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1034 {
1035         return slot->flags & KVM_MEM_READONLY;
1036 }
1037
1038 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1039                                        gfn_t *nr_pages, bool write)
1040 {
1041         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1042                 return KVM_HVA_ERR_BAD;
1043
1044         if (memslot_is_readonly(slot) && write)
1045                 return KVM_HVA_ERR_RO_BAD;
1046
1047         if (nr_pages)
1048                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1049
1050         return __gfn_to_hva_memslot(slot, gfn);
1051 }
1052
1053 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1054                                      gfn_t *nr_pages)
1055 {
1056         return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1057 }
1058
1059 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1060                                         gfn_t gfn)
1061 {
1062         return gfn_to_hva_many(slot, gfn, NULL);
1063 }
1064 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1065
1066 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1067 {
1068         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1069 }
1070 EXPORT_SYMBOL_GPL(gfn_to_hva);
1071
1072 /*
1073  * If writable is set to false, the hva returned by this function is only
1074  * allowed to be read.
1075  */
1076 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1077                                       gfn_t gfn, bool *writable)
1078 {
1079         unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1080
1081         if (!kvm_is_error_hva(hva) && writable)
1082                 *writable = !memslot_is_readonly(slot);
1083
1084         return hva;
1085 }
1086
1087 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1088 {
1089         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1090
1091         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1092 }
1093
1094 static int kvm_read_hva(void *data, void __user *hva, int len)
1095 {
1096         return __copy_from_user(data, hva, len);
1097 }
1098
1099 static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
1100 {
1101         return __copy_from_user_inatomic(data, hva, len);
1102 }
1103
1104 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1105         unsigned long start, int write, struct page **page)
1106 {
1107         int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1108
1109         if (write)
1110                 flags |= FOLL_WRITE;
1111
1112         return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1113 }
1114
1115 static inline int check_user_page_hwpoison(unsigned long addr)
1116 {
1117         int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1118
1119         rc = __get_user_pages(current, current->mm, addr, 1,
1120                               flags, NULL, NULL, NULL);
1121         return rc == -EHWPOISON;
1122 }
1123
1124 /*
1125  * The atomic path to get the writable pfn which will be stored in @pfn,
1126  * true indicates success, otherwise false is returned.
1127  */
1128 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1129                             bool write_fault, bool *writable, pfn_t *pfn)
1130 {
1131         struct page *page[1];
1132         int npages;
1133
1134         if (!(async || atomic))
1135                 return false;
1136
1137         /*
1138          * Fast pin a writable pfn only if it is a write fault request
1139          * or the caller allows to map a writable pfn for a read fault
1140          * request.
1141          */
1142         if (!(write_fault || writable))
1143                 return false;
1144
1145         npages = __get_user_pages_fast(addr, 1, 1, page);
1146         if (npages == 1) {
1147                 *pfn = page_to_pfn(page[0]);
1148
1149                 if (writable)
1150                         *writable = true;
1151                 return true;
1152         }
1153
1154         return false;
1155 }
1156
1157 /*
1158  * The slow path to get the pfn of the specified host virtual address,
1159  * 1 indicates success, -errno is returned if error is detected.
1160  */
1161 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1162                            bool *writable, pfn_t *pfn)
1163 {
1164         struct page *page[1];
1165         int npages = 0;
1166
1167         might_sleep();
1168
1169         if (writable)
1170                 *writable = write_fault;
1171
1172         if (async) {
1173                 down_read(&current->mm->mmap_sem);
1174                 npages = get_user_page_nowait(current, current->mm,
1175                                               addr, write_fault, page);
1176                 up_read(&current->mm->mmap_sem);
1177         } else
1178                 npages = get_user_pages_fast(addr, 1, write_fault,
1179                                              page);
1180         if (npages != 1)
1181                 return npages;
1182
1183         /* map read fault as writable if possible */
1184         if (unlikely(!write_fault) && writable) {
1185                 struct page *wpage[1];
1186
1187                 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1188                 if (npages == 1) {
1189                         *writable = true;
1190                         put_page(page[0]);
1191                         page[0] = wpage[0];
1192                 }
1193
1194                 npages = 1;
1195         }
1196         *pfn = page_to_pfn(page[0]);
1197         return npages;
1198 }
1199
1200 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1201 {
1202         if (unlikely(!(vma->vm_flags & VM_READ)))
1203                 return false;
1204
1205         if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1206                 return false;
1207
1208         return true;
1209 }
1210
1211 /*
1212  * Pin guest page in memory and return its pfn.
1213  * @addr: host virtual address which maps memory to the guest
1214  * @atomic: whether this function can sleep
1215  * @async: whether this function need to wait IO complete if the
1216  *         host page is not in the memory
1217  * @write_fault: whether we should get a writable host page
1218  * @writable: whether it allows to map a writable host page for !@write_fault
1219  *
1220  * The function will map a writable host page for these two cases:
1221  * 1): @write_fault = true
1222  * 2): @write_fault = false && @writable, @writable will tell the caller
1223  *     whether the mapping is writable.
1224  */
1225 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1226                         bool write_fault, bool *writable)
1227 {
1228         struct vm_area_struct *vma;
1229         pfn_t pfn = 0;
1230         int npages;
1231
1232         /* we can do it either atomically or asynchronously, not both */
1233         BUG_ON(atomic && async);
1234
1235         if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1236                 return pfn;
1237
1238         if (atomic)
1239                 return KVM_PFN_ERR_FAULT;
1240
1241         npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1242         if (npages == 1)
1243                 return pfn;
1244
1245         down_read(&current->mm->mmap_sem);
1246         if (npages == -EHWPOISON ||
1247               (!async && check_user_page_hwpoison(addr))) {
1248                 pfn = KVM_PFN_ERR_HWPOISON;
1249                 goto exit;
1250         }
1251
1252         vma = find_vma_intersection(current->mm, addr, addr + 1);
1253
1254         if (vma == NULL)
1255                 pfn = KVM_PFN_ERR_FAULT;
1256         else if ((vma->vm_flags & VM_PFNMAP)) {
1257                 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1258                         vma->vm_pgoff;
1259                 BUG_ON(!kvm_is_mmio_pfn(pfn));
1260         } else {
1261                 if (async && vma_is_valid(vma, write_fault))
1262                         *async = true;
1263                 pfn = KVM_PFN_ERR_FAULT;
1264         }
1265 exit:
1266         up_read(&current->mm->mmap_sem);
1267         return pfn;
1268 }
1269
1270 static pfn_t
1271 __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
1272                      bool *async, bool write_fault, bool *writable)
1273 {
1274         unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1275
1276         if (addr == KVM_HVA_ERR_RO_BAD)
1277                 return KVM_PFN_ERR_RO_FAULT;
1278
1279         if (kvm_is_error_hva(addr))
1280                 return KVM_PFN_NOSLOT;
1281
1282         /* Do not map writable pfn in the readonly memslot. */
1283         if (writable && memslot_is_readonly(slot)) {
1284                 *writable = false;
1285                 writable = NULL;
1286         }
1287
1288         return hva_to_pfn(addr, atomic, async, write_fault,
1289                           writable);
1290 }
1291
1292 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1293                           bool write_fault, bool *writable)
1294 {
1295         struct kvm_memory_slot *slot;
1296
1297         if (async)
1298                 *async = false;
1299
1300         slot = gfn_to_memslot(kvm, gfn);
1301
1302         return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
1303                                     writable);
1304 }
1305
1306 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1307 {
1308         return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1309 }
1310 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1311
1312 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
1313                        bool write_fault, bool *writable)
1314 {
1315         return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1316 }
1317 EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
1318
1319 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1320 {
1321         return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1322 }
1323 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1324
1325 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1326                       bool *writable)
1327 {
1328         return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1329 }
1330 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1331
1332 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1333 {
1334         return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1335 }
1336
1337 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1338 {
1339         return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1340 }
1341 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1342
1343 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1344                                                                   int nr_pages)
1345 {
1346         unsigned long addr;
1347         gfn_t entry;
1348
1349         addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1350         if (kvm_is_error_hva(addr))
1351                 return -1;
1352
1353         if (entry < nr_pages)
1354                 return 0;
1355
1356         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1357 }
1358 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1359
1360 static struct page *kvm_pfn_to_page(pfn_t pfn)
1361 {
1362         if (is_error_noslot_pfn(pfn))
1363                 return KVM_ERR_PTR_BAD_PAGE;
1364
1365         if (kvm_is_mmio_pfn(pfn)) {
1366                 WARN_ON(1);
1367                 return KVM_ERR_PTR_BAD_PAGE;
1368         }
1369
1370         return pfn_to_page(pfn);
1371 }
1372
1373 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1374 {
1375         pfn_t pfn;
1376
1377         pfn = gfn_to_pfn(kvm, gfn);
1378
1379         return kvm_pfn_to_page(pfn);
1380 }
1381
1382 EXPORT_SYMBOL_GPL(gfn_to_page);
1383
1384 void kvm_release_page_clean(struct page *page)
1385 {
1386         WARN_ON(is_error_page(page));
1387
1388         kvm_release_pfn_clean(page_to_pfn(page));
1389 }
1390 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1391
1392 void kvm_release_pfn_clean(pfn_t pfn)
1393 {
1394         if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
1395                 put_page(pfn_to_page(pfn));
1396 }
1397 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1398
1399 void kvm_release_page_dirty(struct page *page)
1400 {
1401         WARN_ON(is_error_page(page));
1402
1403         kvm_release_pfn_dirty(page_to_pfn(page));
1404 }
1405 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1406
1407 static void kvm_release_pfn_dirty(pfn_t pfn)
1408 {
1409         kvm_set_pfn_dirty(pfn);
1410         kvm_release_pfn_clean(pfn);
1411 }
1412
1413 void kvm_set_pfn_dirty(pfn_t pfn)
1414 {
1415         if (!kvm_is_mmio_pfn(pfn)) {
1416                 struct page *page = pfn_to_page(pfn);
1417                 if (!PageReserved(page))
1418                         SetPageDirty(page);
1419         }
1420 }
1421 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1422
1423 void kvm_set_pfn_accessed(pfn_t pfn)
1424 {
1425         if (!kvm_is_mmio_pfn(pfn))
1426                 mark_page_accessed(pfn_to_page(pfn));
1427 }
1428 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1429
1430 void kvm_get_pfn(pfn_t pfn)
1431 {
1432         if (!kvm_is_mmio_pfn(pfn))
1433                 get_page(pfn_to_page(pfn));
1434 }
1435 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1436
1437 static int next_segment(unsigned long len, int offset)
1438 {
1439         if (len > PAGE_SIZE - offset)
1440                 return PAGE_SIZE - offset;
1441         else
1442                 return len;
1443 }
1444
1445 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1446                         int len)
1447 {
1448         int r;
1449         unsigned long addr;
1450
1451         addr = gfn_to_hva_prot(kvm, gfn, NULL);
1452         if (kvm_is_error_hva(addr))
1453                 return -EFAULT;
1454         r = kvm_read_hva(data, (void __user *)addr + offset, len);
1455         if (r)
1456                 return -EFAULT;
1457         return 0;
1458 }
1459 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1460
1461 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1462 {
1463         gfn_t gfn = gpa >> PAGE_SHIFT;
1464         int seg;
1465         int offset = offset_in_page(gpa);
1466         int ret;
1467
1468         while ((seg = next_segment(len, offset)) != 0) {
1469                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1470                 if (ret < 0)
1471                         return ret;
1472                 offset = 0;
1473                 len -= seg;
1474                 data += seg;
1475                 ++gfn;
1476         }
1477         return 0;
1478 }
1479 EXPORT_SYMBOL_GPL(kvm_read_guest);
1480
1481 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1482                           unsigned long len)
1483 {
1484         int r;
1485         unsigned long addr;
1486         gfn_t gfn = gpa >> PAGE_SHIFT;
1487         int offset = offset_in_page(gpa);
1488
1489         addr = gfn_to_hva_prot(kvm, gfn, NULL);
1490         if (kvm_is_error_hva(addr))
1491                 return -EFAULT;
1492         pagefault_disable();
1493         r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1494         pagefault_enable();
1495         if (r)
1496                 return -EFAULT;
1497         return 0;
1498 }
1499 EXPORT_SYMBOL(kvm_read_guest_atomic);
1500
1501 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1502                          int offset, int len)
1503 {
1504         int r;
1505         unsigned long addr;
1506
1507         addr = gfn_to_hva(kvm, gfn);
1508         if (kvm_is_error_hva(addr))
1509                 return -EFAULT;
1510         r = __copy_to_user((void __user *)addr + offset, data, len);
1511         if (r)
1512                 return -EFAULT;
1513         mark_page_dirty(kvm, gfn);
1514         return 0;
1515 }
1516 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1517
1518 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1519                     unsigned long len)
1520 {
1521         gfn_t gfn = gpa >> PAGE_SHIFT;
1522         int seg;
1523         int offset = offset_in_page(gpa);
1524         int ret;
1525
1526         while ((seg = next_segment(len, offset)) != 0) {
1527                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1528                 if (ret < 0)
1529                         return ret;
1530                 offset = 0;
1531                 len -= seg;
1532                 data += seg;
1533                 ++gfn;
1534         }
1535         return 0;
1536 }
1537
1538 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1539                               gpa_t gpa, unsigned long len)
1540 {
1541         struct kvm_memslots *slots = kvm_memslots(kvm);
1542         int offset = offset_in_page(gpa);
1543         gfn_t start_gfn = gpa >> PAGE_SHIFT;
1544         gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1545         gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1546         gfn_t nr_pages_avail;
1547
1548         ghc->gpa = gpa;
1549         ghc->generation = slots->generation;
1550         ghc->len = len;
1551         ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1552         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail);
1553         if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) {
1554                 ghc->hva += offset;
1555         } else {
1556                 /*
1557                  * If the requested region crosses two memslots, we still
1558                  * verify that the entire region is valid here.
1559                  */
1560                 while (start_gfn <= end_gfn) {
1561                         ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1562                         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1563                                                    &nr_pages_avail);
1564                         if (kvm_is_error_hva(ghc->hva))
1565                                 return -EFAULT;
1566                         start_gfn += nr_pages_avail;
1567                 }
1568                 /* Use the slow path for cross page reads and writes. */
1569                 ghc->memslot = NULL;
1570         }
1571         return 0;
1572 }
1573 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1574
1575 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1576                            void *data, unsigned long len)
1577 {
1578         struct kvm_memslots *slots = kvm_memslots(kvm);
1579         int r;
1580
1581         BUG_ON(len > ghc->len);
1582
1583         if (slots->generation != ghc->generation)
1584                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1585
1586         if (unlikely(!ghc->memslot))
1587                 return kvm_write_guest(kvm, ghc->gpa, data, len);
1588
1589         if (kvm_is_error_hva(ghc->hva))
1590                 return -EFAULT;
1591
1592         r = __copy_to_user((void __user *)ghc->hva, data, len);
1593         if (r)
1594                 return -EFAULT;
1595         mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1596
1597         return 0;
1598 }
1599 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1600
1601 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1602                            void *data, unsigned long len)
1603 {
1604         struct kvm_memslots *slots = kvm_memslots(kvm);
1605         int r;
1606
1607         BUG_ON(len > ghc->len);
1608
1609         if (slots->generation != ghc->generation)
1610                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1611
1612         if (unlikely(!ghc->memslot))
1613                 return kvm_read_guest(kvm, ghc->gpa, data, len);
1614
1615         if (kvm_is_error_hva(ghc->hva))
1616                 return -EFAULT;
1617
1618         r = __copy_from_user(data, (void __user *)ghc->hva, len);
1619         if (r)
1620                 return -EFAULT;
1621
1622         return 0;
1623 }
1624 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1625
1626 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1627 {
1628         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
1629
1630         return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
1631 }
1632 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1633
1634 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1635 {
1636         gfn_t gfn = gpa >> PAGE_SHIFT;
1637         int seg;
1638         int offset = offset_in_page(gpa);
1639         int ret;
1640
1641         while ((seg = next_segment(len, offset)) != 0) {
1642                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1643                 if (ret < 0)
1644                         return ret;
1645                 offset = 0;
1646                 len -= seg;
1647                 ++gfn;
1648         }
1649         return 0;
1650 }
1651 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1652
1653 static void mark_page_dirty_in_slot(struct kvm *kvm,
1654                                     struct kvm_memory_slot *memslot,
1655                                     gfn_t gfn)
1656 {
1657         if (memslot && memslot->dirty_bitmap) {
1658                 unsigned long rel_gfn = gfn - memslot->base_gfn;
1659
1660                 set_bit_le(rel_gfn, memslot->dirty_bitmap);
1661         }
1662 }
1663
1664 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1665 {
1666         struct kvm_memory_slot *memslot;
1667
1668         memslot = gfn_to_memslot(kvm, gfn);
1669         mark_page_dirty_in_slot(kvm, memslot, gfn);
1670 }
1671
1672 /*
1673  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1674  */
1675 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1676 {
1677         DEFINE_WAIT(wait);
1678
1679         for (;;) {
1680                 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1681
1682                 if (kvm_arch_vcpu_runnable(vcpu)) {
1683                         kvm_make_request(KVM_REQ_UNHALT, vcpu);
1684                         break;
1685                 }
1686                 if (kvm_cpu_has_pending_timer(vcpu))
1687                         break;
1688                 if (signal_pending(current))
1689                         break;
1690
1691                 schedule();
1692         }
1693
1694         finish_wait(&vcpu->wq, &wait);
1695 }
1696
1697 #ifndef CONFIG_S390
1698 /*
1699  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
1700  */
1701 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
1702 {
1703         int me;
1704         int cpu = vcpu->cpu;
1705         wait_queue_head_t *wqp;
1706
1707         wqp = kvm_arch_vcpu_wq(vcpu);
1708         if (waitqueue_active(wqp)) {
1709                 wake_up_interruptible(wqp);
1710                 ++vcpu->stat.halt_wakeup;
1711         }
1712
1713         me = get_cpu();
1714         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
1715                 if (kvm_arch_vcpu_should_kick(vcpu))
1716                         smp_send_reschedule(cpu);
1717         put_cpu();
1718 }
1719 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
1720 #endif /* !CONFIG_S390 */
1721
1722 bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
1723 {
1724         struct pid *pid;
1725         struct task_struct *task = NULL;
1726         bool ret = false;
1727
1728         rcu_read_lock();
1729         pid = rcu_dereference(target->pid);
1730         if (pid)
1731                 task = get_pid_task(target->pid, PIDTYPE_PID);
1732         rcu_read_unlock();
1733         if (!task)
1734                 return ret;
1735         if (task->flags & PF_VCPU) {
1736                 put_task_struct(task);
1737                 return ret;
1738         }
1739         ret = yield_to(task, 1);
1740         put_task_struct(task);
1741
1742         return ret;
1743 }
1744 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
1745
1746 /*
1747  * Helper that checks whether a VCPU is eligible for directed yield.
1748  * Most eligible candidate to yield is decided by following heuristics:
1749  *
1750  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
1751  *  (preempted lock holder), indicated by @in_spin_loop.
1752  *  Set at the beiginning and cleared at the end of interception/PLE handler.
1753  *
1754  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
1755  *  chance last time (mostly it has become eligible now since we have probably
1756  *  yielded to lockholder in last iteration. This is done by toggling
1757  *  @dy_eligible each time a VCPU checked for eligibility.)
1758  *
1759  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
1760  *  to preempted lock-holder could result in wrong VCPU selection and CPU
1761  *  burning. Giving priority for a potential lock-holder increases lock
1762  *  progress.
1763  *
1764  *  Since algorithm is based on heuristics, accessing another VCPU data without
1765  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
1766  *  and continue with next VCPU and so on.
1767  */
1768 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
1769 {
1770 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1771         bool eligible;
1772
1773         eligible = !vcpu->spin_loop.in_spin_loop ||
1774                     vcpu->spin_loop.dy_eligible;
1775
1776         if (vcpu->spin_loop.in_spin_loop)
1777                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
1778
1779         return eligible;
1780 #else
1781         return true;
1782 #endif
1783 }
1784
1785 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1786 {
1787         struct kvm *kvm = me->kvm;
1788         struct kvm_vcpu *vcpu;
1789         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1790         int yielded = 0;
1791         int try = 3;
1792         int pass;
1793         int i;
1794
1795         kvm_vcpu_set_in_spin_loop(me, true);
1796         /*
1797          * We boost the priority of a VCPU that is runnable but not
1798          * currently running, because it got preempted by something
1799          * else and called schedule in __vcpu_run.  Hopefully that
1800          * VCPU is holding the lock that we need and will release it.
1801          * We approximate round-robin by starting at the last boosted VCPU.
1802          */
1803         for (pass = 0; pass < 2 && !yielded && try; pass++) {
1804                 kvm_for_each_vcpu(i, vcpu, kvm) {
1805                         if (!pass && i <= last_boosted_vcpu) {
1806                                 i = last_boosted_vcpu;
1807                                 continue;
1808                         } else if (pass && i > last_boosted_vcpu)
1809                                 break;
1810                         if (!ACCESS_ONCE(vcpu->preempted))
1811                                 continue;
1812                         if (vcpu == me)
1813                                 continue;
1814                         if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
1815                                 continue;
1816                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
1817                                 continue;
1818
1819                         yielded = kvm_vcpu_yield_to(vcpu);
1820                         if (yielded > 0) {
1821                                 kvm->last_boosted_vcpu = i;
1822                                 break;
1823                         } else if (yielded < 0) {
1824                                 try--;
1825                                 if (!try)
1826                                         break;
1827                         }
1828                 }
1829         }
1830         kvm_vcpu_set_in_spin_loop(me, false);
1831
1832         /* Ensure vcpu is not eligible during next spinloop */
1833         kvm_vcpu_set_dy_eligible(me, false);
1834 }
1835 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1836
1837 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1838 {
1839         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1840         struct page *page;
1841
1842         if (vmf->pgoff == 0)
1843                 page = virt_to_page(vcpu->run);
1844 #ifdef CONFIG_X86
1845         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1846                 page = virt_to_page(vcpu->arch.pio_data);
1847 #endif
1848 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1849         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1850                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1851 #endif
1852         else
1853                 return kvm_arch_vcpu_fault(vcpu, vmf);
1854         get_page(page);
1855         vmf->page = page;
1856         return 0;
1857 }
1858
1859 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1860         .fault = kvm_vcpu_fault,
1861 };
1862
1863 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1864 {
1865         vma->vm_ops = &kvm_vcpu_vm_ops;
1866         return 0;
1867 }
1868
1869 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1870 {
1871         struct kvm_vcpu *vcpu = filp->private_data;
1872
1873         kvm_put_kvm(vcpu->kvm);
1874         return 0;
1875 }
1876
1877 static struct file_operations kvm_vcpu_fops = {
1878         .release        = kvm_vcpu_release,
1879         .unlocked_ioctl = kvm_vcpu_ioctl,
1880 #ifdef CONFIG_COMPAT
1881         .compat_ioctl   = kvm_vcpu_compat_ioctl,
1882 #endif
1883         .mmap           = kvm_vcpu_mmap,
1884         .llseek         = noop_llseek,
1885 };
1886
1887 /*
1888  * Allocates an inode for the vcpu.
1889  */
1890 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1891 {
1892         return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
1893 }
1894
1895 /*
1896  * Creates some virtual cpus.  Good luck creating more than one.
1897  */
1898 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1899 {
1900         int r;
1901         struct kvm_vcpu *vcpu, *v;
1902
1903         if (id >= KVM_MAX_VCPUS)
1904                 return -EINVAL;
1905
1906         vcpu = kvm_arch_vcpu_create(kvm, id);
1907         if (IS_ERR(vcpu))
1908                 return PTR_ERR(vcpu);
1909
1910         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1911
1912         r = kvm_arch_vcpu_setup(vcpu);
1913         if (r)
1914                 goto vcpu_destroy;
1915
1916         mutex_lock(&kvm->lock);
1917         if (!kvm_vcpu_compatible(vcpu)) {
1918                 r = -EINVAL;
1919                 goto unlock_vcpu_destroy;
1920         }
1921         if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
1922                 r = -EINVAL;
1923                 goto unlock_vcpu_destroy;
1924         }
1925
1926         kvm_for_each_vcpu(r, v, kvm)
1927                 if (v->vcpu_id == id) {
1928                         r = -EEXIST;
1929                         goto unlock_vcpu_destroy;
1930                 }
1931
1932         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1933
1934         /* Now it's all set up, let userspace reach it */
1935         kvm_get_kvm(kvm);
1936         r = create_vcpu_fd(vcpu);
1937         if (r < 0) {
1938                 kvm_put_kvm(kvm);
1939                 goto unlock_vcpu_destroy;
1940         }
1941
1942         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
1943         smp_wmb();
1944         atomic_inc(&kvm->online_vcpus);
1945
1946         mutex_unlock(&kvm->lock);
1947         kvm_arch_vcpu_postcreate(vcpu);
1948         return r;
1949
1950 unlock_vcpu_destroy:
1951         mutex_unlock(&kvm->lock);
1952 vcpu_destroy:
1953         kvm_arch_vcpu_destroy(vcpu);
1954         return r;
1955 }
1956
1957 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1958 {
1959         if (sigset) {
1960                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1961                 vcpu->sigset_active = 1;
1962                 vcpu->sigset = *sigset;
1963         } else
1964                 vcpu->sigset_active = 0;
1965         return 0;
1966 }
1967
1968 static long kvm_vcpu_ioctl(struct file *filp,
1969                            unsigned int ioctl, unsigned long arg)
1970 {
1971         struct kvm_vcpu *vcpu = filp->private_data;
1972         void __user *argp = (void __user *)arg;
1973         int r;
1974         struct kvm_fpu *fpu = NULL;
1975         struct kvm_sregs *kvm_sregs = NULL;
1976
1977         if (vcpu->kvm->mm != current->mm)
1978                 return -EIO;
1979
1980 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
1981         /*
1982          * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
1983          * so vcpu_load() would break it.
1984          */
1985         if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
1986                 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1987 #endif
1988
1989
1990         r = vcpu_load(vcpu);
1991         if (r)
1992                 return r;
1993         switch (ioctl) {
1994         case KVM_RUN:
1995                 r = -EINVAL;
1996                 if (arg)
1997                         goto out;
1998                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1999                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2000                 break;
2001         case KVM_GET_REGS: {
2002                 struct kvm_regs *kvm_regs;
2003
2004                 r = -ENOMEM;
2005                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2006                 if (!kvm_regs)
2007                         goto out;
2008                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2009                 if (r)
2010                         goto out_free1;
2011                 r = -EFAULT;
2012                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2013                         goto out_free1;
2014                 r = 0;
2015 out_free1:
2016                 kfree(kvm_regs);
2017                 break;
2018         }
2019         case KVM_SET_REGS: {
2020                 struct kvm_regs *kvm_regs;
2021
2022                 r = -ENOMEM;
2023                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2024                 if (IS_ERR(kvm_regs)) {
2025                         r = PTR_ERR(kvm_regs);
2026                         goto out;
2027                 }
2028                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2029                 kfree(kvm_regs);
2030                 break;
2031         }
2032         case KVM_GET_SREGS: {
2033                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2034                 r = -ENOMEM;
2035                 if (!kvm_sregs)
2036                         goto out;
2037                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2038                 if (r)
2039                         goto out;
2040                 r = -EFAULT;
2041                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2042                         goto out;
2043                 r = 0;
2044                 break;
2045         }
2046         case KVM_SET_SREGS: {
2047                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2048                 if (IS_ERR(kvm_sregs)) {
2049                         r = PTR_ERR(kvm_sregs);
2050                         kvm_sregs = NULL;
2051                         goto out;
2052                 }
2053                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2054                 break;
2055         }
2056         case KVM_GET_MP_STATE: {
2057                 struct kvm_mp_state mp_state;
2058
2059                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2060                 if (r)
2061                         goto out;
2062                 r = -EFAULT;
2063                 if (copy_to_user(argp, &mp_state, sizeof mp_state))
2064                         goto out;
2065                 r = 0;
2066                 break;
2067         }
2068         case KVM_SET_MP_STATE: {
2069                 struct kvm_mp_state mp_state;
2070
2071                 r = -EFAULT;
2072                 if (copy_from_user(&mp_state, argp, sizeof mp_state))
2073                         goto out;
2074                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2075                 break;
2076         }
2077         case KVM_TRANSLATE: {
2078                 struct kvm_translation tr;
2079
2080                 r = -EFAULT;
2081                 if (copy_from_user(&tr, argp, sizeof tr))
2082                         goto out;
2083                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2084                 if (r)
2085                         goto out;
2086                 r = -EFAULT;
2087                 if (copy_to_user(argp, &tr, sizeof tr))
2088                         goto out;
2089                 r = 0;
2090                 break;
2091         }
2092         case KVM_SET_GUEST_DEBUG: {
2093                 struct kvm_guest_debug dbg;
2094
2095                 r = -EFAULT;
2096                 if (copy_from_user(&dbg, argp, sizeof dbg))
2097                         goto out;
2098                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2099                 break;
2100         }
2101         case KVM_SET_SIGNAL_MASK: {
2102                 struct kvm_signal_mask __user *sigmask_arg = argp;
2103                 struct kvm_signal_mask kvm_sigmask;
2104                 sigset_t sigset, *p;
2105
2106                 p = NULL;
2107                 if (argp) {
2108                         r = -EFAULT;
2109                         if (copy_from_user(&kvm_sigmask, argp,
2110                                            sizeof kvm_sigmask))
2111                                 goto out;
2112                         r = -EINVAL;
2113                         if (kvm_sigmask.len != sizeof sigset)
2114                                 goto out;
2115                         r = -EFAULT;
2116                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2117                                            sizeof sigset))
2118                                 goto out;
2119                         p = &sigset;
2120                 }
2121                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2122                 break;
2123         }
2124         case KVM_GET_FPU: {
2125                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2126                 r = -ENOMEM;
2127                 if (!fpu)
2128                         goto out;
2129                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2130                 if (r)
2131                         goto out;
2132                 r = -EFAULT;
2133                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2134                         goto out;
2135                 r = 0;
2136                 break;
2137         }
2138         case KVM_SET_FPU: {
2139                 fpu = memdup_user(argp, sizeof(*fpu));
2140                 if (IS_ERR(fpu)) {
2141                         r = PTR_ERR(fpu);
2142                         fpu = NULL;
2143                         goto out;
2144                 }
2145                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2146                 break;
2147         }
2148         default:
2149                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2150         }
2151 out:
2152         vcpu_put(vcpu);
2153         kfree(fpu);
2154         kfree(kvm_sregs);
2155         return r;
2156 }
2157
2158 #ifdef CONFIG_COMPAT
2159 static long kvm_vcpu_compat_ioctl(struct file *filp,
2160                                   unsigned int ioctl, unsigned long arg)
2161 {
2162         struct kvm_vcpu *vcpu = filp->private_data;
2163         void __user *argp = compat_ptr(arg);
2164         int r;
2165
2166         if (vcpu->kvm->mm != current->mm)
2167                 return -EIO;
2168
2169         switch (ioctl) {
2170         case KVM_SET_SIGNAL_MASK: {
2171                 struct kvm_signal_mask __user *sigmask_arg = argp;
2172                 struct kvm_signal_mask kvm_sigmask;
2173                 compat_sigset_t csigset;
2174                 sigset_t sigset;
2175
2176                 if (argp) {
2177                         r = -EFAULT;
2178                         if (copy_from_user(&kvm_sigmask, argp,
2179                                            sizeof kvm_sigmask))
2180                                 goto out;
2181                         r = -EINVAL;
2182                         if (kvm_sigmask.len != sizeof csigset)
2183                                 goto out;
2184                         r = -EFAULT;
2185                         if (copy_from_user(&csigset, sigmask_arg->sigset,
2186                                            sizeof csigset))
2187                                 goto out;
2188                         sigset_from_compat(&sigset, &csigset);
2189                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2190                 } else
2191                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2192                 break;
2193         }
2194         default:
2195                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2196         }
2197
2198 out:
2199         return r;
2200 }
2201 #endif
2202
2203 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2204                                  int (*accessor)(struct kvm_device *dev,
2205                                                  struct kvm_device_attr *attr),
2206                                  unsigned long arg)
2207 {
2208         struct kvm_device_attr attr;
2209
2210         if (!accessor)
2211                 return -EPERM;
2212
2213         if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2214                 return -EFAULT;
2215
2216         return accessor(dev, &attr);
2217 }
2218
2219 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2220                              unsigned long arg)
2221 {
2222         struct kvm_device *dev = filp->private_data;
2223
2224         switch (ioctl) {
2225         case KVM_SET_DEVICE_ATTR:
2226                 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2227         case KVM_GET_DEVICE_ATTR:
2228                 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2229         case KVM_HAS_DEVICE_ATTR:
2230                 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2231         default:
2232                 if (dev->ops->ioctl)
2233                         return dev->ops->ioctl(dev, ioctl, arg);
2234
2235                 return -ENOTTY;
2236         }
2237 }
2238
2239 static int kvm_device_release(struct inode *inode, struct file *filp)
2240 {
2241         struct kvm_device *dev = filp->private_data;
2242         struct kvm *kvm = dev->kvm;
2243
2244         kvm_put_kvm(kvm);
2245         return 0;
2246 }
2247
2248 static const struct file_operations kvm_device_fops = {
2249         .unlocked_ioctl = kvm_device_ioctl,
2250 #ifdef CONFIG_COMPAT
2251         .compat_ioctl = kvm_device_ioctl,
2252 #endif
2253         .release = kvm_device_release,
2254 };
2255
2256 struct kvm_device *kvm_device_from_filp(struct file *filp)
2257 {
2258         if (filp->f_op != &kvm_device_fops)
2259                 return NULL;
2260
2261         return filp->private_data;
2262 }
2263
2264 static int kvm_ioctl_create_device(struct kvm *kvm,
2265                                    struct kvm_create_device *cd)
2266 {
2267         struct kvm_device_ops *ops = NULL;
2268         struct kvm_device *dev;
2269         bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2270         int ret;
2271
2272         switch (cd->type) {
2273 #ifdef CONFIG_KVM_MPIC
2274         case KVM_DEV_TYPE_FSL_MPIC_20:
2275         case KVM_DEV_TYPE_FSL_MPIC_42:
2276                 ops = &kvm_mpic_ops;
2277                 break;
2278 #endif
2279 #ifdef CONFIG_KVM_XICS
2280         case KVM_DEV_TYPE_XICS:
2281                 ops = &kvm_xics_ops;
2282                 break;
2283 #endif
2284 #ifdef CONFIG_KVM_VFIO
2285         case KVM_DEV_TYPE_VFIO:
2286                 ops = &kvm_vfio_ops;
2287                 break;
2288 #endif
2289 #ifdef CONFIG_KVM_ARM_VGIC
2290         case KVM_DEV_TYPE_ARM_VGIC_V2:
2291                 ops = &kvm_arm_vgic_v2_ops;
2292                 break;
2293 #endif
2294         default:
2295                 return -ENODEV;
2296         }
2297
2298         if (test)
2299                 return 0;
2300
2301         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2302         if (!dev)
2303                 return -ENOMEM;
2304
2305         dev->ops = ops;
2306         dev->kvm = kvm;
2307
2308         ret = ops->create(dev, cd->type);
2309         if (ret < 0) {
2310                 kfree(dev);
2311                 return ret;
2312         }
2313
2314         ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2315         if (ret < 0) {
2316                 ops->destroy(dev);
2317                 return ret;
2318         }
2319
2320         list_add(&dev->vm_node, &kvm->devices);
2321         kvm_get_kvm(kvm);
2322         cd->fd = ret;
2323         return 0;
2324 }
2325
2326 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2327 {
2328         switch (arg) {
2329         case KVM_CAP_USER_MEMORY:
2330         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2331         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2332 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
2333         case KVM_CAP_SET_BOOT_CPU_ID:
2334 #endif
2335         case KVM_CAP_INTERNAL_ERROR_DATA:
2336 #ifdef CONFIG_HAVE_KVM_MSI
2337         case KVM_CAP_SIGNAL_MSI:
2338 #endif
2339 #ifdef CONFIG_HAVE_KVM_IRQFD
2340         case KVM_CAP_IRQFD_RESAMPLE:
2341 #endif
2342         case KVM_CAP_CHECK_EXTENSION_VM:
2343                 return 1;
2344 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2345         case KVM_CAP_IRQ_ROUTING:
2346                 return KVM_MAX_IRQ_ROUTES;
2347 #endif
2348         default:
2349                 break;
2350         }
2351         return kvm_vm_ioctl_check_extension(kvm, arg);
2352 }
2353
2354 static long kvm_vm_ioctl(struct file *filp,
2355                            unsigned int ioctl, unsigned long arg)
2356 {
2357         struct kvm *kvm = filp->private_data;
2358         void __user *argp = (void __user *)arg;
2359         int r;
2360
2361         if (kvm->mm != current->mm)
2362                 return -EIO;
2363         switch (ioctl) {
2364         case KVM_CREATE_VCPU:
2365                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2366                 break;
2367         case KVM_SET_USER_MEMORY_REGION: {
2368                 struct kvm_userspace_memory_region kvm_userspace_mem;
2369
2370                 r = -EFAULT;
2371                 if (copy_from_user(&kvm_userspace_mem, argp,
2372                                                 sizeof kvm_userspace_mem))
2373                         goto out;
2374
2375                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2376                 break;
2377         }
2378         case KVM_GET_DIRTY_LOG: {
2379                 struct kvm_dirty_log log;
2380
2381                 r = -EFAULT;
2382                 if (copy_from_user(&log, argp, sizeof log))
2383                         goto out;
2384                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2385                 break;
2386         }
2387 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2388         case KVM_REGISTER_COALESCED_MMIO: {
2389                 struct kvm_coalesced_mmio_zone zone;
2390                 r = -EFAULT;
2391                 if (copy_from_user(&zone, argp, sizeof zone))
2392                         goto out;
2393                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2394                 break;
2395         }
2396         case KVM_UNREGISTER_COALESCED_MMIO: {
2397                 struct kvm_coalesced_mmio_zone zone;
2398                 r = -EFAULT;
2399                 if (copy_from_user(&zone, argp, sizeof zone))
2400                         goto out;
2401                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2402                 break;
2403         }
2404 #endif
2405         case KVM_IRQFD: {
2406                 struct kvm_irqfd data;
2407
2408                 r = -EFAULT;
2409                 if (copy_from_user(&data, argp, sizeof data))
2410                         goto out;
2411                 r = kvm_irqfd(kvm, &data);
2412                 break;
2413         }
2414         case KVM_IOEVENTFD: {
2415                 struct kvm_ioeventfd data;
2416
2417                 r = -EFAULT;
2418                 if (copy_from_user(&data, argp, sizeof data))
2419                         goto out;
2420                 r = kvm_ioeventfd(kvm, &data);
2421                 break;
2422         }
2423 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
2424         case KVM_SET_BOOT_CPU_ID:
2425                 r = 0;
2426                 mutex_lock(&kvm->lock);
2427                 if (atomic_read(&kvm->online_vcpus) != 0)
2428                         r = -EBUSY;
2429                 else
2430                         kvm->bsp_vcpu_id = arg;
2431                 mutex_unlock(&kvm->lock);
2432                 break;
2433 #endif
2434 #ifdef CONFIG_HAVE_KVM_MSI
2435         case KVM_SIGNAL_MSI: {
2436                 struct kvm_msi msi;
2437
2438                 r = -EFAULT;
2439                 if (copy_from_user(&msi, argp, sizeof msi))
2440                         goto out;
2441                 r = kvm_send_userspace_msi(kvm, &msi);
2442                 break;
2443         }
2444 #endif
2445 #ifdef __KVM_HAVE_IRQ_LINE
2446         case KVM_IRQ_LINE_STATUS:
2447         case KVM_IRQ_LINE: {
2448                 struct kvm_irq_level irq_event;
2449
2450                 r = -EFAULT;
2451                 if (copy_from_user(&irq_event, argp, sizeof irq_event))
2452                         goto out;
2453
2454                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
2455                                         ioctl == KVM_IRQ_LINE_STATUS);
2456                 if (r)
2457                         goto out;
2458
2459                 r = -EFAULT;
2460                 if (ioctl == KVM_IRQ_LINE_STATUS) {
2461                         if (copy_to_user(argp, &irq_event, sizeof irq_event))
2462                                 goto out;
2463                 }
2464
2465                 r = 0;
2466                 break;
2467         }
2468 #endif
2469 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2470         case KVM_SET_GSI_ROUTING: {
2471                 struct kvm_irq_routing routing;
2472                 struct kvm_irq_routing __user *urouting;
2473                 struct kvm_irq_routing_entry *entries;
2474
2475                 r = -EFAULT;
2476                 if (copy_from_user(&routing, argp, sizeof(routing)))
2477                         goto out;
2478                 r = -EINVAL;
2479                 if (routing.nr >= KVM_MAX_IRQ_ROUTES)
2480                         goto out;
2481                 if (routing.flags)
2482                         goto out;
2483                 r = -ENOMEM;
2484                 entries = vmalloc(routing.nr * sizeof(*entries));
2485                 if (!entries)
2486                         goto out;
2487                 r = -EFAULT;
2488                 urouting = argp;
2489                 if (copy_from_user(entries, urouting->entries,
2490                                    routing.nr * sizeof(*entries)))
2491                         goto out_free_irq_routing;
2492                 r = kvm_set_irq_routing(kvm, entries, routing.nr,
2493                                         routing.flags);
2494         out_free_irq_routing:
2495                 vfree(entries);
2496                 break;
2497         }
2498 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
2499         case KVM_CREATE_DEVICE: {
2500                 struct kvm_create_device cd;
2501
2502                 r = -EFAULT;
2503                 if (copy_from_user(&cd, argp, sizeof(cd)))
2504                         goto out;
2505
2506                 r = kvm_ioctl_create_device(kvm, &cd);
2507                 if (r)
2508                         goto out;
2509
2510                 r = -EFAULT;
2511                 if (copy_to_user(argp, &cd, sizeof(cd)))
2512                         goto out;
2513
2514                 r = 0;
2515                 break;
2516         }
2517         case KVM_CHECK_EXTENSION:
2518                 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
2519                 break;
2520         default:
2521                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2522                 if (r == -ENOTTY)
2523                         r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2524         }
2525 out:
2526         return r;
2527 }
2528
2529 #ifdef CONFIG_COMPAT
2530 struct compat_kvm_dirty_log {
2531         __u32 slot;
2532         __u32 padding1;
2533         union {
2534                 compat_uptr_t dirty_bitmap; /* one bit per page */
2535                 __u64 padding2;
2536         };
2537 };
2538
2539 static long kvm_vm_compat_ioctl(struct file *filp,
2540                            unsigned int ioctl, unsigned long arg)
2541 {
2542         struct kvm *kvm = filp->private_data;
2543         int r;
2544
2545         if (kvm->mm != current->mm)
2546                 return -EIO;
2547         switch (ioctl) {
2548         case KVM_GET_DIRTY_LOG: {
2549                 struct compat_kvm_dirty_log compat_log;
2550                 struct kvm_dirty_log log;
2551
2552                 r = -EFAULT;
2553                 if (copy_from_user(&compat_log, (void __user *)arg,
2554                                    sizeof(compat_log)))
2555                         goto out;
2556                 log.slot         = compat_log.slot;
2557                 log.padding1     = compat_log.padding1;
2558                 log.padding2     = compat_log.padding2;
2559                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2560
2561                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2562                 break;
2563         }
2564         default:
2565                 r = kvm_vm_ioctl(filp, ioctl, arg);
2566         }
2567
2568 out:
2569         return r;
2570 }
2571 #endif
2572
2573 static struct file_operations kvm_vm_fops = {
2574         .release        = kvm_vm_release,
2575         .unlocked_ioctl = kvm_vm_ioctl,
2576 #ifdef CONFIG_COMPAT
2577         .compat_ioctl   = kvm_vm_compat_ioctl,
2578 #endif
2579         .llseek         = noop_llseek,
2580 };
2581
2582 static int kvm_dev_ioctl_create_vm(unsigned long type)
2583 {
2584         int r;
2585         struct kvm *kvm;
2586
2587         kvm = kvm_create_vm(type);
2588         if (IS_ERR(kvm))
2589                 return PTR_ERR(kvm);
2590 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2591         r = kvm_coalesced_mmio_init(kvm);
2592         if (r < 0) {
2593                 kvm_put_kvm(kvm);
2594                 return r;
2595         }
2596 #endif
2597         r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
2598         if (r < 0)
2599                 kvm_put_kvm(kvm);
2600
2601         return r;
2602 }
2603
2604 static long kvm_dev_ioctl(struct file *filp,
2605                           unsigned int ioctl, unsigned long arg)
2606 {
2607         long r = -EINVAL;
2608
2609         switch (ioctl) {
2610         case KVM_GET_API_VERSION:
2611                 if (arg)
2612                         goto out;
2613                 r = KVM_API_VERSION;
2614                 break;
2615         case KVM_CREATE_VM:
2616                 r = kvm_dev_ioctl_create_vm(arg);
2617                 break;
2618         case KVM_CHECK_EXTENSION:
2619                 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
2620                 break;
2621         case KVM_GET_VCPU_MMAP_SIZE:
2622                 if (arg)
2623                         goto out;
2624                 r = PAGE_SIZE;     /* struct kvm_run */
2625 #ifdef CONFIG_X86
2626                 r += PAGE_SIZE;    /* pio data page */
2627 #endif
2628 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2629                 r += PAGE_SIZE;    /* coalesced mmio ring page */
2630 #endif
2631                 break;
2632         case KVM_TRACE_ENABLE:
2633         case KVM_TRACE_PAUSE:
2634         case KVM_TRACE_DISABLE:
2635                 r = -EOPNOTSUPP;
2636                 break;
2637         default:
2638                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
2639         }
2640 out:
2641         return r;
2642 }
2643
2644 static struct file_operations kvm_chardev_ops = {
2645         .unlocked_ioctl = kvm_dev_ioctl,
2646         .compat_ioctl   = kvm_dev_ioctl,
2647         .llseek         = noop_llseek,
2648 };
2649
2650 static struct miscdevice kvm_dev = {
2651         KVM_MINOR,
2652         "kvm",
2653         &kvm_chardev_ops,
2654 };
2655
2656 static void hardware_enable_nolock(void *junk)
2657 {
2658         int cpu = raw_smp_processor_id();
2659         int r;
2660
2661         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2662                 return;
2663
2664         cpumask_set_cpu(cpu, cpus_hardware_enabled);
2665
2666         r = kvm_arch_hardware_enable();
2667
2668         if (r) {
2669                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2670                 atomic_inc(&hardware_enable_failed);
2671                 printk(KERN_INFO "kvm: enabling virtualization on "
2672                                  "CPU%d failed\n", cpu);
2673         }
2674 }
2675
2676 static void hardware_enable(void)
2677 {
2678         raw_spin_lock(&kvm_count_lock);
2679         if (kvm_usage_count)
2680                 hardware_enable_nolock(NULL);
2681         raw_spin_unlock(&kvm_count_lock);
2682 }
2683
2684 static void hardware_disable_nolock(void *junk)
2685 {
2686         int cpu = raw_smp_processor_id();
2687
2688         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2689                 return;
2690         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2691         kvm_arch_hardware_disable();
2692 }
2693
2694 static void hardware_disable(void)
2695 {
2696         raw_spin_lock(&kvm_count_lock);
2697         if (kvm_usage_count)
2698                 hardware_disable_nolock(NULL);
2699         raw_spin_unlock(&kvm_count_lock);
2700 }
2701
2702 static void hardware_disable_all_nolock(void)
2703 {
2704         BUG_ON(!kvm_usage_count);
2705
2706         kvm_usage_count--;
2707         if (!kvm_usage_count)
2708                 on_each_cpu(hardware_disable_nolock, NULL, 1);
2709 }
2710
2711 static void hardware_disable_all(void)
2712 {
2713         raw_spin_lock(&kvm_count_lock);
2714         hardware_disable_all_nolock();
2715         raw_spin_unlock(&kvm_count_lock);
2716 }
2717
2718 static int hardware_enable_all(void)
2719 {
2720         int r = 0;
2721
2722         raw_spin_lock(&kvm_count_lock);
2723
2724         kvm_usage_count++;
2725         if (kvm_usage_count == 1) {
2726                 atomic_set(&hardware_enable_failed, 0);
2727                 on_each_cpu(hardware_enable_nolock, NULL, 1);
2728
2729                 if (atomic_read(&hardware_enable_failed)) {
2730                         hardware_disable_all_nolock();
2731                         r = -EBUSY;
2732                 }
2733         }
2734
2735         raw_spin_unlock(&kvm_count_lock);
2736
2737         return r;
2738 }
2739
2740 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2741                            void *v)
2742 {
2743         int cpu = (long)v;
2744
2745         val &= ~CPU_TASKS_FROZEN;
2746         switch (val) {
2747         case CPU_DYING:
2748                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2749                        cpu);
2750                 hardware_disable();
2751                 break;
2752         case CPU_STARTING:
2753                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2754                        cpu);
2755                 hardware_enable();
2756                 break;
2757         }
2758         return NOTIFY_OK;
2759 }
2760
2761 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2762                       void *v)
2763 {
2764         /*
2765          * Some (well, at least mine) BIOSes hang on reboot if
2766          * in vmx root mode.
2767          *
2768          * And Intel TXT required VMX off for all cpu when system shutdown.
2769          */
2770         printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2771         kvm_rebooting = true;
2772         on_each_cpu(hardware_disable_nolock, NULL, 1);
2773         return NOTIFY_OK;
2774 }
2775
2776 static struct notifier_block kvm_reboot_notifier = {
2777         .notifier_call = kvm_reboot,
2778         .priority = 0,
2779 };
2780
2781 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2782 {
2783         int i;
2784
2785         for (i = 0; i < bus->dev_count; i++) {
2786                 struct kvm_io_device *pos = bus->range[i].dev;
2787
2788                 kvm_iodevice_destructor(pos);
2789         }
2790         kfree(bus);
2791 }
2792
2793 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
2794 {
2795         const struct kvm_io_range *r1 = p1;
2796         const struct kvm_io_range *r2 = p2;
2797
2798         if (r1->addr < r2->addr)
2799                 return -1;
2800         if (r1->addr + r1->len > r2->addr + r2->len)
2801                 return 1;
2802         return 0;
2803 }
2804
2805 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
2806                           gpa_t addr, int len)
2807 {
2808         bus->range[bus->dev_count++] = (struct kvm_io_range) {
2809                 .addr = addr,
2810                 .len = len,
2811                 .dev = dev,
2812         };
2813
2814         sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
2815                 kvm_io_bus_sort_cmp, NULL);
2816
2817         return 0;
2818 }
2819
2820 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2821                              gpa_t addr, int len)
2822 {
2823         struct kvm_io_range *range, key;
2824         int off;
2825
2826         key = (struct kvm_io_range) {
2827                 .addr = addr,
2828                 .len = len,
2829         };
2830
2831         range = bsearch(&key, bus->range, bus->dev_count,
2832                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
2833         if (range == NULL)
2834                 return -ENOENT;
2835
2836         off = range - bus->range;
2837
2838         while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
2839                 off--;
2840
2841         return off;
2842 }
2843
2844 /* kvm_io_bus_write - called under kvm->slots_lock */
2845 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2846                      int len, const void *val)
2847 {
2848         int idx;
2849         struct kvm_io_bus *bus;
2850         struct kvm_io_range range;
2851
2852         range = (struct kvm_io_range) {
2853                 .addr = addr,
2854                 .len = len,
2855         };
2856
2857         bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2858         idx = kvm_io_bus_get_first_dev(bus, addr, len);
2859         if (idx < 0)
2860                 return -EOPNOTSUPP;
2861
2862         while (idx < bus->dev_count &&
2863                 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
2864                 if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
2865                         return 0;
2866                 idx++;
2867         }
2868
2869         return -EOPNOTSUPP;
2870 }
2871
2872 /* kvm_io_bus_read - called under kvm->slots_lock */
2873 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2874                     int len, void *val)
2875 {
2876         int idx;
2877         struct kvm_io_bus *bus;
2878         struct kvm_io_range range;
2879
2880         range = (struct kvm_io_range) {
2881                 .addr = addr,
2882                 .len = len,
2883         };
2884
2885         bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2886         idx = kvm_io_bus_get_first_dev(bus, addr, len);
2887         if (idx < 0)
2888                 return -EOPNOTSUPP;
2889
2890         while (idx < bus->dev_count &&
2891                 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
2892                 if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
2893                         return 0;
2894                 idx++;
2895         }
2896
2897         return -EOPNOTSUPP;
2898 }
2899
2900 /* Caller must hold slots_lock. */
2901 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2902                             int len, struct kvm_io_device *dev)
2903 {
2904         struct kvm_io_bus *new_bus, *bus;
2905
2906         bus = kvm->buses[bus_idx];
2907         if (bus->dev_count > NR_IOBUS_DEVS - 1)
2908                 return -ENOSPC;
2909
2910         new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
2911                           sizeof(struct kvm_io_range)), GFP_KERNEL);
2912         if (!new_bus)
2913                 return -ENOMEM;
2914         memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
2915                sizeof(struct kvm_io_range)));
2916         kvm_io_bus_insert_dev(new_bus, dev, addr, len);
2917         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2918         synchronize_srcu_expedited(&kvm->srcu);
2919         kfree(bus);
2920
2921         return 0;
2922 }
2923
2924 /* Caller must hold slots_lock. */
2925 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2926                               struct kvm_io_device *dev)
2927 {
2928         int i, r;
2929         struct kvm_io_bus *new_bus, *bus;
2930
2931         bus = kvm->buses[bus_idx];
2932         r = -ENOENT;
2933         for (i = 0; i < bus->dev_count; i++)
2934                 if (bus->range[i].dev == dev) {
2935                         r = 0;
2936                         break;
2937                 }
2938
2939         if (r)
2940                 return r;
2941
2942         new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
2943                           sizeof(struct kvm_io_range)), GFP_KERNEL);
2944         if (!new_bus)
2945                 return -ENOMEM;
2946
2947         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
2948         new_bus->dev_count--;
2949         memcpy(new_bus->range + i, bus->range + i + 1,
2950                (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
2951
2952         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2953         synchronize_srcu_expedited(&kvm->srcu);
2954         kfree(bus);
2955         return r;
2956 }
2957
2958 static struct notifier_block kvm_cpu_notifier = {
2959         .notifier_call = kvm_cpu_hotplug,
2960 };
2961
2962 static int vm_stat_get(void *_offset, u64 *val)
2963 {
2964         unsigned offset = (long)_offset;
2965         struct kvm *kvm;
2966
2967         *val = 0;
2968         spin_lock(&kvm_lock);
2969         list_for_each_entry(kvm, &vm_list, vm_list)
2970                 *val += *(u32 *)((void *)kvm + offset);
2971         spin_unlock(&kvm_lock);
2972         return 0;
2973 }
2974
2975 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
2976
2977 static int vcpu_stat_get(void *_offset, u64 *val)
2978 {
2979         unsigned offset = (long)_offset;
2980         struct kvm *kvm;
2981         struct kvm_vcpu *vcpu;
2982         int i;
2983
2984         *val = 0;
2985         spin_lock(&kvm_lock);
2986         list_for_each_entry(kvm, &vm_list, vm_list)
2987                 kvm_for_each_vcpu(i, vcpu, kvm)
2988                         *val += *(u32 *)((void *)vcpu + offset);
2989
2990         spin_unlock(&kvm_lock);
2991         return 0;
2992 }
2993
2994 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
2995
2996 static const struct file_operations *stat_fops[] = {
2997         [KVM_STAT_VCPU] = &vcpu_stat_fops,
2998         [KVM_STAT_VM]   = &vm_stat_fops,
2999 };
3000
3001 static int kvm_init_debug(void)
3002 {
3003         int r = -EEXIST;
3004         struct kvm_stats_debugfs_item *p;
3005
3006         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3007         if (kvm_debugfs_dir == NULL)
3008                 goto out;
3009
3010         for (p = debugfs_entries; p->name; ++p) {
3011                 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
3012                                                 (void *)(long)p->offset,
3013                                                 stat_fops[p->kind]);
3014                 if (p->dentry == NULL)
3015                         goto out_dir;
3016         }
3017
3018         return 0;
3019
3020 out_dir:
3021         debugfs_remove_recursive(kvm_debugfs_dir);
3022 out:
3023         return r;
3024 }
3025
3026 static void kvm_exit_debug(void)
3027 {
3028         struct kvm_stats_debugfs_item *p;
3029
3030         for (p = debugfs_entries; p->name; ++p)
3031                 debugfs_remove(p->dentry);
3032         debugfs_remove(kvm_debugfs_dir);
3033 }
3034
3035 static int kvm_suspend(void)
3036 {
3037         if (kvm_usage_count)
3038                 hardware_disable_nolock(NULL);
3039         return 0;
3040 }
3041
3042 static void kvm_resume(void)
3043 {
3044         if (kvm_usage_count) {
3045                 WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3046                 hardware_enable_nolock(NULL);
3047         }
3048 }
3049
3050 static struct syscore_ops kvm_syscore_ops = {
3051         .suspend = kvm_suspend,
3052         .resume = kvm_resume,
3053 };
3054
3055 static inline
3056 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3057 {
3058         return container_of(pn, struct kvm_vcpu, preempt_notifier);
3059 }
3060
3061 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3062 {
3063         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3064         if (vcpu->preempted)
3065                 vcpu->preempted = false;
3066
3067         kvm_arch_sched_in(vcpu, cpu);
3068
3069         kvm_arch_vcpu_load(vcpu, cpu);
3070 }
3071
3072 static void kvm_sched_out(struct preempt_notifier *pn,
3073                           struct task_struct *next)
3074 {
3075         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3076
3077         if (current->state == TASK_RUNNING)
3078                 vcpu->preempted = true;
3079         kvm_arch_vcpu_put(vcpu);
3080 }
3081
3082 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3083                   struct module *module)
3084 {
3085         int r;
3086         int cpu;
3087
3088         r = kvm_arch_init(opaque);
3089         if (r)
3090                 goto out_fail;
3091
3092         /*
3093          * kvm_arch_init makes sure there's at most one caller
3094          * for architectures that support multiple implementations,
3095          * like intel and amd on x86.
3096          * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
3097          * conflicts in case kvm is already setup for another implementation.
3098          */
3099         r = kvm_irqfd_init();
3100         if (r)
3101                 goto out_irqfd;
3102
3103         if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3104                 r = -ENOMEM;
3105                 goto out_free_0;
3106         }
3107
3108         r = kvm_arch_hardware_setup();
3109         if (r < 0)
3110                 goto out_free_0a;
3111
3112         for_each_online_cpu(cpu) {
3113                 smp_call_function_single(cpu,
3114                                 kvm_arch_check_processor_compat,
3115                                 &r, 1);
3116                 if (r < 0)
3117                         goto out_free_1;
3118         }
3119
3120         r = register_cpu_notifier(&kvm_cpu_notifier);
3121         if (r)
3122                 goto out_free_2;
3123         register_reboot_notifier(&kvm_reboot_notifier);
3124
3125         /* A kmem cache lets us meet the alignment requirements of fx_save. */
3126         if (!vcpu_align)
3127                 vcpu_align = __alignof__(struct kvm_vcpu);
3128         kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
3129                                            0, NULL);
3130         if (!kvm_vcpu_cache) {
3131                 r = -ENOMEM;
3132                 goto out_free_3;
3133         }
3134
3135         r = kvm_async_pf_init();
3136         if (r)
3137                 goto out_free;
3138
3139         kvm_chardev_ops.owner = module;
3140         kvm_vm_fops.owner = module;
3141         kvm_vcpu_fops.owner = module;
3142
3143         r = misc_register(&kvm_dev);
3144         if (r) {
3145                 printk(KERN_ERR "kvm: misc device register failed\n");
3146                 goto out_unreg;
3147         }
3148
3149         register_syscore_ops(&kvm_syscore_ops);
3150
3151         kvm_preempt_ops.sched_in = kvm_sched_in;
3152         kvm_preempt_ops.sched_out = kvm_sched_out;
3153
3154         r = kvm_init_debug();
3155         if (r) {
3156                 printk(KERN_ERR "kvm: create debugfs files failed\n");
3157                 goto out_undebugfs;
3158         }
3159
3160         return 0;
3161
3162 out_undebugfs:
3163         unregister_syscore_ops(&kvm_syscore_ops);
3164         misc_deregister(&kvm_dev);
3165 out_unreg:
3166         kvm_async_pf_deinit();
3167 out_free:
3168         kmem_cache_destroy(kvm_vcpu_cache);
3169 out_free_3:
3170         unregister_reboot_notifier(&kvm_reboot_notifier);
3171         unregister_cpu_notifier(&kvm_cpu_notifier);
3172 out_free_2:
3173 out_free_1:
3174         kvm_arch_hardware_unsetup();
3175 out_free_0a:
3176         free_cpumask_var(cpus_hardware_enabled);
3177 out_free_0:
3178         kvm_irqfd_exit();
3179 out_irqfd:
3180         kvm_arch_exit();
3181 out_fail:
3182         return r;
3183 }
3184 EXPORT_SYMBOL_GPL(kvm_init);
3185
3186 void kvm_exit(void)
3187 {
3188         kvm_exit_debug();
3189         misc_deregister(&kvm_dev);
3190         kmem_cache_destroy(kvm_vcpu_cache);
3191         kvm_async_pf_deinit();
3192         unregister_syscore_ops(&kvm_syscore_ops);
3193         unregister_reboot_notifier(&kvm_reboot_notifier);
3194         unregister_cpu_notifier(&kvm_cpu_notifier);
3195         on_each_cpu(hardware_disable_nolock, NULL, 1);
3196         kvm_arch_hardware_unsetup();
3197         kvm_arch_exit();
3198         kvm_irqfd_exit();
3199         free_cpumask_var(cpus_hardware_enabled);
3200 }
3201 EXPORT_SYMBOL_GPL(kvm_exit);