2 * Core of Xen paravirt_ops implementation.
4 * This file contains the xen_paravirt_ops structure itself, and the
6 * - privileged instructions
11 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
14 #include <linux/cpu.h>
15 #include <linux/kernel.h>
16 #include <linux/init.h>
17 #include <linux/smp.h>
18 #include <linux/preempt.h>
19 #include <linux/hardirq.h>
20 #include <linux/percpu.h>
21 #include <linux/delay.h>
22 #include <linux/start_kernel.h>
23 #include <linux/sched.h>
24 #include <linux/kprobes.h>
25 #include <linux/bootmem.h>
26 #include <linux/module.h>
28 #include <linux/page-flags.h>
29 #include <linux/highmem.h>
30 #include <linux/console.h>
31 #include <linux/pci.h>
32 #include <linux/gfp.h>
33 #include <linux/memblock.h>
36 #include <xen/interface/xen.h>
37 #include <xen/interface/version.h>
38 #include <xen/interface/physdev.h>
39 #include <xen/interface/vcpu.h>
40 #include <xen/interface/memory.h>
41 #include <xen/features.h>
44 #include <xen/hvc-console.h>
46 #include <asm/paravirt.h>
49 #include <asm/xen/pci.h>
50 #include <asm/xen/hypercall.h>
51 #include <asm/xen/hypervisor.h>
52 #include <asm/fixmap.h>
53 #include <asm/processor.h>
54 #include <asm/proto.h>
55 #include <asm/msr-index.h>
56 #include <asm/traps.h>
57 #include <asm/setup.h>
59 #include <asm/pgalloc.h>
60 #include <asm/pgtable.h>
61 #include <asm/tlbflush.h>
62 #include <asm/reboot.h>
63 #include <asm/stackprotector.h>
64 #include <asm/hypervisor.h>
65 #include <asm/mwait.h>
68 #include <linux/acpi.h>
70 #include <acpi/pdc_intel.h>
71 #include <acpi/processor.h>
72 #include <xen/interface/platform.h>
77 #include "multicalls.h"
79 EXPORT_SYMBOL_GPL(hypercall_page);
81 DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
82 DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
84 enum xen_domain_type xen_domain_type = XEN_NATIVE;
85 EXPORT_SYMBOL_GPL(xen_domain_type);
87 unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
88 EXPORT_SYMBOL(machine_to_phys_mapping);
89 unsigned long machine_to_phys_nr;
90 EXPORT_SYMBOL(machine_to_phys_nr);
92 struct start_info *xen_start_info;
93 EXPORT_SYMBOL_GPL(xen_start_info);
95 struct shared_info xen_dummy_shared_info;
97 void *xen_initial_gdt;
99 RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
100 __read_mostly int xen_have_vector_callback;
101 EXPORT_SYMBOL_GPL(xen_have_vector_callback);
104 * Point at some empty memory to start with. We map the real shared_info
105 * page as soon as fixmap is up and running.
107 struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
110 * Flag to determine whether vcpu info placement is available on all
111 * VCPUs. We assume it is to start with, and then set it to zero on
112 * the first failure. This is because it can succeed on some VCPUs
113 * and not others, since it can involve hypervisor memory allocation,
114 * or because the guest failed to guarantee all the appropriate
115 * constraints on all VCPUs (ie buffer can't cross a page boundary).
117 * Note that any particular CPU may be using a placed vcpu structure,
118 * but we can only optimise if the all are.
120 * 0: not available, 1: available
122 static int have_vcpu_info_placement = 1;
124 static void clamp_max_cpus(void)
127 if (setup_max_cpus > MAX_VIRT_CPUS)
128 setup_max_cpus = MAX_VIRT_CPUS;
132 static void xen_vcpu_setup(int cpu)
134 struct vcpu_register_vcpu_info info;
136 struct vcpu_info *vcpup;
138 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
140 if (cpu < MAX_VIRT_CPUS)
141 per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
143 if (!have_vcpu_info_placement) {
144 if (cpu >= MAX_VIRT_CPUS)
149 vcpup = &per_cpu(xen_vcpu_info, cpu);
150 info.mfn = arbitrary_virt_to_mfn(vcpup);
151 info.offset = offset_in_page(vcpup);
153 /* Check to see if the hypervisor will put the vcpu_info
154 structure where we want it, which allows direct access via
155 a percpu-variable. */
156 err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
159 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
160 have_vcpu_info_placement = 0;
163 /* This cpu is using the registered vcpu info, even if
164 later ones fail to. */
165 per_cpu(xen_vcpu, cpu) = vcpup;
170 * On restore, set the vcpu placement up again.
171 * If it fails, then we're in a bad state, since
172 * we can't back out from using it...
174 void xen_vcpu_restore(void)
178 for_each_online_cpu(cpu) {
179 bool other_cpu = (cpu != smp_processor_id());
182 HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
185 xen_setup_runstate_info(cpu);
187 if (have_vcpu_info_placement)
191 HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
196 static void __init xen_banner(void)
198 unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
199 struct xen_extraversion extra;
200 HYPERVISOR_xen_version(XENVER_extraversion, &extra);
202 printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
204 printk(KERN_INFO "Xen version: %d.%d%s%s\n",
205 version >> 16, version & 0xffff, extra.extraversion,
206 xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
209 static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
210 static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;
212 static __read_mostly unsigned int cpuid_leaf1_ecx_set_mask;
213 static __read_mostly unsigned int cpuid_leaf5_ecx_val;
214 static __read_mostly unsigned int cpuid_leaf5_edx_val;
216 static void xen_cpuid(unsigned int *ax, unsigned int *bx,
217 unsigned int *cx, unsigned int *dx)
219 unsigned maskebx = ~0;
220 unsigned maskecx = ~0;
221 unsigned maskedx = ~0;
224 * Mask out inconvenient features, to try and disable as many
225 * unsupported kernel subsystems as possible.
229 maskecx = cpuid_leaf1_ecx_mask;
230 setecx = cpuid_leaf1_ecx_set_mask;
231 maskedx = cpuid_leaf1_edx_mask;
234 case CPUID_MWAIT_LEAF:
235 /* Synthesize the values.. */
238 *cx = cpuid_leaf5_ecx_val;
239 *dx = cpuid_leaf5_edx_val;
243 /* Suppress extended topology stuff */
248 asm(XEN_EMULATE_PREFIX "cpuid"
253 : "0" (*ax), "2" (*cx));
262 static bool __init xen_check_mwait(void)
265 struct xen_platform_op op = {
266 .cmd = XENPF_set_processor_pminfo,
267 .u.set_pminfo.id = -1,
268 .u.set_pminfo.type = XEN_PM_PDC,
271 unsigned int ax, bx, cx, dx;
272 unsigned int mwait_mask;
274 /* We need to determine whether it is OK to expose the MWAIT
275 * capability to the kernel to harvest deeper than C3 states from ACPI
276 * _CST using the processor_harvest_xen.c module. For this to work, we
277 * need to gather the MWAIT_LEAF values (which the cstate.c code
278 * checks against). The hypervisor won't expose the MWAIT flag because
279 * it would break backwards compatibility; so we will find out directly
280 * from the hardware and hypercall.
282 if (!xen_initial_domain())
288 native_cpuid(&ax, &bx, &cx, &dx);
290 mwait_mask = (1 << (X86_FEATURE_EST % 32)) |
291 (1 << (X86_FEATURE_MWAIT % 32));
293 if ((cx & mwait_mask) != mwait_mask)
296 /* We need to emulate the MWAIT_LEAF and for that we need both
297 * ecx and edx. The hypercall provides only partial information.
300 ax = CPUID_MWAIT_LEAF;
305 native_cpuid(&ax, &bx, &cx, &dx);
307 /* Ask the Hypervisor whether to clear ACPI_PDC_C_C2C3_FFH. If so,
308 * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3.
310 buf[0] = ACPI_PDC_REVISION_ID;
312 buf[2] = (ACPI_PDC_C_CAPABILITY_SMP | ACPI_PDC_EST_CAPABILITY_SWSMP);
314 set_xen_guest_handle(op.u.set_pminfo.pdc, buf);
316 if ((HYPERVISOR_dom0_op(&op) == 0) &&
317 (buf[2] & (ACPI_PDC_C_C1_FFH | ACPI_PDC_C_C2C3_FFH))) {
318 cpuid_leaf5_ecx_val = cx;
319 cpuid_leaf5_edx_val = dx;
326 static void __init xen_init_cpuid_mask(void)
328 unsigned int ax, bx, cx, dx;
329 unsigned int xsave_mask;
331 cpuid_leaf1_edx_mask =
332 ~((1 << X86_FEATURE_MCE) | /* disable MCE */
333 (1 << X86_FEATURE_MCA) | /* disable MCA */
334 (1 << X86_FEATURE_MTRR) | /* disable MTRR */
335 (1 << X86_FEATURE_ACC)); /* thermal monitoring */
337 if (!xen_initial_domain())
338 cpuid_leaf1_edx_mask &=
339 ~((1 << X86_FEATURE_APIC) | /* disable local APIC */
340 (1 << X86_FEATURE_ACPI)); /* disable ACPI */
343 xen_cpuid(&ax, &bx, &cx, &dx);
346 (1 << (X86_FEATURE_XSAVE % 32)) |
347 (1 << (X86_FEATURE_OSXSAVE % 32));
349 /* Xen will set CR4.OSXSAVE if supported and not disabled by force */
350 if ((cx & xsave_mask) != xsave_mask)
351 cpuid_leaf1_ecx_mask &= ~xsave_mask; /* disable XSAVE & OSXSAVE */
353 if (xen_check_mwait())
354 cpuid_leaf1_ecx_set_mask = (1 << (X86_FEATURE_MWAIT % 32));
357 static void xen_set_debugreg(int reg, unsigned long val)
359 HYPERVISOR_set_debugreg(reg, val);
362 static unsigned long xen_get_debugreg(int reg)
364 return HYPERVISOR_get_debugreg(reg);
367 static void xen_end_context_switch(struct task_struct *next)
370 paravirt_end_context_switch(next);
373 static unsigned long xen_store_tr(void)
379 * Set the page permissions for a particular virtual address. If the
380 * address is a vmalloc mapping (or other non-linear mapping), then
381 * find the linear mapping of the page and also set its protections to
384 static void set_aliased_prot(void *v, pgprot_t prot)
392 ptep = lookup_address((unsigned long)v, &level);
393 BUG_ON(ptep == NULL);
395 pfn = pte_pfn(*ptep);
396 page = pfn_to_page(pfn);
398 pte = pfn_pte(pfn, prot);
400 if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
403 if (!PageHighMem(page)) {
404 void *av = __va(PFN_PHYS(pfn));
407 if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
413 static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
415 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
418 for(i = 0; i < entries; i += entries_per_page)
419 set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
422 static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
424 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
427 for(i = 0; i < entries; i += entries_per_page)
428 set_aliased_prot(ldt + i, PAGE_KERNEL);
431 static void xen_set_ldt(const void *addr, unsigned entries)
433 struct mmuext_op *op;
434 struct multicall_space mcs = xen_mc_entry(sizeof(*op));
436 trace_xen_cpu_set_ldt(addr, entries);
439 op->cmd = MMUEXT_SET_LDT;
440 op->arg1.linear_addr = (unsigned long)addr;
441 op->arg2.nr_ents = entries;
443 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
445 xen_mc_issue(PARAVIRT_LAZY_CPU);
448 static void xen_load_gdt(const struct desc_ptr *dtr)
450 unsigned long va = dtr->address;
451 unsigned int size = dtr->size + 1;
452 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
453 unsigned long frames[pages];
457 * A GDT can be up to 64k in size, which corresponds to 8192
458 * 8-byte entries, or 16 4k pages..
461 BUG_ON(size > 65536);
462 BUG_ON(va & ~PAGE_MASK);
464 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
467 unsigned long pfn, mfn;
471 * The GDT is per-cpu and is in the percpu data area.
472 * That can be virtually mapped, so we need to do a
473 * page-walk to get the underlying MFN for the
474 * hypercall. The page can also be in the kernel's
475 * linear range, so we need to RO that mapping too.
477 ptep = lookup_address(va, &level);
478 BUG_ON(ptep == NULL);
480 pfn = pte_pfn(*ptep);
481 mfn = pfn_to_mfn(pfn);
482 virt = __va(PFN_PHYS(pfn));
486 make_lowmem_page_readonly((void *)va);
487 make_lowmem_page_readonly(virt);
490 if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
495 * load_gdt for early boot, when the gdt is only mapped once
497 static void __init xen_load_gdt_boot(const struct desc_ptr *dtr)
499 unsigned long va = dtr->address;
500 unsigned int size = dtr->size + 1;
501 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
502 unsigned long frames[pages];
506 * A GDT can be up to 64k in size, which corresponds to 8192
507 * 8-byte entries, or 16 4k pages..
510 BUG_ON(size > 65536);
511 BUG_ON(va & ~PAGE_MASK);
513 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
515 unsigned long pfn, mfn;
517 pfn = virt_to_pfn(va);
518 mfn = pfn_to_mfn(pfn);
520 pte = pfn_pte(pfn, PAGE_KERNEL_RO);
522 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
528 if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
532 static void load_TLS_descriptor(struct thread_struct *t,
533 unsigned int cpu, unsigned int i)
535 struct desc_struct *gdt = get_cpu_gdt_table(cpu);
536 xmaddr_t maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
537 struct multicall_space mc = __xen_mc_entry(0);
539 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
542 static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
545 * XXX sleazy hack: If we're being called in a lazy-cpu zone
546 * and lazy gs handling is enabled, it means we're in a
547 * context switch, and %gs has just been saved. This means we
548 * can zero it out to prevent faults on exit from the
549 * hypervisor if the next process has no %gs. Either way, it
550 * has been saved, and the new value will get loaded properly.
551 * This will go away as soon as Xen has been modified to not
552 * save/restore %gs for normal hypercalls.
554 * On x86_64, this hack is not used for %gs, because gs points
555 * to KERNEL_GS_BASE (and uses it for PDA references), so we
556 * must not zero %gs on x86_64
558 * For x86_64, we need to zero %fs, otherwise we may get an
559 * exception between the new %fs descriptor being loaded and
560 * %fs being effectively cleared at __switch_to().
562 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
572 load_TLS_descriptor(t, cpu, 0);
573 load_TLS_descriptor(t, cpu, 1);
574 load_TLS_descriptor(t, cpu, 2);
576 xen_mc_issue(PARAVIRT_LAZY_CPU);
580 static void xen_load_gs_index(unsigned int idx)
582 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
587 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
590 xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
591 u64 entry = *(u64 *)ptr;
593 trace_xen_cpu_write_ldt_entry(dt, entrynum, entry);
598 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
604 static int cvt_gate_to_trap(int vector, const gate_desc *val,
605 struct trap_info *info)
609 if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
612 info->vector = vector;
614 addr = gate_offset(*val);
617 * Look for known traps using IST, and substitute them
618 * appropriately. The debugger ones are the only ones we care
619 * about. Xen will handle faults like double_fault and
620 * machine_check, so we should never see them. Warn if
621 * there's an unexpected IST-using fault handler.
623 if (addr == (unsigned long)debug)
624 addr = (unsigned long)xen_debug;
625 else if (addr == (unsigned long)int3)
626 addr = (unsigned long)xen_int3;
627 else if (addr == (unsigned long)stack_segment)
628 addr = (unsigned long)xen_stack_segment;
629 else if (addr == (unsigned long)double_fault ||
630 addr == (unsigned long)nmi) {
631 /* Don't need to handle these */
633 #ifdef CONFIG_X86_MCE
634 } else if (addr == (unsigned long)machine_check) {
638 /* Some other trap using IST? */
639 if (WARN_ON(val->ist != 0))
642 #endif /* CONFIG_X86_64 */
643 info->address = addr;
645 info->cs = gate_segment(*val);
646 info->flags = val->dpl;
647 /* interrupt gates clear IF */
648 if (val->type == GATE_INTERRUPT)
649 info->flags |= 1 << 2;
654 /* Locations of each CPU's IDT */
655 static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
657 /* Set an IDT entry. If the entry is part of the current IDT, then
659 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
661 unsigned long p = (unsigned long)&dt[entrynum];
662 unsigned long start, end;
664 trace_xen_cpu_write_idt_entry(dt, entrynum, g);
668 start = __this_cpu_read(idt_desc.address);
669 end = start + __this_cpu_read(idt_desc.size) + 1;
673 native_write_idt_entry(dt, entrynum, g);
675 if (p >= start && (p + 8) <= end) {
676 struct trap_info info[2];
680 if (cvt_gate_to_trap(entrynum, g, &info[0]))
681 if (HYPERVISOR_set_trap_table(info))
688 static void xen_convert_trap_info(const struct desc_ptr *desc,
689 struct trap_info *traps)
691 unsigned in, out, count;
693 count = (desc->size+1) / sizeof(gate_desc);
696 for (in = out = 0; in < count; in++) {
697 gate_desc *entry = (gate_desc*)(desc->address) + in;
699 if (cvt_gate_to_trap(in, entry, &traps[out]))
702 traps[out].address = 0;
705 void xen_copy_trap_info(struct trap_info *traps)
707 const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
709 xen_convert_trap_info(desc, traps);
712 /* Load a new IDT into Xen. In principle this can be per-CPU, so we
713 hold a spinlock to protect the static traps[] array (static because
714 it avoids allocation, and saves stack space). */
715 static void xen_load_idt(const struct desc_ptr *desc)
717 static DEFINE_SPINLOCK(lock);
718 static struct trap_info traps[257];
720 trace_xen_cpu_load_idt(desc);
724 __get_cpu_var(idt_desc) = *desc;
726 xen_convert_trap_info(desc, traps);
729 if (HYPERVISOR_set_trap_table(traps))
735 /* Write a GDT descriptor entry. Ignore LDT descriptors, since
736 they're handled differently. */
737 static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
738 const void *desc, int type)
740 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);
751 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
754 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
764 * Version of write_gdt_entry for use at early boot-time needed to
765 * update an entry as simply as possible.
767 static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
768 const void *desc, int type)
770 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);
779 xmaddr_t maddr = virt_to_machine(&dt[entry]);
781 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
782 dt[entry] = *(struct desc_struct *)desc;
788 static void xen_load_sp0(struct tss_struct *tss,
789 struct thread_struct *thread)
791 struct multicall_space mcs;
793 mcs = xen_mc_entry(0);
794 MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
795 xen_mc_issue(PARAVIRT_LAZY_CPU);
798 static void xen_set_iopl_mask(unsigned mask)
800 struct physdev_set_iopl set_iopl;
802 /* Force the change at ring 0. */
803 set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
804 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
807 static void xen_io_delay(void)
811 #ifdef CONFIG_X86_LOCAL_APIC
812 static u32 xen_apic_read(u32 reg)
817 static void xen_apic_write(u32 reg, u32 val)
819 /* Warn to see if there's any stray references */
823 static u64 xen_apic_icr_read(void)
828 static void xen_apic_icr_write(u32 low, u32 id)
830 /* Warn to see if there's any stray references */
834 static void xen_apic_wait_icr_idle(void)
839 static u32 xen_safe_apic_wait_icr_idle(void)
844 static void set_xen_basic_apic_ops(void)
846 apic->read = xen_apic_read;
847 apic->write = xen_apic_write;
848 apic->icr_read = xen_apic_icr_read;
849 apic->icr_write = xen_apic_icr_write;
850 apic->wait_icr_idle = xen_apic_wait_icr_idle;
851 apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
856 static void xen_clts(void)
858 struct multicall_space mcs;
860 mcs = xen_mc_entry(0);
862 MULTI_fpu_taskswitch(mcs.mc, 0);
864 xen_mc_issue(PARAVIRT_LAZY_CPU);
867 static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
869 static unsigned long xen_read_cr0(void)
871 unsigned long cr0 = this_cpu_read(xen_cr0_value);
873 if (unlikely(cr0 == 0)) {
874 cr0 = native_read_cr0();
875 this_cpu_write(xen_cr0_value, cr0);
881 static void xen_write_cr0(unsigned long cr0)
883 struct multicall_space mcs;
885 this_cpu_write(xen_cr0_value, cr0);
887 /* Only pay attention to cr0.TS; everything else is
889 mcs = xen_mc_entry(0);
891 MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);
893 xen_mc_issue(PARAVIRT_LAZY_CPU);
896 static void xen_write_cr4(unsigned long cr4)
901 native_write_cr4(cr4);
904 static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
915 case MSR_FS_BASE: which = SEGBASE_FS; goto set;
916 case MSR_KERNEL_GS_BASE: which = SEGBASE_GS_USER; goto set;
917 case MSR_GS_BASE: which = SEGBASE_GS_KERNEL; goto set;
920 base = ((u64)high << 32) | low;
921 if (HYPERVISOR_set_segment_base(which, base) != 0)
929 case MSR_SYSCALL_MASK:
930 case MSR_IA32_SYSENTER_CS:
931 case MSR_IA32_SYSENTER_ESP:
932 case MSR_IA32_SYSENTER_EIP:
933 /* Fast syscall setup is all done in hypercalls, so
934 these are all ignored. Stub them out here to stop
935 Xen console noise. */
938 case MSR_IA32_CR_PAT:
939 if (smp_processor_id() == 0)
940 xen_set_pat(((u64)high << 32) | low);
944 ret = native_write_msr_safe(msr, low, high);
950 void xen_setup_shared_info(void)
952 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
953 set_fixmap(FIX_PARAVIRT_BOOTMAP,
954 xen_start_info->shared_info);
956 HYPERVISOR_shared_info =
957 (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
959 HYPERVISOR_shared_info =
960 (struct shared_info *)__va(xen_start_info->shared_info);
963 /* In UP this is as good a place as any to set up shared info */
964 xen_setup_vcpu_info_placement();
967 xen_setup_mfn_list_list();
970 /* This is called once we have the cpu_possible_mask */
971 void xen_setup_vcpu_info_placement(void)
975 for_each_possible_cpu(cpu)
978 /* xen_vcpu_setup managed to place the vcpu_info within the
979 percpu area for all cpus, so make use of it */
980 if (have_vcpu_info_placement) {
981 pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
982 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
983 pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
984 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
985 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
989 static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
990 unsigned long addr, unsigned len)
992 char *start, *end, *reloc;
995 start = end = reloc = NULL;
997 #define SITE(op, x) \
998 case PARAVIRT_PATCH(op.x): \
999 if (have_vcpu_info_placement) { \
1000 start = (char *)xen_##x##_direct; \
1001 end = xen_##x##_direct_end; \
1002 reloc = xen_##x##_direct_reloc; \
1007 SITE(pv_irq_ops, irq_enable);
1008 SITE(pv_irq_ops, irq_disable);
1009 SITE(pv_irq_ops, save_fl);
1010 SITE(pv_irq_ops, restore_fl);
1014 if (start == NULL || (end-start) > len)
1017 ret = paravirt_patch_insns(insnbuf, len, start, end);
1019 /* Note: because reloc is assigned from something that
1020 appears to be an array, gcc assumes it's non-null,
1021 but doesn't know its relationship with start and
1023 if (reloc > start && reloc < end) {
1024 int reloc_off = reloc - start;
1025 long *relocp = (long *)(insnbuf + reloc_off);
1026 long delta = start - (char *)addr;
1034 ret = paravirt_patch_default(type, clobbers, insnbuf,
1042 static const struct pv_info xen_info __initconst = {
1043 .paravirt_enabled = 1,
1044 .shared_kernel_pmd = 0,
1046 #ifdef CONFIG_X86_64
1047 .extra_user_64bit_cs = FLAT_USER_CS64,
1053 static const struct pv_init_ops xen_init_ops __initconst = {
1057 static const struct pv_cpu_ops xen_cpu_ops __initconst = {
1060 .set_debugreg = xen_set_debugreg,
1061 .get_debugreg = xen_get_debugreg,
1065 .read_cr0 = xen_read_cr0,
1066 .write_cr0 = xen_write_cr0,
1068 .read_cr4 = native_read_cr4,
1069 .read_cr4_safe = native_read_cr4_safe,
1070 .write_cr4 = xen_write_cr4,
1072 .wbinvd = native_wbinvd,
1074 .read_msr = native_read_msr_safe,
1075 .write_msr = xen_write_msr_safe,
1076 .read_tsc = native_read_tsc,
1077 .read_pmc = native_read_pmc,
1080 .irq_enable_sysexit = xen_sysexit,
1081 #ifdef CONFIG_X86_64
1082 .usergs_sysret32 = xen_sysret32,
1083 .usergs_sysret64 = xen_sysret64,
1086 .load_tr_desc = paravirt_nop,
1087 .set_ldt = xen_set_ldt,
1088 .load_gdt = xen_load_gdt,
1089 .load_idt = xen_load_idt,
1090 .load_tls = xen_load_tls,
1091 #ifdef CONFIG_X86_64
1092 .load_gs_index = xen_load_gs_index,
1095 .alloc_ldt = xen_alloc_ldt,
1096 .free_ldt = xen_free_ldt,
1098 .store_gdt = native_store_gdt,
1099 .store_idt = native_store_idt,
1100 .store_tr = xen_store_tr,
1102 .write_ldt_entry = xen_write_ldt_entry,
1103 .write_gdt_entry = xen_write_gdt_entry,
1104 .write_idt_entry = xen_write_idt_entry,
1105 .load_sp0 = xen_load_sp0,
1107 .set_iopl_mask = xen_set_iopl_mask,
1108 .io_delay = xen_io_delay,
1110 /* Xen takes care of %gs when switching to usermode for us */
1111 .swapgs = paravirt_nop,
1113 .start_context_switch = paravirt_start_context_switch,
1114 .end_context_switch = xen_end_context_switch,
1117 static const struct pv_apic_ops xen_apic_ops __initconst = {
1118 #ifdef CONFIG_X86_LOCAL_APIC
1119 .startup_ipi_hook = paravirt_nop,
1123 static void xen_reboot(int reason)
1125 struct sched_shutdown r = { .reason = reason };
1127 if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
1131 static void xen_restart(char *msg)
1133 xen_reboot(SHUTDOWN_reboot);
1136 static void xen_emergency_restart(void)
1138 xen_reboot(SHUTDOWN_reboot);
1141 static void xen_machine_halt(void)
1143 xen_reboot(SHUTDOWN_poweroff);
1146 static void xen_machine_power_off(void)
1150 xen_reboot(SHUTDOWN_poweroff);
1153 static void xen_crash_shutdown(struct pt_regs *regs)
1155 xen_reboot(SHUTDOWN_crash);
1159 xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
1161 xen_reboot(SHUTDOWN_crash);
1165 static struct notifier_block xen_panic_block = {
1166 .notifier_call= xen_panic_event,
1169 int xen_panic_handler_init(void)
1171 atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
1175 static const struct machine_ops xen_machine_ops __initconst = {
1176 .restart = xen_restart,
1177 .halt = xen_machine_halt,
1178 .power_off = xen_machine_power_off,
1179 .shutdown = xen_machine_halt,
1180 .crash_shutdown = xen_crash_shutdown,
1181 .emergency_restart = xen_emergency_restart,
1185 * Set up the GDT and segment registers for -fstack-protector. Until
1186 * we do this, we have to be careful not to call any stack-protected
1187 * function, which is most of the kernel.
1189 static void __init xen_setup_stackprotector(void)
1191 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
1192 pv_cpu_ops.load_gdt = xen_load_gdt_boot;
1194 setup_stack_canary_segment(0);
1195 switch_to_new_gdt(0);
1197 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
1198 pv_cpu_ops.load_gdt = xen_load_gdt;
1201 /* First C function to be called on Xen boot */
1202 asmlinkage void __init xen_start_kernel(void)
1204 struct physdev_set_iopl set_iopl;
1208 if (!xen_start_info)
1211 xen_domain_type = XEN_PV_DOMAIN;
1213 xen_setup_machphys_mapping();
1215 /* Install Xen paravirt ops */
1217 pv_init_ops = xen_init_ops;
1218 pv_cpu_ops = xen_cpu_ops;
1219 pv_apic_ops = xen_apic_ops;
1221 x86_init.resources.memory_setup = xen_memory_setup;
1222 x86_init.oem.arch_setup = xen_arch_setup;
1223 x86_init.oem.banner = xen_banner;
1225 xen_init_time_ops();
1228 * Set up some pagetable state before starting to set any ptes.
1233 /* Prevent unwanted bits from being set in PTEs. */
1234 __supported_pte_mask &= ~_PAGE_GLOBAL;
1236 if (!xen_initial_domain())
1238 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1240 __supported_pte_mask |= _PAGE_IOMAP;
1243 * Prevent page tables from being allocated in highmem, even
1244 * if CONFIG_HIGHPTE is enabled.
1246 __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
1248 /* Work out if we support NX */
1251 xen_setup_features();
1254 if (!xen_feature(XENFEAT_auto_translated_physmap))
1255 xen_build_dynamic_phys_to_machine();
1258 * Set up kernel GDT and segment registers, mainly so that
1259 * -fstack-protector code can be executed.
1261 xen_setup_stackprotector();
1264 xen_init_cpuid_mask();
1266 #ifdef CONFIG_X86_LOCAL_APIC
1268 * set up the basic apic ops.
1270 set_xen_basic_apic_ops();
1273 if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1274 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
1275 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
1278 machine_ops = xen_machine_ops;
1281 * The only reliable way to retain the initial address of the
1282 * percpu gdt_page is to remember it here, so we can go and
1283 * mark it RW later, when the initial percpu area is freed.
1285 xen_initial_gdt = &per_cpu(gdt_page, 0);
1289 #ifdef CONFIG_ACPI_NUMA
1291 * The pages we from Xen are not related to machine pages, so
1292 * any NUMA information the kernel tries to get from ACPI will
1293 * be meaningless. Prevent it from trying.
1298 pgd = (pgd_t *)xen_start_info->pt_base;
1300 /* Don't do the full vcpu_info placement stuff until we have a
1301 possible map and a non-dummy shared_info. */
1302 per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1304 local_irq_disable();
1305 early_boot_irqs_disabled = true;
1307 xen_raw_console_write("mapping kernel into physical memory\n");
1308 pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
1309 xen_ident_map_ISA();
1311 /* Allocate and initialize top and mid mfn levels for p2m structure */
1312 xen_build_mfn_list_list();
1314 /* keep using Xen gdt for now; no urgent need to change it */
1316 #ifdef CONFIG_X86_32
1317 pv_info.kernel_rpl = 1;
1318 if (xen_feature(XENFEAT_supervisor_mode_kernel))
1319 pv_info.kernel_rpl = 0;
1321 pv_info.kernel_rpl = 0;
1323 /* set the limit of our address space */
1326 /* We used to do this in xen_arch_setup, but that is too late on AMD
1327 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
1328 * which pokes 0xcf8 port.
1331 rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
1333 xen_raw_printk("physdev_op failed %d\n", rc);
1335 #ifdef CONFIG_X86_32
1336 /* set up basic CPUID stuff */
1337 cpu_detect(&new_cpu_data);
1338 new_cpu_data.hard_math = 1;
1339 new_cpu_data.wp_works_ok = 1;
1340 new_cpu_data.x86_capability[0] = cpuid_edx(1);
1343 /* Poke various useful things into boot_params */
1344 boot_params.hdr.type_of_loader = (9 << 4) | 0;
1345 boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1346 ? __pa(xen_start_info->mod_start) : 0;
1347 boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1348 boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1350 if (!xen_initial_domain()) {
1351 add_preferred_console("xenboot", 0, NULL);
1352 add_preferred_console("tty", 0, NULL);
1353 add_preferred_console("hvc", 0, NULL);
1355 x86_init.pci.arch_init = pci_xen_init;
1357 const struct dom0_vga_console_info *info =
1358 (void *)((char *)xen_start_info +
1359 xen_start_info->console.dom0.info_off);
1361 xen_init_vga(info, xen_start_info->console.dom0.info_size);
1362 xen_start_info->console.domU.mfn = 0;
1363 xen_start_info->console.domU.evtchn = 0;
1365 /* Make sure ACS will be enabled */
1370 xen_raw_console_write("about to get started...\n");
1372 xen_setup_runstate_info(0);
1374 /* Start the world */
1375 #ifdef CONFIG_X86_32
1376 i386_start_kernel();
1378 x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1382 static int init_hvm_pv_info(int *major, int *minor)
1384 uint32_t eax, ebx, ecx, edx, pages, msr, base;
1387 base = xen_cpuid_base();
1388 cpuid(base + 1, &eax, &ebx, &ecx, &edx);
1391 *minor = eax & 0xffff;
1392 printk(KERN_INFO "Xen version %d.%d.\n", *major, *minor);
1394 cpuid(base + 2, &pages, &msr, &ecx, &edx);
1396 pfn = __pa(hypercall_page);
1397 wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
1399 xen_setup_features();
1401 pv_info.name = "Xen HVM";
1403 xen_domain_type = XEN_HVM_DOMAIN;
1408 void __ref xen_hvm_init_shared_info(void)
1411 struct xen_add_to_physmap xatp;
1412 static struct shared_info *shared_info_page = 0;
1414 if (!shared_info_page)
1415 shared_info_page = (struct shared_info *)
1416 extend_brk(PAGE_SIZE, PAGE_SIZE);
1417 xatp.domid = DOMID_SELF;
1419 xatp.space = XENMAPSPACE_shared_info;
1420 xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
1421 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
1424 HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
1426 /* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
1427 * page, we use it in the event channel upcall and in some pvclock
1428 * related functions. We don't need the vcpu_info placement
1429 * optimizations because we don't use any pv_mmu or pv_irq op on
1431 * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
1432 * online but xen_hvm_init_shared_info is run at resume time too and
1433 * in that case multiple vcpus might be online. */
1434 for_each_online_cpu(cpu) {
1435 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
1439 #ifdef CONFIG_XEN_PVHVM
1440 static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self,
1441 unsigned long action, void *hcpu)
1443 int cpu = (long)hcpu;
1445 case CPU_UP_PREPARE:
1446 xen_vcpu_setup(cpu);
1447 if (xen_have_vector_callback)
1448 xen_init_lock_cpu(cpu);
1456 static struct notifier_block xen_hvm_cpu_notifier __cpuinitdata = {
1457 .notifier_call = xen_hvm_cpu_notify,
1460 static void __init xen_hvm_guest_init(void)
1465 r = init_hvm_pv_info(&major, &minor);
1469 xen_hvm_init_shared_info();
1471 if (xen_feature(XENFEAT_hvm_callback_vector))
1472 xen_have_vector_callback = 1;
1474 register_cpu_notifier(&xen_hvm_cpu_notifier);
1475 xen_unplug_emulated_devices();
1476 x86_init.irqs.intr_init = xen_init_IRQ;
1477 xen_hvm_init_time_ops();
1478 xen_hvm_init_mmu_ops();
1481 static bool __init xen_hvm_platform(void)
1483 if (xen_pv_domain())
1486 if (!xen_cpuid_base())
1492 bool xen_hvm_need_lapic(void)
1494 if (xen_pv_domain())
1496 if (!xen_hvm_domain())
1498 if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback)
1502 EXPORT_SYMBOL_GPL(xen_hvm_need_lapic);
1504 const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
1506 .detect = xen_hvm_platform,
1507 .init_platform = xen_hvm_guest_init,
1509 EXPORT_SYMBOL(x86_hyper_xen_hvm);