1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstDX (8<<1) /* Destination is in DX register */
51 #define DstMask (0xf<<1)
52 /* Source operand type. */
53 #define SrcNone (0<<5) /* No source operand. */
54 #define SrcReg (1<<5) /* Register operand. */
55 #define SrcMem (2<<5) /* Memory operand. */
56 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
57 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
58 #define SrcImm (5<<5) /* Immediate operand. */
59 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
60 #define SrcOne (7<<5) /* Implied '1' */
61 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
62 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
63 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
64 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
65 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
66 #define SrcAcc (0xd<<5) /* Source Accumulator */
67 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
68 #define SrcDX (0xf<<5) /* Source is in DX register */
69 #define SrcMask (0xf<<5)
70 /* Generic ModRM decode. */
72 /* Destination is only written; never read. */
75 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
76 #define String (1<<13) /* String instruction (rep capable) */
77 #define Stack (1<<14) /* Stack instruction (push/pop) */
78 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
79 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
80 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
81 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
82 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
83 #define Sse (1<<18) /* SSE Vector instruction */
85 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
86 #define VendorSpecific (1<<22) /* Vendor specific instruction */
87 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
88 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
89 #define Undefined (1<<25) /* No Such Instruction */
90 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
91 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
93 /* Source 2 operand type */
94 #define Src2None (0<<29)
95 #define Src2CL (1<<29)
96 #define Src2ImmByte (2<<29)
97 #define Src2One (3<<29)
98 #define Src2Imm (4<<29)
99 #define Src2Mask (7<<29)
101 #define X2(x...) x, x
102 #define X3(x...) X2(x), x
103 #define X4(x...) X2(x), X2(x)
104 #define X5(x...) X4(x), x
105 #define X6(x...) X4(x), X2(x)
106 #define X7(x...) X4(x), X3(x)
107 #define X8(x...) X4(x), X4(x)
108 #define X16(x...) X8(x), X8(x)
114 int (*execute)(struct x86_emulate_ctxt *ctxt);
115 struct opcode *group;
116 struct group_dual *gdual;
117 struct gprefix *gprefix;
119 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
123 struct opcode mod012[8];
124 struct opcode mod3[8];
128 struct opcode pfx_no;
129 struct opcode pfx_66;
130 struct opcode pfx_f2;
131 struct opcode pfx_f3;
134 /* EFLAGS bit definitions. */
135 #define EFLG_ID (1<<21)
136 #define EFLG_VIP (1<<20)
137 #define EFLG_VIF (1<<19)
138 #define EFLG_AC (1<<18)
139 #define EFLG_VM (1<<17)
140 #define EFLG_RF (1<<16)
141 #define EFLG_IOPL (3<<12)
142 #define EFLG_NT (1<<14)
143 #define EFLG_OF (1<<11)
144 #define EFLG_DF (1<<10)
145 #define EFLG_IF (1<<9)
146 #define EFLG_TF (1<<8)
147 #define EFLG_SF (1<<7)
148 #define EFLG_ZF (1<<6)
149 #define EFLG_AF (1<<4)
150 #define EFLG_PF (1<<2)
151 #define EFLG_CF (1<<0)
153 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
154 #define EFLG_RESERVED_ONE_MASK 2
157 * Instruction emulation:
158 * Most instructions are emulated directly via a fragment of inline assembly
159 * code. This allows us to save/restore EFLAGS and thus very easily pick up
160 * any modified flags.
163 #if defined(CONFIG_X86_64)
164 #define _LO32 "k" /* force 32-bit operand */
165 #define _STK "%%rsp" /* stack pointer */
166 #elif defined(__i386__)
167 #define _LO32 "" /* force 32-bit operand */
168 #define _STK "%%esp" /* stack pointer */
172 * These EFLAGS bits are restored from saved value during emulation, and
173 * any changes are written back to the saved value after emulation.
175 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
177 /* Before executing instruction: restore necessary bits in EFLAGS. */
178 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
179 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
180 "movl %"_sav",%"_LO32 _tmp"; " \
183 "movl %"_msk",%"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
186 "notl %"_LO32 _tmp"; " \
187 "andl %"_LO32 _tmp",("_STK"); " \
188 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
190 "orl %"_LO32 _tmp",("_STK"); " \
194 /* After executing instruction: write-back necessary bits in EFLAGS. */
195 #define _POST_EFLAGS(_sav, _msk, _tmp) \
196 /* _sav |= EFLAGS & _msk; */ \
199 "andl %"_msk",%"_LO32 _tmp"; " \
200 "orl %"_LO32 _tmp",%"_sav"; "
208 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
210 __asm__ __volatile__ ( \
211 _PRE_EFLAGS("0", "4", "2") \
212 _op _suffix " %"_x"3,%1; " \
213 _POST_EFLAGS("0", "4", "2") \
214 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
216 : _y ((_src).val), "i" (EFLAGS_MASK)); \
220 /* Raw emulation: instruction has two explicit operands. */
221 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
223 unsigned long _tmp; \
225 switch ((_dst).bytes) { \
227 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
230 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
233 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
238 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
240 unsigned long _tmp; \
241 switch ((_dst).bytes) { \
243 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
246 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
247 _wx, _wy, _lx, _ly, _qx, _qy); \
252 /* Source operand is byte-sized and may be restricted to just %cl. */
253 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
254 __emulate_2op(_op, _src, _dst, _eflags, \
255 "b", "c", "b", "c", "b", "c", "b", "c")
257 /* Source operand is byte, word, long or quad sized. */
258 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
259 __emulate_2op(_op, _src, _dst, _eflags, \
260 "b", "q", "w", "r", _LO32, "r", "", "r")
262 /* Source operand is word, long or quad sized. */
263 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
264 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
265 "w", "r", _LO32, "r", "", "r")
267 /* Instruction has three operands and one operand is stored in ECX register */
268 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
270 unsigned long _tmp; \
271 _type _clv = (_cl).val; \
272 _type _srcv = (_src).val; \
273 _type _dstv = (_dst).val; \
275 __asm__ __volatile__ ( \
276 _PRE_EFLAGS("0", "5", "2") \
277 _op _suffix " %4,%1 \n" \
278 _POST_EFLAGS("0", "5", "2") \
279 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
280 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
283 (_cl).val = (unsigned long) _clv; \
284 (_src).val = (unsigned long) _srcv; \
285 (_dst).val = (unsigned long) _dstv; \
288 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
290 switch ((_dst).bytes) { \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "w", unsigned short); \
296 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "l", unsigned int); \
300 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
301 "q", unsigned long)); \
306 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
308 unsigned long _tmp; \
310 __asm__ __volatile__ ( \
311 _PRE_EFLAGS("0", "3", "2") \
312 _op _suffix " %1; " \
313 _POST_EFLAGS("0", "3", "2") \
314 : "=m" (_eflags), "+m" ((_dst).val), \
316 : "i" (EFLAGS_MASK)); \
319 /* Instruction has only one explicit operand (no source operand). */
320 #define emulate_1op(_op, _dst, _eflags) \
322 switch ((_dst).bytes) { \
323 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
324 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
325 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
326 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
330 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
332 unsigned long _tmp; \
334 __asm__ __volatile__ ( \
335 _PRE_EFLAGS("0", "4", "1") \
336 _op _suffix " %5; " \
337 _POST_EFLAGS("0", "4", "1") \
338 : "=m" (_eflags), "=&r" (_tmp), \
339 "+a" (_rax), "+d" (_rdx) \
340 : "i" (EFLAGS_MASK), "m" ((_src).val), \
341 "a" (_rax), "d" (_rdx)); \
344 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "5", "1") \
351 _op _suffix " %6; " \
353 _POST_EFLAGS("0", "5", "1") \
354 ".pushsection .fixup,\"ax\" \n\t" \
355 "3: movb $1, %4 \n\t" \
358 _ASM_EXTABLE(1b, 3b) \
359 : "=m" (_eflags), "=&r" (_tmp), \
360 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
361 : "i" (EFLAGS_MASK), "m" ((_src).val), \
362 "a" (_rax), "d" (_rdx)); \
365 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
366 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
368 switch((_src).bytes) { \
370 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
374 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
378 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
382 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
388 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
390 switch((_src).bytes) { \
392 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
393 _eflags, "b", _ex); \
396 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
397 _eflags, "w", _ex); \
400 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
401 _eflags, "l", _ex); \
404 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
405 _eflags, "q", _ex)); \
410 /* Fetch next part of the instruction being emulated. */
411 #define insn_fetch(_type, _size, _eip) \
412 ({ unsigned long _x; \
413 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
414 if (rc != X86EMUL_CONTINUE) \
420 #define insn_fetch_arr(_arr, _size, _eip) \
421 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
422 if (rc != X86EMUL_CONTINUE) \
427 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
428 enum x86_intercept intercept,
429 enum x86_intercept_stage stage)
431 struct x86_instruction_info info = {
432 .intercept = intercept,
433 .rep_prefix = ctxt->decode.rep_prefix,
434 .modrm_mod = ctxt->decode.modrm_mod,
435 .modrm_reg = ctxt->decode.modrm_reg,
436 .modrm_rm = ctxt->decode.modrm_rm,
437 .src_val = ctxt->decode.src.val64,
438 .src_bytes = ctxt->decode.src.bytes,
439 .dst_bytes = ctxt->decode.dst.bytes,
440 .ad_bytes = ctxt->decode.ad_bytes,
441 .next_rip = ctxt->eip,
444 return ctxt->ops->intercept(ctxt, &info, stage);
447 static inline unsigned long ad_mask(struct decode_cache *c)
449 return (1UL << (c->ad_bytes << 3)) - 1;
452 /* Access/update address held in a register, based on addressing mode. */
453 static inline unsigned long
454 address_mask(struct decode_cache *c, unsigned long reg)
456 if (c->ad_bytes == sizeof(unsigned long))
459 return reg & ad_mask(c);
462 static inline unsigned long
463 register_address(struct decode_cache *c, unsigned long reg)
465 return address_mask(c, reg);
469 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
471 if (c->ad_bytes == sizeof(unsigned long))
474 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
477 static inline void jmp_rel(struct decode_cache *c, int rel)
479 register_address_increment(c, &c->eip, rel);
482 static u32 desc_limit_scaled(struct desc_struct *desc)
484 u32 limit = get_desc_limit(desc);
486 return desc->g ? (limit << 12) | 0xfff : limit;
489 static void set_seg_override(struct decode_cache *c, int seg)
491 c->has_seg_override = true;
492 c->seg_override = seg;
495 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
496 struct x86_emulate_ops *ops, int seg)
498 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
501 return ops->get_cached_segment_base(ctxt, seg);
504 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
505 struct decode_cache *c)
507 if (!c->has_seg_override)
510 return c->seg_override;
513 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
514 u32 error, bool valid)
516 ctxt->exception.vector = vec;
517 ctxt->exception.error_code = error;
518 ctxt->exception.error_code_valid = valid;
519 return X86EMUL_PROPAGATE_FAULT;
522 static int emulate_db(struct x86_emulate_ctxt *ctxt)
524 return emulate_exception(ctxt, DB_VECTOR, 0, false);
527 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
529 return emulate_exception(ctxt, GP_VECTOR, err, true);
532 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
534 return emulate_exception(ctxt, SS_VECTOR, err, true);
537 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
539 return emulate_exception(ctxt, UD_VECTOR, 0, false);
542 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
544 return emulate_exception(ctxt, TS_VECTOR, err, true);
547 static int emulate_de(struct x86_emulate_ctxt *ctxt)
549 return emulate_exception(ctxt, DE_VECTOR, 0, false);
552 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
554 return emulate_exception(ctxt, NM_VECTOR, 0, false);
557 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
560 struct desc_struct desc;
562 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
566 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
571 struct desc_struct desc;
573 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
574 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
577 static int __linearize(struct x86_emulate_ctxt *ctxt,
578 struct segmented_address addr,
579 unsigned size, bool write, bool fetch,
582 struct decode_cache *c = &ctxt->decode;
583 struct desc_struct desc;
590 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
591 switch (ctxt->mode) {
592 case X86EMUL_MODE_REAL:
594 case X86EMUL_MODE_PROT64:
595 if (((signed long)la << 16) >> 16 != la)
596 return emulate_gp(ctxt, 0);
599 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
603 /* code segment or read-only data segment */
604 if (((desc.type & 8) || !(desc.type & 2)) && write)
606 /* unreadable code segment */
607 if (!fetch && (desc.type & 8) && !(desc.type & 2))
609 lim = desc_limit_scaled(&desc);
610 if ((desc.type & 8) || !(desc.type & 4)) {
611 /* expand-up segment */
612 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
615 /* exapand-down segment */
616 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
618 lim = desc.d ? 0xffffffff : 0xffff;
619 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
622 cpl = ctxt->ops->cpl(ctxt);
625 if (!(desc.type & 8)) {
629 } else if ((desc.type & 8) && !(desc.type & 4)) {
630 /* nonconforming code segment */
633 } else if ((desc.type & 8) && (desc.type & 4)) {
634 /* conforming code segment */
640 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
643 return X86EMUL_CONTINUE;
645 if (addr.seg == VCPU_SREG_SS)
646 return emulate_ss(ctxt, addr.seg);
648 return emulate_gp(ctxt, addr.seg);
651 static int linearize(struct x86_emulate_ctxt *ctxt,
652 struct segmented_address addr,
653 unsigned size, bool write,
656 return __linearize(ctxt, addr, size, write, false, linear);
660 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
661 struct segmented_address addr,
668 rc = linearize(ctxt, addr, size, false, &linear);
669 if (rc != X86EMUL_CONTINUE)
671 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
674 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
675 struct x86_emulate_ops *ops,
676 unsigned long eip, u8 *dest)
678 struct fetch_cache *fc = &ctxt->decode.fetch;
682 if (eip == fc->end) {
683 unsigned long linear;
684 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
685 cur_size = fc->end - fc->start;
686 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
687 rc = __linearize(ctxt, addr, size, false, true, &linear);
688 if (rc != X86EMUL_CONTINUE)
690 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
691 size, &ctxt->exception);
692 if (rc != X86EMUL_CONTINUE)
696 *dest = fc->data[eip - fc->start];
697 return X86EMUL_CONTINUE;
700 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
701 struct x86_emulate_ops *ops,
702 unsigned long eip, void *dest, unsigned size)
706 /* x86 instructions are limited to 15 bytes. */
707 if (eip + size - ctxt->eip > 15)
708 return X86EMUL_UNHANDLEABLE;
710 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
711 if (rc != X86EMUL_CONTINUE)
714 return X86EMUL_CONTINUE;
718 * Given the 'reg' portion of a ModRM byte, and a register block, return a
719 * pointer into the block that addresses the relevant register.
720 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
722 static void *decode_register(u8 modrm_reg, unsigned long *regs,
727 p = ®s[modrm_reg];
728 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
729 p = (unsigned char *)®s[modrm_reg & 3] + 1;
733 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
734 struct segmented_address addr,
735 u16 *size, unsigned long *address, int op_bytes)
742 rc = segmented_read_std(ctxt, addr, size, 2);
743 if (rc != X86EMUL_CONTINUE)
746 rc = segmented_read_std(ctxt, addr, address, op_bytes);
750 static int test_cc(unsigned int condition, unsigned int flags)
754 switch ((condition & 15) >> 1) {
756 rc |= (flags & EFLG_OF);
758 case 1: /* b/c/nae */
759 rc |= (flags & EFLG_CF);
762 rc |= (flags & EFLG_ZF);
765 rc |= (flags & (EFLG_CF|EFLG_ZF));
768 rc |= (flags & EFLG_SF);
771 rc |= (flags & EFLG_PF);
774 rc |= (flags & EFLG_ZF);
777 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
781 /* Odd condition identifiers (lsb == 1) have inverted sense. */
782 return (!!rc ^ (condition & 1));
785 static void fetch_register_operand(struct operand *op)
789 op->val = *(u8 *)op->addr.reg;
792 op->val = *(u16 *)op->addr.reg;
795 op->val = *(u32 *)op->addr.reg;
798 op->val = *(u64 *)op->addr.reg;
803 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
805 ctxt->ops->get_fpu(ctxt);
807 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
808 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
809 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
810 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
811 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
812 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
813 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
814 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
816 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
817 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
818 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
819 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
820 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
821 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
822 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
823 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
827 ctxt->ops->put_fpu(ctxt);
830 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
833 ctxt->ops->get_fpu(ctxt);
835 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
836 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
837 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
838 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
839 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
840 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
841 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
842 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
844 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
845 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
846 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
847 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
848 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
849 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
850 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
851 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
855 ctxt->ops->put_fpu(ctxt);
858 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
860 struct decode_cache *c,
863 unsigned reg = c->modrm_reg;
864 int highbyte_regs = c->rex_prefix == 0;
867 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
873 read_sse_reg(ctxt, &op->vec_val, reg);
878 if ((c->d & ByteOp) && !inhibit_bytereg) {
879 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
882 op->addr.reg = decode_register(reg, c->regs, 0);
883 op->bytes = c->op_bytes;
885 fetch_register_operand(op);
886 op->orig_val = op->val;
889 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
890 struct x86_emulate_ops *ops,
893 struct decode_cache *c = &ctxt->decode;
895 int index_reg = 0, base_reg = 0, scale;
896 int rc = X86EMUL_CONTINUE;
900 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
901 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
902 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
905 c->modrm = insn_fetch(u8, 1, c->eip);
906 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
907 c->modrm_reg |= (c->modrm & 0x38) >> 3;
908 c->modrm_rm |= (c->modrm & 0x07);
909 c->modrm_seg = VCPU_SREG_DS;
911 if (c->modrm_mod == 3) {
913 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
914 op->addr.reg = decode_register(c->modrm_rm,
915 c->regs, c->d & ByteOp);
919 op->addr.xmm = c->modrm_rm;
920 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
923 fetch_register_operand(op);
929 if (c->ad_bytes == 2) {
930 unsigned bx = c->regs[VCPU_REGS_RBX];
931 unsigned bp = c->regs[VCPU_REGS_RBP];
932 unsigned si = c->regs[VCPU_REGS_RSI];
933 unsigned di = c->regs[VCPU_REGS_RDI];
935 /* 16-bit ModR/M decode. */
936 switch (c->modrm_mod) {
938 if (c->modrm_rm == 6)
939 modrm_ea += insn_fetch(u16, 2, c->eip);
942 modrm_ea += insn_fetch(s8, 1, c->eip);
945 modrm_ea += insn_fetch(u16, 2, c->eip);
948 switch (c->modrm_rm) {
968 if (c->modrm_mod != 0)
975 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
976 (c->modrm_rm == 6 && c->modrm_mod != 0))
977 c->modrm_seg = VCPU_SREG_SS;
978 modrm_ea = (u16)modrm_ea;
980 /* 32/64-bit ModR/M decode. */
981 if ((c->modrm_rm & 7) == 4) {
982 sib = insn_fetch(u8, 1, c->eip);
983 index_reg |= (sib >> 3) & 7;
987 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
988 modrm_ea += insn_fetch(s32, 4, c->eip);
990 modrm_ea += c->regs[base_reg];
992 modrm_ea += c->regs[index_reg] << scale;
993 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
994 if (ctxt->mode == X86EMUL_MODE_PROT64)
997 modrm_ea += c->regs[c->modrm_rm];
998 switch (c->modrm_mod) {
1000 if (c->modrm_rm == 5)
1001 modrm_ea += insn_fetch(s32, 4, c->eip);
1004 modrm_ea += insn_fetch(s8, 1, c->eip);
1007 modrm_ea += insn_fetch(s32, 4, c->eip);
1011 op->addr.mem.ea = modrm_ea;
1016 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1017 struct x86_emulate_ops *ops,
1020 struct decode_cache *c = &ctxt->decode;
1021 int rc = X86EMUL_CONTINUE;
1024 switch (c->ad_bytes) {
1026 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1029 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1032 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1039 static void fetch_bit_operand(struct decode_cache *c)
1043 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1044 mask = ~(c->dst.bytes * 8 - 1);
1046 if (c->src.bytes == 2)
1047 sv = (s16)c->src.val & (s16)mask;
1048 else if (c->src.bytes == 4)
1049 sv = (s32)c->src.val & (s32)mask;
1051 c->dst.addr.mem.ea += (sv >> 3);
1054 /* only subword offset */
1055 c->src.val &= (c->dst.bytes << 3) - 1;
1058 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1059 struct x86_emulate_ops *ops,
1060 unsigned long addr, void *dest, unsigned size)
1063 struct read_cache *mc = &ctxt->decode.mem_read;
1066 int n = min(size, 8u);
1068 if (mc->pos < mc->end)
1071 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1073 if (rc != X86EMUL_CONTINUE)
1078 memcpy(dest, mc->data + mc->pos, n);
1083 return X86EMUL_CONTINUE;
1086 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1087 struct segmented_address addr,
1094 rc = linearize(ctxt, addr, size, false, &linear);
1095 if (rc != X86EMUL_CONTINUE)
1097 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1100 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1101 struct segmented_address addr,
1108 rc = linearize(ctxt, addr, size, true, &linear);
1109 if (rc != X86EMUL_CONTINUE)
1111 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1115 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1116 struct segmented_address addr,
1117 const void *orig_data, const void *data,
1123 rc = linearize(ctxt, addr, size, true, &linear);
1124 if (rc != X86EMUL_CONTINUE)
1126 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1127 size, &ctxt->exception);
1130 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1131 struct x86_emulate_ops *ops,
1132 unsigned int size, unsigned short port,
1135 struct read_cache *rc = &ctxt->decode.io_read;
1137 if (rc->pos == rc->end) { /* refill pio read ahead */
1138 struct decode_cache *c = &ctxt->decode;
1139 unsigned int in_page, n;
1140 unsigned int count = c->rep_prefix ?
1141 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1142 in_page = (ctxt->eflags & EFLG_DF) ?
1143 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1144 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1145 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1149 rc->pos = rc->end = 0;
1150 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1155 memcpy(dest, rc->data + rc->pos, size);
1160 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1161 struct x86_emulate_ops *ops,
1162 u16 selector, struct desc_ptr *dt)
1164 if (selector & 1 << 2) {
1165 struct desc_struct desc;
1168 memset (dt, 0, sizeof *dt);
1169 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1172 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1173 dt->address = get_desc_base(&desc);
1175 ops->get_gdt(ctxt, dt);
1178 /* allowed just for 8 bytes segments */
1179 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1180 struct x86_emulate_ops *ops,
1181 u16 selector, struct desc_struct *desc)
1184 u16 index = selector >> 3;
1188 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1190 if (dt.size < index * 8 + 7)
1191 return emulate_gp(ctxt, selector & 0xfffc);
1192 addr = dt.address + index * 8;
1193 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1198 /* allowed just for 8 bytes segments */
1199 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1200 struct x86_emulate_ops *ops,
1201 u16 selector, struct desc_struct *desc)
1204 u16 index = selector >> 3;
1208 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1210 if (dt.size < index * 8 + 7)
1211 return emulate_gp(ctxt, selector & 0xfffc);
1213 addr = dt.address + index * 8;
1214 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1219 /* Does not support long mode */
1220 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1221 struct x86_emulate_ops *ops,
1222 u16 selector, int seg)
1224 struct desc_struct seg_desc;
1226 unsigned err_vec = GP_VECTOR;
1228 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1231 memset(&seg_desc, 0, sizeof seg_desc);
1233 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1234 || ctxt->mode == X86EMUL_MODE_REAL) {
1235 /* set real mode segment descriptor */
1236 set_desc_base(&seg_desc, selector << 4);
1237 set_desc_limit(&seg_desc, 0xffff);
1244 /* NULL selector is not valid for TR, CS and SS */
1245 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1249 /* TR should be in GDT only */
1250 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1253 if (null_selector) /* for NULL selector skip all following checks */
1256 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1257 if (ret != X86EMUL_CONTINUE)
1260 err_code = selector & 0xfffc;
1261 err_vec = GP_VECTOR;
1263 /* can't load system descriptor into segment selecor */
1264 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1268 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1274 cpl = ops->cpl(ctxt);
1279 * segment is not a writable data segment or segment
1280 * selector's RPL != CPL or segment selector's RPL != CPL
1282 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1286 if (!(seg_desc.type & 8))
1289 if (seg_desc.type & 4) {
1295 if (rpl > cpl || dpl != cpl)
1298 /* CS(RPL) <- CPL */
1299 selector = (selector & 0xfffc) | cpl;
1302 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1305 case VCPU_SREG_LDTR:
1306 if (seg_desc.s || seg_desc.type != 2)
1309 default: /* DS, ES, FS, or GS */
1311 * segment is not a data or readable code segment or
1312 * ((segment is a data or nonconforming code segment)
1313 * and (both RPL and CPL > DPL))
1315 if ((seg_desc.type & 0xa) == 0x8 ||
1316 (((seg_desc.type & 0xc) != 0xc) &&
1317 (rpl > dpl && cpl > dpl)))
1323 /* mark segment as accessed */
1325 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1326 if (ret != X86EMUL_CONTINUE)
1330 ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1331 return X86EMUL_CONTINUE;
1333 emulate_exception(ctxt, err_vec, err_code, true);
1334 return X86EMUL_PROPAGATE_FAULT;
1337 static void write_register_operand(struct operand *op)
1339 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1340 switch (op->bytes) {
1342 *(u8 *)op->addr.reg = (u8)op->val;
1345 *(u16 *)op->addr.reg = (u16)op->val;
1348 *op->addr.reg = (u32)op->val;
1349 break; /* 64b: zero-extend */
1351 *op->addr.reg = op->val;
1356 static int writeback(struct x86_emulate_ctxt *ctxt)
1359 struct decode_cache *c = &ctxt->decode;
1361 switch (c->dst.type) {
1363 write_register_operand(&c->dst);
1367 rc = segmented_cmpxchg(ctxt,
1373 rc = segmented_write(ctxt,
1377 if (rc != X86EMUL_CONTINUE)
1381 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1389 return X86EMUL_CONTINUE;
1392 static int em_push(struct x86_emulate_ctxt *ctxt)
1394 struct decode_cache *c = &ctxt->decode;
1395 struct segmented_address addr;
1397 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1398 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1399 addr.seg = VCPU_SREG_SS;
1401 /* Disable writeback. */
1402 c->dst.type = OP_NONE;
1403 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1406 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1407 void *dest, int len)
1409 struct decode_cache *c = &ctxt->decode;
1411 struct segmented_address addr;
1413 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1414 addr.seg = VCPU_SREG_SS;
1415 rc = segmented_read(ctxt, addr, dest, len);
1416 if (rc != X86EMUL_CONTINUE)
1419 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1423 static int em_pop(struct x86_emulate_ctxt *ctxt)
1425 struct decode_cache *c = &ctxt->decode;
1427 return emulate_pop(ctxt, &c->dst.val, c->op_bytes);
1430 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1431 struct x86_emulate_ops *ops,
1432 void *dest, int len)
1435 unsigned long val, change_mask;
1436 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1437 int cpl = ops->cpl(ctxt);
1439 rc = emulate_pop(ctxt, &val, len);
1440 if (rc != X86EMUL_CONTINUE)
1443 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1444 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1446 switch(ctxt->mode) {
1447 case X86EMUL_MODE_PROT64:
1448 case X86EMUL_MODE_PROT32:
1449 case X86EMUL_MODE_PROT16:
1451 change_mask |= EFLG_IOPL;
1453 change_mask |= EFLG_IF;
1455 case X86EMUL_MODE_VM86:
1457 return emulate_gp(ctxt, 0);
1458 change_mask |= EFLG_IF;
1460 default: /* real mode */
1461 change_mask |= (EFLG_IOPL | EFLG_IF);
1465 *(unsigned long *)dest =
1466 (ctxt->eflags & ~change_mask) | (val & change_mask);
1471 static int em_popf(struct x86_emulate_ctxt *ctxt)
1473 struct decode_cache *c = &ctxt->decode;
1475 c->dst.type = OP_REG;
1476 c->dst.addr.reg = &ctxt->eflags;
1477 c->dst.bytes = c->op_bytes;
1478 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1481 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1482 struct x86_emulate_ops *ops, int seg)
1484 struct decode_cache *c = &ctxt->decode;
1486 c->src.val = get_segment_selector(ctxt, seg);
1488 return em_push(ctxt);
1491 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1492 struct x86_emulate_ops *ops, int seg)
1494 struct decode_cache *c = &ctxt->decode;
1495 unsigned long selector;
1498 rc = emulate_pop(ctxt, &selector, c->op_bytes);
1499 if (rc != X86EMUL_CONTINUE)
1502 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1506 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1508 struct decode_cache *c = &ctxt->decode;
1509 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1510 int rc = X86EMUL_CONTINUE;
1511 int reg = VCPU_REGS_RAX;
1513 while (reg <= VCPU_REGS_RDI) {
1514 (reg == VCPU_REGS_RSP) ?
1515 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1518 if (rc != X86EMUL_CONTINUE)
1527 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1529 struct decode_cache *c = &ctxt->decode;
1531 c->src.val = (unsigned long)ctxt->eflags;
1532 return em_push(ctxt);
1535 static int em_popa(struct x86_emulate_ctxt *ctxt)
1537 struct decode_cache *c = &ctxt->decode;
1538 int rc = X86EMUL_CONTINUE;
1539 int reg = VCPU_REGS_RDI;
1541 while (reg >= VCPU_REGS_RAX) {
1542 if (reg == VCPU_REGS_RSP) {
1543 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1548 rc = emulate_pop(ctxt, &c->regs[reg], c->op_bytes);
1549 if (rc != X86EMUL_CONTINUE)
1556 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1557 struct x86_emulate_ops *ops, int irq)
1559 struct decode_cache *c = &ctxt->decode;
1566 /* TODO: Add limit checks */
1567 c->src.val = ctxt->eflags;
1569 if (rc != X86EMUL_CONTINUE)
1572 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1574 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1576 if (rc != X86EMUL_CONTINUE)
1579 c->src.val = c->eip;
1581 if (rc != X86EMUL_CONTINUE)
1584 ops->get_idt(ctxt, &dt);
1586 eip_addr = dt.address + (irq << 2);
1587 cs_addr = dt.address + (irq << 2) + 2;
1589 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1590 if (rc != X86EMUL_CONTINUE)
1593 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1598 if (rc != X86EMUL_CONTINUE)
1606 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1607 struct x86_emulate_ops *ops, int irq)
1609 switch(ctxt->mode) {
1610 case X86EMUL_MODE_REAL:
1611 return emulate_int_real(ctxt, ops, irq);
1612 case X86EMUL_MODE_VM86:
1613 case X86EMUL_MODE_PROT16:
1614 case X86EMUL_MODE_PROT32:
1615 case X86EMUL_MODE_PROT64:
1617 /* Protected mode interrupts unimplemented yet */
1618 return X86EMUL_UNHANDLEABLE;
1622 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1623 struct x86_emulate_ops *ops)
1625 struct decode_cache *c = &ctxt->decode;
1626 int rc = X86EMUL_CONTINUE;
1627 unsigned long temp_eip = 0;
1628 unsigned long temp_eflags = 0;
1629 unsigned long cs = 0;
1630 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1631 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1632 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1633 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1635 /* TODO: Add stack limit check */
1637 rc = emulate_pop(ctxt, &temp_eip, c->op_bytes);
1639 if (rc != X86EMUL_CONTINUE)
1642 if (temp_eip & ~0xffff)
1643 return emulate_gp(ctxt, 0);
1645 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1647 if (rc != X86EMUL_CONTINUE)
1650 rc = emulate_pop(ctxt, &temp_eflags, c->op_bytes);
1652 if (rc != X86EMUL_CONTINUE)
1655 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1657 if (rc != X86EMUL_CONTINUE)
1663 if (c->op_bytes == 4)
1664 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1665 else if (c->op_bytes == 2) {
1666 ctxt->eflags &= ~0xffff;
1667 ctxt->eflags |= temp_eflags;
1670 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1671 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1676 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1677 struct x86_emulate_ops* ops)
1679 switch(ctxt->mode) {
1680 case X86EMUL_MODE_REAL:
1681 return emulate_iret_real(ctxt, ops);
1682 case X86EMUL_MODE_VM86:
1683 case X86EMUL_MODE_PROT16:
1684 case X86EMUL_MODE_PROT32:
1685 case X86EMUL_MODE_PROT64:
1687 /* iret from protected mode unimplemented yet */
1688 return X86EMUL_UNHANDLEABLE;
1692 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1694 struct decode_cache *c = &ctxt->decode;
1698 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1700 rc = load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS);
1701 if (rc != X86EMUL_CONTINUE)
1705 memcpy(&c->eip, c->src.valptr, c->op_bytes);
1706 return X86EMUL_CONTINUE;
1709 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1711 struct decode_cache *c = &ctxt->decode;
1713 return emulate_pop(ctxt, &c->dst.val, c->dst.bytes);
1716 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1718 struct decode_cache *c = &ctxt->decode;
1719 switch (c->modrm_reg) {
1721 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1724 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1727 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1730 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1732 case 4: /* sal/shl */
1733 case 6: /* sal/shl */
1734 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1737 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1740 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1743 return X86EMUL_CONTINUE;
1746 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1748 struct decode_cache *c = &ctxt->decode;
1749 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1750 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1753 switch (c->modrm_reg) {
1754 case 0 ... 1: /* test */
1755 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1758 c->dst.val = ~c->dst.val;
1761 emulate_1op("neg", c->dst, ctxt->eflags);
1764 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1767 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1770 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1774 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1778 return X86EMUL_UNHANDLEABLE;
1781 return emulate_de(ctxt);
1782 return X86EMUL_CONTINUE;
1785 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1787 struct decode_cache *c = &ctxt->decode;
1788 int rc = X86EMUL_CONTINUE;
1790 switch (c->modrm_reg) {
1792 emulate_1op("inc", c->dst, ctxt->eflags);
1795 emulate_1op("dec", c->dst, ctxt->eflags);
1797 case 2: /* call near abs */ {
1800 c->eip = c->src.val;
1801 c->src.val = old_eip;
1805 case 4: /* jmp abs */
1806 c->eip = c->src.val;
1808 case 5: /* jmp far */
1809 rc = em_jmp_far(ctxt);
1818 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1820 struct decode_cache *c = &ctxt->decode;
1821 u64 old = c->dst.orig_val64;
1823 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1824 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1825 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1826 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1827 ctxt->eflags &= ~EFLG_ZF;
1829 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1830 (u32) c->regs[VCPU_REGS_RBX];
1832 ctxt->eflags |= EFLG_ZF;
1834 return X86EMUL_CONTINUE;
1837 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1838 struct x86_emulate_ops *ops)
1840 struct decode_cache *c = &ctxt->decode;
1844 rc = emulate_pop(ctxt, &c->eip, c->op_bytes);
1845 if (rc != X86EMUL_CONTINUE)
1847 if (c->op_bytes == 4)
1848 c->eip = (u32)c->eip;
1849 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1850 if (rc != X86EMUL_CONTINUE)
1852 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1856 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1857 struct x86_emulate_ops *ops, int seg)
1859 struct decode_cache *c = &ctxt->decode;
1863 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1865 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1866 if (rc != X86EMUL_CONTINUE)
1869 c->dst.val = c->src.val;
1874 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1875 struct x86_emulate_ops *ops, struct desc_struct *cs,
1876 struct desc_struct *ss)
1880 memset(cs, 0, sizeof(struct desc_struct));
1881 ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1882 memset(ss, 0, sizeof(struct desc_struct));
1884 cs->l = 0; /* will be adjusted later */
1885 set_desc_base(cs, 0); /* flat segment */
1886 cs->g = 1; /* 4kb granularity */
1887 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1888 cs->type = 0x0b; /* Read, Execute, Accessed */
1890 cs->dpl = 0; /* will be adjusted later */
1894 set_desc_base(ss, 0); /* flat segment */
1895 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1896 ss->g = 1; /* 4kb granularity */
1898 ss->type = 0x03; /* Read/Write, Accessed */
1899 ss->d = 1; /* 32bit stack segment */
1905 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1907 struct decode_cache *c = &ctxt->decode;
1908 struct desc_struct cs, ss;
1913 /* syscall is not available in real mode */
1914 if (ctxt->mode == X86EMUL_MODE_REAL ||
1915 ctxt->mode == X86EMUL_MODE_VM86)
1916 return emulate_ud(ctxt);
1918 ops->get_msr(ctxt, MSR_EFER, &efer);
1919 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1921 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1923 cs_sel = (u16)(msr_data & 0xfffc);
1924 ss_sel = (u16)(msr_data + 8);
1926 if (efer & EFER_LMA) {
1930 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1931 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1933 c->regs[VCPU_REGS_RCX] = c->eip;
1934 if (efer & EFER_LMA) {
1935 #ifdef CONFIG_X86_64
1936 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1939 ctxt->mode == X86EMUL_MODE_PROT64 ?
1940 MSR_LSTAR : MSR_CSTAR, &msr_data);
1943 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1944 ctxt->eflags &= ~(msr_data | EFLG_RF);
1948 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1949 c->eip = (u32)msr_data;
1951 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1954 return X86EMUL_CONTINUE;
1958 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1960 struct decode_cache *c = &ctxt->decode;
1961 struct desc_struct cs, ss;
1966 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1967 /* inject #GP if in real mode */
1968 if (ctxt->mode == X86EMUL_MODE_REAL)
1969 return emulate_gp(ctxt, 0);
1971 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1972 * Therefore, we inject an #UD.
1974 if (ctxt->mode == X86EMUL_MODE_PROT64)
1975 return emulate_ud(ctxt);
1977 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1979 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1980 switch (ctxt->mode) {
1981 case X86EMUL_MODE_PROT32:
1982 if ((msr_data & 0xfffc) == 0x0)
1983 return emulate_gp(ctxt, 0);
1985 case X86EMUL_MODE_PROT64:
1986 if (msr_data == 0x0)
1987 return emulate_gp(ctxt, 0);
1991 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1992 cs_sel = (u16)msr_data;
1993 cs_sel &= ~SELECTOR_RPL_MASK;
1994 ss_sel = cs_sel + 8;
1995 ss_sel &= ~SELECTOR_RPL_MASK;
1996 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2001 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2002 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2004 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2007 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2008 c->regs[VCPU_REGS_RSP] = msr_data;
2010 return X86EMUL_CONTINUE;
2014 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2016 struct decode_cache *c = &ctxt->decode;
2017 struct desc_struct cs, ss;
2022 /* inject #GP if in real mode or Virtual 8086 mode */
2023 if (ctxt->mode == X86EMUL_MODE_REAL ||
2024 ctxt->mode == X86EMUL_MODE_VM86)
2025 return emulate_gp(ctxt, 0);
2027 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2029 if ((c->rex_prefix & 0x8) != 0x0)
2030 usermode = X86EMUL_MODE_PROT64;
2032 usermode = X86EMUL_MODE_PROT32;
2036 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2038 case X86EMUL_MODE_PROT32:
2039 cs_sel = (u16)(msr_data + 16);
2040 if ((msr_data & 0xfffc) == 0x0)
2041 return emulate_gp(ctxt, 0);
2042 ss_sel = (u16)(msr_data + 24);
2044 case X86EMUL_MODE_PROT64:
2045 cs_sel = (u16)(msr_data + 32);
2046 if (msr_data == 0x0)
2047 return emulate_gp(ctxt, 0);
2048 ss_sel = cs_sel + 8;
2053 cs_sel |= SELECTOR_RPL_MASK;
2054 ss_sel |= SELECTOR_RPL_MASK;
2056 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2057 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2059 c->eip = c->regs[VCPU_REGS_RDX];
2060 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2062 return X86EMUL_CONTINUE;
2065 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2066 struct x86_emulate_ops *ops)
2069 if (ctxt->mode == X86EMUL_MODE_REAL)
2071 if (ctxt->mode == X86EMUL_MODE_VM86)
2073 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2074 return ops->cpl(ctxt) > iopl;
2077 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2078 struct x86_emulate_ops *ops,
2081 struct desc_struct tr_seg;
2084 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2085 unsigned mask = (1 << len) - 1;
2088 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2091 if (desc_limit_scaled(&tr_seg) < 103)
2093 base = get_desc_base(&tr_seg);
2094 #ifdef CONFIG_X86_64
2095 base |= ((u64)base3) << 32;
2097 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2098 if (r != X86EMUL_CONTINUE)
2100 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2102 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2103 if (r != X86EMUL_CONTINUE)
2105 if ((perm >> bit_idx) & mask)
2110 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2111 struct x86_emulate_ops *ops,
2117 if (emulator_bad_iopl(ctxt, ops))
2118 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2121 ctxt->perm_ok = true;
2126 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2127 struct x86_emulate_ops *ops,
2128 struct tss_segment_16 *tss)
2130 struct decode_cache *c = &ctxt->decode;
2133 tss->flag = ctxt->eflags;
2134 tss->ax = c->regs[VCPU_REGS_RAX];
2135 tss->cx = c->regs[VCPU_REGS_RCX];
2136 tss->dx = c->regs[VCPU_REGS_RDX];
2137 tss->bx = c->regs[VCPU_REGS_RBX];
2138 tss->sp = c->regs[VCPU_REGS_RSP];
2139 tss->bp = c->regs[VCPU_REGS_RBP];
2140 tss->si = c->regs[VCPU_REGS_RSI];
2141 tss->di = c->regs[VCPU_REGS_RDI];
2143 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2144 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2145 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2146 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2147 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2150 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2151 struct x86_emulate_ops *ops,
2152 struct tss_segment_16 *tss)
2154 struct decode_cache *c = &ctxt->decode;
2158 ctxt->eflags = tss->flag | 2;
2159 c->regs[VCPU_REGS_RAX] = tss->ax;
2160 c->regs[VCPU_REGS_RCX] = tss->cx;
2161 c->regs[VCPU_REGS_RDX] = tss->dx;
2162 c->regs[VCPU_REGS_RBX] = tss->bx;
2163 c->regs[VCPU_REGS_RSP] = tss->sp;
2164 c->regs[VCPU_REGS_RBP] = tss->bp;
2165 c->regs[VCPU_REGS_RSI] = tss->si;
2166 c->regs[VCPU_REGS_RDI] = tss->di;
2169 * SDM says that segment selectors are loaded before segment
2172 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2173 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2174 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2175 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2176 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2179 * Now load segment descriptors. If fault happenes at this stage
2180 * it is handled in a context of new task
2182 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2183 if (ret != X86EMUL_CONTINUE)
2185 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2186 if (ret != X86EMUL_CONTINUE)
2188 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2189 if (ret != X86EMUL_CONTINUE)
2191 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2192 if (ret != X86EMUL_CONTINUE)
2194 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2195 if (ret != X86EMUL_CONTINUE)
2198 return X86EMUL_CONTINUE;
2201 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2202 struct x86_emulate_ops *ops,
2203 u16 tss_selector, u16 old_tss_sel,
2204 ulong old_tss_base, struct desc_struct *new_desc)
2206 struct tss_segment_16 tss_seg;
2208 u32 new_tss_base = get_desc_base(new_desc);
2210 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2212 if (ret != X86EMUL_CONTINUE)
2213 /* FIXME: need to provide precise fault address */
2216 save_state_to_tss16(ctxt, ops, &tss_seg);
2218 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2220 if (ret != X86EMUL_CONTINUE)
2221 /* FIXME: need to provide precise fault address */
2224 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2226 if (ret != X86EMUL_CONTINUE)
2227 /* FIXME: need to provide precise fault address */
2230 if (old_tss_sel != 0xffff) {
2231 tss_seg.prev_task_link = old_tss_sel;
2233 ret = ops->write_std(ctxt, new_tss_base,
2234 &tss_seg.prev_task_link,
2235 sizeof tss_seg.prev_task_link,
2237 if (ret != X86EMUL_CONTINUE)
2238 /* FIXME: need to provide precise fault address */
2242 return load_state_from_tss16(ctxt, ops, &tss_seg);
2245 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2246 struct x86_emulate_ops *ops,
2247 struct tss_segment_32 *tss)
2249 struct decode_cache *c = &ctxt->decode;
2251 tss->cr3 = ops->get_cr(ctxt, 3);
2253 tss->eflags = ctxt->eflags;
2254 tss->eax = c->regs[VCPU_REGS_RAX];
2255 tss->ecx = c->regs[VCPU_REGS_RCX];
2256 tss->edx = c->regs[VCPU_REGS_RDX];
2257 tss->ebx = c->regs[VCPU_REGS_RBX];
2258 tss->esp = c->regs[VCPU_REGS_RSP];
2259 tss->ebp = c->regs[VCPU_REGS_RBP];
2260 tss->esi = c->regs[VCPU_REGS_RSI];
2261 tss->edi = c->regs[VCPU_REGS_RDI];
2263 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2264 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2265 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2266 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2267 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2268 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2269 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2272 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2273 struct x86_emulate_ops *ops,
2274 struct tss_segment_32 *tss)
2276 struct decode_cache *c = &ctxt->decode;
2279 if (ops->set_cr(ctxt, 3, tss->cr3))
2280 return emulate_gp(ctxt, 0);
2282 ctxt->eflags = tss->eflags | 2;
2283 c->regs[VCPU_REGS_RAX] = tss->eax;
2284 c->regs[VCPU_REGS_RCX] = tss->ecx;
2285 c->regs[VCPU_REGS_RDX] = tss->edx;
2286 c->regs[VCPU_REGS_RBX] = tss->ebx;
2287 c->regs[VCPU_REGS_RSP] = tss->esp;
2288 c->regs[VCPU_REGS_RBP] = tss->ebp;
2289 c->regs[VCPU_REGS_RSI] = tss->esi;
2290 c->regs[VCPU_REGS_RDI] = tss->edi;
2293 * SDM says that segment selectors are loaded before segment
2296 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2297 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2298 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2299 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2300 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2301 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2302 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2305 * Now load segment descriptors. If fault happenes at this stage
2306 * it is handled in a context of new task
2308 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2309 if (ret != X86EMUL_CONTINUE)
2311 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2312 if (ret != X86EMUL_CONTINUE)
2314 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2315 if (ret != X86EMUL_CONTINUE)
2317 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2318 if (ret != X86EMUL_CONTINUE)
2320 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2321 if (ret != X86EMUL_CONTINUE)
2323 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2324 if (ret != X86EMUL_CONTINUE)
2326 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2327 if (ret != X86EMUL_CONTINUE)
2330 return X86EMUL_CONTINUE;
2333 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2334 struct x86_emulate_ops *ops,
2335 u16 tss_selector, u16 old_tss_sel,
2336 ulong old_tss_base, struct desc_struct *new_desc)
2338 struct tss_segment_32 tss_seg;
2340 u32 new_tss_base = get_desc_base(new_desc);
2342 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2344 if (ret != X86EMUL_CONTINUE)
2345 /* FIXME: need to provide precise fault address */
2348 save_state_to_tss32(ctxt, ops, &tss_seg);
2350 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2352 if (ret != X86EMUL_CONTINUE)
2353 /* FIXME: need to provide precise fault address */
2356 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2358 if (ret != X86EMUL_CONTINUE)
2359 /* FIXME: need to provide precise fault address */
2362 if (old_tss_sel != 0xffff) {
2363 tss_seg.prev_task_link = old_tss_sel;
2365 ret = ops->write_std(ctxt, new_tss_base,
2366 &tss_seg.prev_task_link,
2367 sizeof tss_seg.prev_task_link,
2369 if (ret != X86EMUL_CONTINUE)
2370 /* FIXME: need to provide precise fault address */
2374 return load_state_from_tss32(ctxt, ops, &tss_seg);
2377 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2378 struct x86_emulate_ops *ops,
2379 u16 tss_selector, int reason,
2380 bool has_error_code, u32 error_code)
2382 struct desc_struct curr_tss_desc, next_tss_desc;
2384 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2385 ulong old_tss_base =
2386 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2389 /* FIXME: old_tss_base == ~0 ? */
2391 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2392 if (ret != X86EMUL_CONTINUE)
2394 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2395 if (ret != X86EMUL_CONTINUE)
2398 /* FIXME: check that next_tss_desc is tss */
2400 if (reason != TASK_SWITCH_IRET) {
2401 if ((tss_selector & 3) > next_tss_desc.dpl ||
2402 ops->cpl(ctxt) > next_tss_desc.dpl)
2403 return emulate_gp(ctxt, 0);
2406 desc_limit = desc_limit_scaled(&next_tss_desc);
2407 if (!next_tss_desc.p ||
2408 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2409 desc_limit < 0x2b)) {
2410 emulate_ts(ctxt, tss_selector & 0xfffc);
2411 return X86EMUL_PROPAGATE_FAULT;
2414 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2415 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2416 write_segment_descriptor(ctxt, ops, old_tss_sel,
2420 if (reason == TASK_SWITCH_IRET)
2421 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2423 /* set back link to prev task only if NT bit is set in eflags
2424 note that old_tss_sel is not used afetr this point */
2425 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2426 old_tss_sel = 0xffff;
2428 if (next_tss_desc.type & 8)
2429 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2430 old_tss_base, &next_tss_desc);
2432 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2433 old_tss_base, &next_tss_desc);
2434 if (ret != X86EMUL_CONTINUE)
2437 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2438 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2440 if (reason != TASK_SWITCH_IRET) {
2441 next_tss_desc.type |= (1 << 1); /* set busy flag */
2442 write_segment_descriptor(ctxt, ops, tss_selector,
2446 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2447 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2449 if (has_error_code) {
2450 struct decode_cache *c = &ctxt->decode;
2452 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2454 c->src.val = (unsigned long) error_code;
2455 ret = em_push(ctxt);
2461 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2462 u16 tss_selector, int reason,
2463 bool has_error_code, u32 error_code)
2465 struct x86_emulate_ops *ops = ctxt->ops;
2466 struct decode_cache *c = &ctxt->decode;
2470 c->dst.type = OP_NONE;
2472 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2473 has_error_code, error_code);
2475 if (rc == X86EMUL_CONTINUE)
2478 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2481 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2482 int reg, struct operand *op)
2484 struct decode_cache *c = &ctxt->decode;
2485 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2487 register_address_increment(c, &c->regs[reg], df * op->bytes);
2488 op->addr.mem.ea = register_address(c, c->regs[reg]);
2489 op->addr.mem.seg = seg;
2492 static int em_das(struct x86_emulate_ctxt *ctxt)
2494 struct decode_cache *c = &ctxt->decode;
2496 bool af, cf, old_cf;
2498 cf = ctxt->eflags & X86_EFLAGS_CF;
2504 af = ctxt->eflags & X86_EFLAGS_AF;
2505 if ((al & 0x0f) > 9 || af) {
2507 cf = old_cf | (al >= 250);
2512 if (old_al > 0x99 || old_cf) {
2518 /* Set PF, ZF, SF */
2519 c->src.type = OP_IMM;
2522 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2523 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2525 ctxt->eflags |= X86_EFLAGS_CF;
2527 ctxt->eflags |= X86_EFLAGS_AF;
2528 return X86EMUL_CONTINUE;
2531 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2533 struct decode_cache *c = &ctxt->decode;
2538 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2541 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2542 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2543 return X86EMUL_CONTINUE;
2546 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2548 c->src.val = old_cs;
2550 if (rc != X86EMUL_CONTINUE)
2553 c->src.val = old_eip;
2554 return em_push(ctxt);
2557 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2559 struct decode_cache *c = &ctxt->decode;
2562 c->dst.type = OP_REG;
2563 c->dst.addr.reg = &c->eip;
2564 c->dst.bytes = c->op_bytes;
2565 rc = emulate_pop(ctxt, &c->dst.val, c->op_bytes);
2566 if (rc != X86EMUL_CONTINUE)
2568 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2569 return X86EMUL_CONTINUE;
2572 static int em_add(struct x86_emulate_ctxt *ctxt)
2574 struct decode_cache *c = &ctxt->decode;
2576 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2577 return X86EMUL_CONTINUE;
2580 static int em_or(struct x86_emulate_ctxt *ctxt)
2582 struct decode_cache *c = &ctxt->decode;
2584 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2585 return X86EMUL_CONTINUE;
2588 static int em_adc(struct x86_emulate_ctxt *ctxt)
2590 struct decode_cache *c = &ctxt->decode;
2592 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2593 return X86EMUL_CONTINUE;
2596 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2598 struct decode_cache *c = &ctxt->decode;
2600 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2601 return X86EMUL_CONTINUE;
2604 static int em_and(struct x86_emulate_ctxt *ctxt)
2606 struct decode_cache *c = &ctxt->decode;
2608 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2609 return X86EMUL_CONTINUE;
2612 static int em_sub(struct x86_emulate_ctxt *ctxt)
2614 struct decode_cache *c = &ctxt->decode;
2616 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2617 return X86EMUL_CONTINUE;
2620 static int em_xor(struct x86_emulate_ctxt *ctxt)
2622 struct decode_cache *c = &ctxt->decode;
2624 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2625 return X86EMUL_CONTINUE;
2628 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2630 struct decode_cache *c = &ctxt->decode;
2632 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2633 /* Disable writeback. */
2634 c->dst.type = OP_NONE;
2635 return X86EMUL_CONTINUE;
2638 static int em_imul(struct x86_emulate_ctxt *ctxt)
2640 struct decode_cache *c = &ctxt->decode;
2642 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2643 return X86EMUL_CONTINUE;
2646 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2648 struct decode_cache *c = &ctxt->decode;
2650 c->dst.val = c->src2.val;
2651 return em_imul(ctxt);
2654 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2656 struct decode_cache *c = &ctxt->decode;
2658 c->dst.type = OP_REG;
2659 c->dst.bytes = c->src.bytes;
2660 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2661 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2663 return X86EMUL_CONTINUE;
2666 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2668 struct decode_cache *c = &ctxt->decode;
2671 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2672 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2673 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2674 return X86EMUL_CONTINUE;
2677 static int em_mov(struct x86_emulate_ctxt *ctxt)
2679 struct decode_cache *c = &ctxt->decode;
2680 c->dst.val = c->src.val;
2681 return X86EMUL_CONTINUE;
2684 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2686 struct decode_cache *c = &ctxt->decode;
2687 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2688 return X86EMUL_CONTINUE;
2691 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2693 struct decode_cache *c = &ctxt->decode;
2697 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2698 if (rc == X86EMUL_CONTINUE)
2699 ctxt->ops->invlpg(ctxt, linear);
2700 /* Disable writeback. */
2701 c->dst.type = OP_NONE;
2702 return X86EMUL_CONTINUE;
2705 static int em_clts(struct x86_emulate_ctxt *ctxt)
2709 cr0 = ctxt->ops->get_cr(ctxt, 0);
2711 ctxt->ops->set_cr(ctxt, 0, cr0);
2712 return X86EMUL_CONTINUE;
2715 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2717 struct decode_cache *c = &ctxt->decode;
2720 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2721 return X86EMUL_UNHANDLEABLE;
2723 rc = ctxt->ops->fix_hypercall(ctxt);
2724 if (rc != X86EMUL_CONTINUE)
2727 /* Let the processor re-execute the fixed hypercall */
2729 /* Disable writeback. */
2730 c->dst.type = OP_NONE;
2731 return X86EMUL_CONTINUE;
2734 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2736 struct decode_cache *c = &ctxt->decode;
2737 struct desc_ptr desc_ptr;
2740 rc = read_descriptor(ctxt, c->src.addr.mem,
2741 &desc_ptr.size, &desc_ptr.address,
2743 if (rc != X86EMUL_CONTINUE)
2745 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2746 /* Disable writeback. */
2747 c->dst.type = OP_NONE;
2748 return X86EMUL_CONTINUE;
2751 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2753 struct decode_cache *c = &ctxt->decode;
2756 rc = ctxt->ops->fix_hypercall(ctxt);
2758 /* Disable writeback. */
2759 c->dst.type = OP_NONE;
2763 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2765 struct decode_cache *c = &ctxt->decode;
2766 struct desc_ptr desc_ptr;
2769 rc = read_descriptor(ctxt, c->src.addr.mem,
2770 &desc_ptr.size, &desc_ptr.address,
2772 if (rc != X86EMUL_CONTINUE)
2774 ctxt->ops->set_idt(ctxt, &desc_ptr);
2775 /* Disable writeback. */
2776 c->dst.type = OP_NONE;
2777 return X86EMUL_CONTINUE;
2780 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2782 struct decode_cache *c = &ctxt->decode;
2785 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2786 return X86EMUL_CONTINUE;
2789 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2791 struct decode_cache *c = &ctxt->decode;
2792 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2793 | (c->src.val & 0x0f));
2794 c->dst.type = OP_NONE;
2795 return X86EMUL_CONTINUE;
2798 static bool valid_cr(int nr)
2810 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2812 struct decode_cache *c = &ctxt->decode;
2814 if (!valid_cr(c->modrm_reg))
2815 return emulate_ud(ctxt);
2817 return X86EMUL_CONTINUE;
2820 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2822 struct decode_cache *c = &ctxt->decode;
2823 u64 new_val = c->src.val64;
2824 int cr = c->modrm_reg;
2827 static u64 cr_reserved_bits[] = {
2828 0xffffffff00000000ULL,
2829 0, 0, 0, /* CR3 checked later */
2836 return emulate_ud(ctxt);
2838 if (new_val & cr_reserved_bits[cr])
2839 return emulate_gp(ctxt, 0);
2844 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2845 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2846 return emulate_gp(ctxt, 0);
2848 cr4 = ctxt->ops->get_cr(ctxt, 4);
2849 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2851 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2852 !(cr4 & X86_CR4_PAE))
2853 return emulate_gp(ctxt, 0);
2860 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2861 if (efer & EFER_LMA)
2862 rsvd = CR3_L_MODE_RESERVED_BITS;
2863 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2864 rsvd = CR3_PAE_RESERVED_BITS;
2865 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2866 rsvd = CR3_NONPAE_RESERVED_BITS;
2869 return emulate_gp(ctxt, 0);
2876 cr4 = ctxt->ops->get_cr(ctxt, 4);
2877 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2879 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2880 return emulate_gp(ctxt, 0);
2886 return X86EMUL_CONTINUE;
2889 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2893 ctxt->ops->get_dr(ctxt, 7, &dr7);
2895 /* Check if DR7.Global_Enable is set */
2896 return dr7 & (1 << 13);
2899 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2901 struct decode_cache *c = &ctxt->decode;
2902 int dr = c->modrm_reg;
2906 return emulate_ud(ctxt);
2908 cr4 = ctxt->ops->get_cr(ctxt, 4);
2909 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2910 return emulate_ud(ctxt);
2912 if (check_dr7_gd(ctxt))
2913 return emulate_db(ctxt);
2915 return X86EMUL_CONTINUE;
2918 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2920 struct decode_cache *c = &ctxt->decode;
2921 u64 new_val = c->src.val64;
2922 int dr = c->modrm_reg;
2924 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2925 return emulate_gp(ctxt, 0);
2927 return check_dr_read(ctxt);
2930 static int check_svme(struct x86_emulate_ctxt *ctxt)
2934 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2936 if (!(efer & EFER_SVME))
2937 return emulate_ud(ctxt);
2939 return X86EMUL_CONTINUE;
2942 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2944 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2946 /* Valid physical address? */
2947 if (rax & 0xffff000000000000ULL)
2948 return emulate_gp(ctxt, 0);
2950 return check_svme(ctxt);
2953 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2955 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2957 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2958 return emulate_ud(ctxt);
2960 return X86EMUL_CONTINUE;
2963 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2965 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2966 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2968 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2970 return emulate_gp(ctxt, 0);
2972 return X86EMUL_CONTINUE;
2975 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2977 struct decode_cache *c = &ctxt->decode;
2979 c->dst.bytes = min(c->dst.bytes, 4u);
2980 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2981 return emulate_gp(ctxt, 0);
2983 return X86EMUL_CONTINUE;
2986 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2988 struct decode_cache *c = &ctxt->decode;
2990 c->src.bytes = min(c->src.bytes, 4u);
2991 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2992 return emulate_gp(ctxt, 0);
2994 return X86EMUL_CONTINUE;
2997 #define D(_y) { .flags = (_y) }
2998 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2999 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3000 .check_perm = (_p) }
3002 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3003 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3004 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3005 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3006 #define II(_f, _e, _i) \
3007 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3008 #define IIP(_f, _e, _i, _p) \
3009 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3010 .check_perm = (_p) }
3011 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3013 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3014 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3015 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3017 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3018 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3019 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3021 static struct opcode group7_rm1[] = {
3022 DI(SrcNone | ModRM | Priv, monitor),
3023 DI(SrcNone | ModRM | Priv, mwait),
3027 static struct opcode group7_rm3[] = {
3028 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3029 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3030 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3031 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3032 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3033 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3034 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3035 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3038 static struct opcode group7_rm7[] = {
3040 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3044 static struct opcode group1[] = {
3055 static struct opcode group1A[] = {
3056 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3059 static struct opcode group3[] = {
3060 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3061 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3062 X4(D(SrcMem | ModRM)),
3065 static struct opcode group4[] = {
3066 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3070 static struct opcode group5[] = {
3071 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3072 D(SrcMem | ModRM | Stack),
3073 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3074 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3075 D(SrcMem | ModRM | Stack), N,
3078 static struct opcode group6[] = {
3079 DI(ModRM | Prot, sldt),
3080 DI(ModRM | Prot, str),
3081 DI(ModRM | Prot | Priv, lldt),
3082 DI(ModRM | Prot | Priv, ltr),
3086 static struct group_dual group7 = { {
3087 DI(ModRM | Mov | DstMem | Priv, sgdt),
3088 DI(ModRM | Mov | DstMem | Priv, sidt),
3089 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3090 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3091 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3092 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3093 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3095 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3097 N, EXT(0, group7_rm3),
3098 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3099 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3102 static struct opcode group8[] = {
3104 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3105 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3108 static struct group_dual group9 = { {
3109 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3111 N, N, N, N, N, N, N, N,
3114 static struct opcode group11[] = {
3115 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3118 static struct gprefix pfx_0f_6f_0f_7f = {
3119 N, N, N, I(Sse, em_movdqu),
3122 static struct opcode opcode_table[256] = {
3124 I6ALU(Lock, em_add),
3125 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3128 D(ImplicitOps | Stack | No64), N,
3130 I6ALU(Lock, em_adc),
3131 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3133 I6ALU(Lock, em_sbb),
3134 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3136 I6ALU(Lock, em_and), N, N,
3138 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3140 I6ALU(Lock, em_xor), N, N,
3142 I6ALU(0, em_cmp), N, N,
3146 X8(I(SrcReg | Stack, em_push)),
3148 X8(I(DstReg | Stack, em_pop)),
3150 I(ImplicitOps | Stack | No64, em_pusha),
3151 I(ImplicitOps | Stack | No64, em_popa),
3152 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3155 I(SrcImm | Mov | Stack, em_push),
3156 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3157 I(SrcImmByte | Mov | Stack, em_push),
3158 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3159 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3160 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3164 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3165 G(DstMem | SrcImm | ModRM | Group, group1),
3166 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3167 G(DstMem | SrcImmByte | ModRM | Group, group1),
3168 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3170 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3171 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3172 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3173 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3175 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3177 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3178 I(SrcImmFAddr | No64, em_call_far), N,
3179 II(ImplicitOps | Stack, em_pushf, pushf),
3180 II(ImplicitOps | Stack, em_popf, popf), N, N,
3182 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3183 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3184 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3185 I2bv(SrcSI | DstDI | String, em_cmp),
3187 D2bv(DstAcc | SrcImm),
3188 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3189 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3190 I2bv(SrcAcc | DstDI | String, em_cmp),
3192 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3194 X8(I(DstReg | SrcImm | Mov, em_mov)),
3196 D2bv(DstMem | SrcImmByte | ModRM),
3197 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3198 D(ImplicitOps | Stack),
3199 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3200 G(ByteOp, group11), G(0, group11),
3202 N, N, N, D(ImplicitOps | Stack),
3203 D(ImplicitOps), DI(SrcImmByte, intn),
3204 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3206 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3209 N, N, N, N, N, N, N, N,
3212 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3213 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3215 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3216 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3217 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3218 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3220 N, DI(ImplicitOps, icebp), N, N,
3221 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3222 G(ByteOp, group3), G(0, group3),
3224 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3225 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3228 static struct opcode twobyte_table[256] = {
3230 G(0, group6), GD(0, &group7), N, N,
3231 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3232 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3233 N, D(ImplicitOps | ModRM), N, N,
3235 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3237 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3238 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3239 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3240 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3242 N, N, N, N, N, N, N, N,
3244 DI(ImplicitOps | Priv, wrmsr),
3245 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3246 DI(ImplicitOps | Priv, rdmsr),
3247 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3248 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3250 N, N, N, N, N, N, N, N,
3252 X16(D(DstReg | SrcMem | ModRM | Mov)),
3254 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3259 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3264 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3268 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3270 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3271 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3272 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3273 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3275 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3276 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3277 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3278 D(DstMem | SrcReg | Src2CL | ModRM),
3279 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3281 D2bv(DstMem | SrcReg | ModRM | Lock),
3282 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3283 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3284 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3287 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3288 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3289 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3291 D2bv(DstMem | SrcReg | ModRM | Lock),
3292 N, D(DstMem | SrcReg | ModRM | Mov),
3293 N, N, N, GD(0, &group9),
3294 N, N, N, N, N, N, N, N,
3296 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3298 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3300 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3316 static unsigned imm_size(struct decode_cache *c)
3320 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3326 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3327 unsigned size, bool sign_extension)
3329 struct decode_cache *c = &ctxt->decode;
3330 struct x86_emulate_ops *ops = ctxt->ops;
3331 int rc = X86EMUL_CONTINUE;
3335 op->addr.mem.ea = c->eip;
3336 /* NB. Immediates are sign-extended as necessary. */
3337 switch (op->bytes) {
3339 op->val = insn_fetch(s8, 1, c->eip);
3342 op->val = insn_fetch(s16, 2, c->eip);
3345 op->val = insn_fetch(s32, 4, c->eip);
3348 if (!sign_extension) {
3349 switch (op->bytes) {
3357 op->val &= 0xffffffff;
3366 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3368 struct x86_emulate_ops *ops = ctxt->ops;
3369 struct decode_cache *c = &ctxt->decode;
3370 int rc = X86EMUL_CONTINUE;
3371 int mode = ctxt->mode;
3372 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3373 bool op_prefix = false;
3374 struct opcode opcode;
3375 struct operand memop = { .type = OP_NONE };
3378 c->fetch.start = c->eip;
3379 c->fetch.end = c->fetch.start + insn_len;
3381 memcpy(c->fetch.data, insn, insn_len);
3384 case X86EMUL_MODE_REAL:
3385 case X86EMUL_MODE_VM86:
3386 case X86EMUL_MODE_PROT16:
3387 def_op_bytes = def_ad_bytes = 2;
3389 case X86EMUL_MODE_PROT32:
3390 def_op_bytes = def_ad_bytes = 4;
3392 #ifdef CONFIG_X86_64
3393 case X86EMUL_MODE_PROT64:
3402 c->op_bytes = def_op_bytes;
3403 c->ad_bytes = def_ad_bytes;
3405 /* Legacy prefixes. */
3407 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3408 case 0x66: /* operand-size override */
3410 /* switch between 2/4 bytes */
3411 c->op_bytes = def_op_bytes ^ 6;
3413 case 0x67: /* address-size override */
3414 if (mode == X86EMUL_MODE_PROT64)
3415 /* switch between 4/8 bytes */
3416 c->ad_bytes = def_ad_bytes ^ 12;
3418 /* switch between 2/4 bytes */
3419 c->ad_bytes = def_ad_bytes ^ 6;
3421 case 0x26: /* ES override */
3422 case 0x2e: /* CS override */
3423 case 0x36: /* SS override */
3424 case 0x3e: /* DS override */
3425 set_seg_override(c, (c->b >> 3) & 3);
3427 case 0x64: /* FS override */
3428 case 0x65: /* GS override */
3429 set_seg_override(c, c->b & 7);
3431 case 0x40 ... 0x4f: /* REX */
3432 if (mode != X86EMUL_MODE_PROT64)
3434 c->rex_prefix = c->b;
3436 case 0xf0: /* LOCK */
3439 case 0xf2: /* REPNE/REPNZ */
3440 case 0xf3: /* REP/REPE/REPZ */
3441 c->rep_prefix = c->b;
3447 /* Any legacy prefix after a REX prefix nullifies its effect. */
3455 if (c->rex_prefix & 8)
3456 c->op_bytes = 8; /* REX.W */
3458 /* Opcode byte(s). */
3459 opcode = opcode_table[c->b];
3460 /* Two-byte opcode? */
3463 c->b = insn_fetch(u8, 1, c->eip);
3464 opcode = twobyte_table[c->b];
3466 c->d = opcode.flags;
3468 while (c->d & GroupMask) {
3469 switch (c->d & GroupMask) {
3471 c->modrm = insn_fetch(u8, 1, c->eip);
3473 goffset = (c->modrm >> 3) & 7;
3474 opcode = opcode.u.group[goffset];
3477 c->modrm = insn_fetch(u8, 1, c->eip);
3479 goffset = (c->modrm >> 3) & 7;
3480 if ((c->modrm >> 6) == 3)
3481 opcode = opcode.u.gdual->mod3[goffset];
3483 opcode = opcode.u.gdual->mod012[goffset];
3486 goffset = c->modrm & 7;
3487 opcode = opcode.u.group[goffset];
3490 if (c->rep_prefix && op_prefix)
3491 return X86EMUL_UNHANDLEABLE;
3492 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3493 switch (simd_prefix) {
3494 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3495 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3496 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3497 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3501 return X86EMUL_UNHANDLEABLE;
3505 c->d |= opcode.flags;
3508 c->execute = opcode.u.execute;
3509 c->check_perm = opcode.check_perm;
3510 c->intercept = opcode.intercept;
3513 if (c->d == 0 || (c->d & Undefined))
3516 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3519 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3522 if (c->d & Op3264) {
3523 if (mode == X86EMUL_MODE_PROT64)
3532 /* ModRM and SIB bytes. */
3534 rc = decode_modrm(ctxt, ops, &memop);
3535 if (!c->has_seg_override)
3536 set_seg_override(c, c->modrm_seg);
3537 } else if (c->d & MemAbs)
3538 rc = decode_abs(ctxt, ops, &memop);
3539 if (rc != X86EMUL_CONTINUE)
3542 if (!c->has_seg_override)
3543 set_seg_override(c, VCPU_SREG_DS);
3545 memop.addr.mem.seg = seg_override(ctxt, c);
3547 if (memop.type == OP_MEM && c->ad_bytes != 8)
3548 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3550 if (memop.type == OP_MEM && c->rip_relative)
3551 memop.addr.mem.ea += c->eip;
3554 * Decode and fetch the source operand: register, memory
3557 switch (c->d & SrcMask) {
3561 decode_register_operand(ctxt, &c->src, c, 0);
3570 memop.bytes = (c->d & ByteOp) ? 1 :
3576 rc = decode_imm(ctxt, &c->src, 2, false);
3579 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3582 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3585 rc = decode_imm(ctxt, &c->src, 1, true);
3588 rc = decode_imm(ctxt, &c->src, 1, false);
3591 c->src.type = OP_REG;
3592 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3593 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3594 fetch_register_operand(&c->src);
3601 c->src.type = OP_MEM;
3602 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3603 c->src.addr.mem.ea =
3604 register_address(c, c->regs[VCPU_REGS_RSI]);
3605 c->src.addr.mem.seg = seg_override(ctxt, c);
3609 c->src.type = OP_IMM;
3610 c->src.addr.mem.ea = c->eip;
3611 c->src.bytes = c->op_bytes + 2;
3612 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3615 memop.bytes = c->op_bytes + 2;
3619 c->src.type = OP_REG;
3621 c->src.addr.reg = &c->regs[VCPU_REGS_RDX];
3622 fetch_register_operand(&c->src);
3626 if (rc != X86EMUL_CONTINUE)
3630 * Decode and fetch the second source operand: register, memory
3633 switch (c->d & Src2Mask) {
3638 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3641 rc = decode_imm(ctxt, &c->src2, 1, true);
3648 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3652 if (rc != X86EMUL_CONTINUE)
3655 /* Decode and fetch the destination operand: register or memory. */
3656 switch (c->d & DstMask) {
3658 decode_register_operand(ctxt, &c->dst, c,
3659 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3662 c->dst.type = OP_IMM;
3663 c->dst.addr.mem.ea = c->eip;
3665 c->dst.val = insn_fetch(u8, 1, c->eip);
3670 if ((c->d & DstMask) == DstMem64)
3673 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3675 fetch_bit_operand(c);
3676 c->dst.orig_val = c->dst.val;
3679 c->dst.type = OP_REG;
3680 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3681 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3682 fetch_register_operand(&c->dst);
3683 c->dst.orig_val = c->dst.val;
3686 c->dst.type = OP_MEM;
3687 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3688 c->dst.addr.mem.ea =
3689 register_address(c, c->regs[VCPU_REGS_RDI]);
3690 c->dst.addr.mem.seg = VCPU_SREG_ES;
3694 c->dst.type = OP_REG;
3696 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
3697 fetch_register_operand(&c->dst);
3700 /* Special instructions do their own operand decoding. */
3702 c->dst.type = OP_NONE; /* Disable writeback. */
3707 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3710 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3712 struct decode_cache *c = &ctxt->decode;
3714 /* The second termination condition only applies for REPE
3715 * and REPNE. Test if the repeat string operation prefix is
3716 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3717 * corresponding termination condition according to:
3718 * - if REPE/REPZ and ZF = 0 then done
3719 * - if REPNE/REPNZ and ZF = 1 then done
3721 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3722 (c->b == 0xae) || (c->b == 0xaf))
3723 && (((c->rep_prefix == REPE_PREFIX) &&
3724 ((ctxt->eflags & EFLG_ZF) == 0))
3725 || ((c->rep_prefix == REPNE_PREFIX) &&
3726 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3733 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3735 struct x86_emulate_ops *ops = ctxt->ops;
3737 struct decode_cache *c = &ctxt->decode;
3738 int rc = X86EMUL_CONTINUE;
3739 int saved_dst_type = c->dst.type;
3740 int irq; /* Used for int 3, int, and into */
3742 ctxt->decode.mem_read.pos = 0;
3744 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3745 rc = emulate_ud(ctxt);
3749 /* LOCK prefix is allowed only with some instructions */
3750 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3751 rc = emulate_ud(ctxt);
3755 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3756 rc = emulate_ud(ctxt);
3761 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3762 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3763 rc = emulate_ud(ctxt);
3767 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3768 rc = emulate_nm(ctxt);
3772 if (unlikely(ctxt->guest_mode) && c->intercept) {
3773 rc = emulator_check_intercept(ctxt, c->intercept,
3774 X86_ICPT_PRE_EXCEPT);
3775 if (rc != X86EMUL_CONTINUE)
3779 /* Privileged instruction can be executed only in CPL=0 */
3780 if ((c->d & Priv) && ops->cpl(ctxt)) {
3781 rc = emulate_gp(ctxt, 0);
3785 /* Instruction can only be executed in protected mode */
3786 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3787 rc = emulate_ud(ctxt);
3791 /* Do instruction specific permission checks */
3792 if (c->check_perm) {
3793 rc = c->check_perm(ctxt);
3794 if (rc != X86EMUL_CONTINUE)
3798 if (unlikely(ctxt->guest_mode) && c->intercept) {
3799 rc = emulator_check_intercept(ctxt, c->intercept,
3800 X86_ICPT_POST_EXCEPT);
3801 if (rc != X86EMUL_CONTINUE)
3805 if (c->rep_prefix && (c->d & String)) {
3806 /* All REP prefixes have the same first termination condition */
3807 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3813 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3814 rc = segmented_read(ctxt, c->src.addr.mem,
3815 c->src.valptr, c->src.bytes);
3816 if (rc != X86EMUL_CONTINUE)
3818 c->src.orig_val64 = c->src.val64;
3821 if (c->src2.type == OP_MEM) {
3822 rc = segmented_read(ctxt, c->src2.addr.mem,
3823 &c->src2.val, c->src2.bytes);
3824 if (rc != X86EMUL_CONTINUE)
3828 if ((c->d & DstMask) == ImplicitOps)
3832 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3833 /* optimisation - avoid slow emulated read if Mov */
3834 rc = segmented_read(ctxt, c->dst.addr.mem,
3835 &c->dst.val, c->dst.bytes);
3836 if (rc != X86EMUL_CONTINUE)
3839 c->dst.orig_val = c->dst.val;
3843 if (unlikely(ctxt->guest_mode) && c->intercept) {
3844 rc = emulator_check_intercept(ctxt, c->intercept,
3845 X86_ICPT_POST_MEMACCESS);
3846 if (rc != X86EMUL_CONTINUE)
3851 rc = c->execute(ctxt);
3852 if (rc != X86EMUL_CONTINUE)
3861 case 0x06: /* push es */
3862 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3864 case 0x07: /* pop es */
3865 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3867 case 0x0e: /* push cs */
3868 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3870 case 0x16: /* push ss */
3871 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3873 case 0x17: /* pop ss */
3874 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3876 case 0x1e: /* push ds */
3877 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3879 case 0x1f: /* pop ds */
3880 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3882 case 0x40 ... 0x47: /* inc r16/r32 */
3883 emulate_1op("inc", c->dst, ctxt->eflags);
3885 case 0x48 ... 0x4f: /* dec r16/r32 */
3886 emulate_1op("dec", c->dst, ctxt->eflags);
3888 case 0x63: /* movsxd */
3889 if (ctxt->mode != X86EMUL_MODE_PROT64)
3890 goto cannot_emulate;
3891 c->dst.val = (s32) c->src.val;
3893 case 0x6c: /* insb */
3894 case 0x6d: /* insw/insd */
3895 c->src.val = c->regs[VCPU_REGS_RDX];
3897 case 0x6e: /* outsb */
3898 case 0x6f: /* outsw/outsd */
3899 c->dst.val = c->regs[VCPU_REGS_RDX];
3902 case 0x70 ... 0x7f: /* jcc (short) */
3903 if (test_cc(c->b, ctxt->eflags))
3904 jmp_rel(c, c->src.val);
3908 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3910 case 0x86 ... 0x87: /* xchg */
3912 /* Write back the register source. */
3913 c->src.val = c->dst.val;
3914 write_register_operand(&c->src);
3916 * Write back the memory destination with implicit LOCK
3919 c->dst.val = c->src.orig_val;
3922 case 0x8c: /* mov r/m, sreg */
3923 if (c->modrm_reg > VCPU_SREG_GS) {
3924 rc = emulate_ud(ctxt);
3927 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3929 case 0x8d: /* lea r16/r32, m */
3930 c->dst.val = c->src.addr.mem.ea;
3932 case 0x8e: { /* mov seg, r/m16 */
3937 if (c->modrm_reg == VCPU_SREG_CS ||
3938 c->modrm_reg > VCPU_SREG_GS) {
3939 rc = emulate_ud(ctxt);
3943 if (c->modrm_reg == VCPU_SREG_SS)
3944 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3946 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3948 c->dst.type = OP_NONE; /* Disable writeback. */
3951 case 0x8f: /* pop (sole member of Grp1a) */
3952 rc = em_grp1a(ctxt);
3954 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3955 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3958 case 0x98: /* cbw/cwde/cdqe */
3959 switch (c->op_bytes) {
3960 case 2: c->dst.val = (s8)c->dst.val; break;
3961 case 4: c->dst.val = (s16)c->dst.val; break;
3962 case 8: c->dst.val = (s32)c->dst.val; break;
3965 case 0xa8 ... 0xa9: /* test ax, imm */
3970 case 0xc3: /* ret */
3971 c->dst.type = OP_REG;
3972 c->dst.addr.reg = &c->eip;
3973 c->dst.bytes = c->op_bytes;
3976 case 0xc4: /* les */
3977 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3979 case 0xc5: /* lds */
3980 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3982 case 0xcb: /* ret far */
3983 rc = emulate_ret_far(ctxt, ops);
3985 case 0xcc: /* int3 */
3988 case 0xcd: /* int n */
3991 rc = emulate_int(ctxt, ops, irq);
3993 case 0xce: /* into */
3994 if (ctxt->eflags & EFLG_OF) {
3999 case 0xcf: /* iret */
4000 rc = emulate_iret(ctxt, ops);
4002 case 0xd0 ... 0xd1: /* Grp2 */
4005 case 0xd2 ... 0xd3: /* Grp2 */
4006 c->src.val = c->regs[VCPU_REGS_RCX];
4009 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
4010 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4011 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
4012 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
4013 jmp_rel(c, c->src.val);
4015 case 0xe3: /* jcxz/jecxz/jrcxz */
4016 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
4017 jmp_rel(c, c->src.val);
4019 case 0xe4: /* inb */
4022 case 0xe6: /* outb */
4023 case 0xe7: /* out */
4025 case 0xe8: /* call (near) */ {
4026 long int rel = c->src.val;
4027 c->src.val = (unsigned long) c->eip;
4032 case 0xe9: /* jmp rel */
4034 case 0xea: /* jmp far */
4035 rc = em_jmp_far(ctxt);
4038 jmp: /* jmp rel short */
4039 jmp_rel(c, c->src.val);
4040 c->dst.type = OP_NONE; /* Disable writeback. */
4042 case 0xec: /* in al,dx */
4043 case 0xed: /* in (e/r)ax,dx */
4045 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4047 goto done; /* IO is needed */
4049 case 0xee: /* out dx,al */
4050 case 0xef: /* out dx,(e/r)ax */
4052 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4054 c->dst.type = OP_NONE; /* Disable writeback. */
4056 case 0xf4: /* hlt */
4057 ctxt->ops->halt(ctxt);
4059 case 0xf5: /* cmc */
4060 /* complement carry flag from eflags reg */
4061 ctxt->eflags ^= EFLG_CF;
4063 case 0xf6 ... 0xf7: /* Grp3 */
4066 case 0xf8: /* clc */
4067 ctxt->eflags &= ~EFLG_CF;
4069 case 0xf9: /* stc */
4070 ctxt->eflags |= EFLG_CF;
4072 case 0xfa: /* cli */
4073 if (emulator_bad_iopl(ctxt, ops)) {
4074 rc = emulate_gp(ctxt, 0);
4077 ctxt->eflags &= ~X86_EFLAGS_IF;
4079 case 0xfb: /* sti */
4080 if (emulator_bad_iopl(ctxt, ops)) {
4081 rc = emulate_gp(ctxt, 0);
4084 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4085 ctxt->eflags |= X86_EFLAGS_IF;
4088 case 0xfc: /* cld */
4089 ctxt->eflags &= ~EFLG_DF;
4091 case 0xfd: /* std */
4092 ctxt->eflags |= EFLG_DF;
4094 case 0xfe: /* Grp4 */
4095 rc = em_grp45(ctxt);
4097 case 0xff: /* Grp5 */
4098 rc = em_grp45(ctxt);
4101 goto cannot_emulate;
4104 if (rc != X86EMUL_CONTINUE)
4108 rc = writeback(ctxt);
4109 if (rc != X86EMUL_CONTINUE)
4113 * restore dst type in case the decoding will be reused
4114 * (happens for string instruction )
4116 c->dst.type = saved_dst_type;
4118 if ((c->d & SrcMask) == SrcSI)
4119 string_addr_inc(ctxt, seg_override(ctxt, c),
4120 VCPU_REGS_RSI, &c->src);
4122 if ((c->d & DstMask) == DstDI)
4123 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4126 if (c->rep_prefix && (c->d & String)) {
4127 struct read_cache *r = &ctxt->decode.io_read;
4128 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4130 if (!string_insn_completed(ctxt)) {
4132 * Re-enter guest when pio read ahead buffer is empty
4133 * or, if it is not used, after each 1024 iteration.
4135 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4136 (r->end == 0 || r->end != r->pos)) {
4138 * Reset read cache. Usually happens before
4139 * decode, but since instruction is restarted
4140 * we have to do it here.
4142 ctxt->decode.mem_read.end = 0;
4143 return EMULATION_RESTART;
4145 goto done; /* skip rip writeback */
4152 if (rc == X86EMUL_PROPAGATE_FAULT)
4153 ctxt->have_exception = true;
4154 if (rc == X86EMUL_INTERCEPTED)
4155 return EMULATION_INTERCEPTED;
4157 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4161 case 0x05: /* syscall */
4162 rc = emulate_syscall(ctxt, ops);
4167 case 0x09: /* wbinvd */
4168 (ctxt->ops->wbinvd)(ctxt);
4170 case 0x08: /* invd */
4171 case 0x0d: /* GrpP (prefetch) */
4172 case 0x18: /* Grp16 (prefetch/nop) */
4174 case 0x20: /* mov cr, reg */
4175 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4177 case 0x21: /* mov from dr to reg */
4178 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4180 case 0x22: /* mov reg, cr */
4181 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4182 emulate_gp(ctxt, 0);
4183 rc = X86EMUL_PROPAGATE_FAULT;
4186 c->dst.type = OP_NONE;
4188 case 0x23: /* mov from reg to dr */
4189 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4190 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4191 ~0ULL : ~0U)) < 0) {
4192 /* #UD condition is already handled by the code above */
4193 emulate_gp(ctxt, 0);
4194 rc = X86EMUL_PROPAGATE_FAULT;
4198 c->dst.type = OP_NONE; /* no writeback */
4202 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4203 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4204 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4205 emulate_gp(ctxt, 0);
4206 rc = X86EMUL_PROPAGATE_FAULT;
4209 rc = X86EMUL_CONTINUE;
4213 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4214 emulate_gp(ctxt, 0);
4215 rc = X86EMUL_PROPAGATE_FAULT;
4218 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4219 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4221 rc = X86EMUL_CONTINUE;
4223 case 0x34: /* sysenter */
4224 rc = emulate_sysenter(ctxt, ops);
4226 case 0x35: /* sysexit */
4227 rc = emulate_sysexit(ctxt, ops);
4229 case 0x40 ... 0x4f: /* cmov */
4230 c->dst.val = c->dst.orig_val = c->src.val;
4231 if (!test_cc(c->b, ctxt->eflags))
4232 c->dst.type = OP_NONE; /* no writeback */
4234 case 0x80 ... 0x8f: /* jnz rel, etc*/
4235 if (test_cc(c->b, ctxt->eflags))
4236 jmp_rel(c, c->src.val);
4238 case 0x90 ... 0x9f: /* setcc r/m8 */
4239 c->dst.val = test_cc(c->b, ctxt->eflags);
4241 case 0xa0: /* push fs */
4242 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4244 case 0xa1: /* pop fs */
4245 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4249 c->dst.type = OP_NONE;
4250 /* only subword offset */
4251 c->src.val &= (c->dst.bytes << 3) - 1;
4252 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4254 case 0xa4: /* shld imm8, r, r/m */
4255 case 0xa5: /* shld cl, r, r/m */
4256 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4258 case 0xa8: /* push gs */
4259 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4261 case 0xa9: /* pop gs */
4262 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4266 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4268 case 0xac: /* shrd imm8, r, r/m */
4269 case 0xad: /* shrd cl, r, r/m */
4270 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4272 case 0xae: /* clflush */
4274 case 0xb0 ... 0xb1: /* cmpxchg */
4276 * Save real source value, then compare EAX against
4279 c->src.orig_val = c->src.val;
4280 c->src.val = c->regs[VCPU_REGS_RAX];
4281 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4282 if (ctxt->eflags & EFLG_ZF) {
4283 /* Success: write back to memory. */
4284 c->dst.val = c->src.orig_val;
4286 /* Failure: write the value we saw to EAX. */
4287 c->dst.type = OP_REG;
4288 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4291 case 0xb2: /* lss */
4292 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4296 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4298 case 0xb4: /* lfs */
4299 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4301 case 0xb5: /* lgs */
4302 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4304 case 0xb6 ... 0xb7: /* movzx */
4305 c->dst.bytes = c->op_bytes;
4306 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4309 case 0xba: /* Grp8 */
4310 switch (c->modrm_reg & 3) {
4323 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4325 case 0xbc: { /* bsf */
4327 __asm__ ("bsf %2, %0; setz %1"
4328 : "=r"(c->dst.val), "=q"(zf)
4330 ctxt->eflags &= ~X86_EFLAGS_ZF;
4332 ctxt->eflags |= X86_EFLAGS_ZF;
4333 c->dst.type = OP_NONE; /* Disable writeback. */
4337 case 0xbd: { /* bsr */
4339 __asm__ ("bsr %2, %0; setz %1"
4340 : "=r"(c->dst.val), "=q"(zf)
4342 ctxt->eflags &= ~X86_EFLAGS_ZF;
4344 ctxt->eflags |= X86_EFLAGS_ZF;
4345 c->dst.type = OP_NONE; /* Disable writeback. */
4349 case 0xbe ... 0xbf: /* movsx */
4350 c->dst.bytes = c->op_bytes;
4351 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4354 case 0xc0 ... 0xc1: /* xadd */
4355 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4356 /* Write back the register source. */
4357 c->src.val = c->dst.orig_val;
4358 write_register_operand(&c->src);
4360 case 0xc3: /* movnti */
4361 c->dst.bytes = c->op_bytes;
4362 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4365 case 0xc7: /* Grp9 (cmpxchg8b) */
4369 goto cannot_emulate;
4372 if (rc != X86EMUL_CONTINUE)
4378 return EMULATION_FAILED;