1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include <linux/nospec.h>
25 #include "kvm_cache_regs.h"
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
28 #include <asm/debugreg.h>
29 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
164 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
165 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
166 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
167 #define NoWrite ((u64)1 << 45) /* No writeback */
168 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
169 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
170 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
171 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
172 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
173 #define NearBranch ((u64)1 << 52) /* Near branches */
174 #define No16 ((u64)1 << 53) /* No 16 bit operand */
175 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
176 #define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
178 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
180 #define X2(x...) x, x
181 #define X3(x...) X2(x), x
182 #define X4(x...) X2(x), X2(x)
183 #define X5(x...) X4(x), x
184 #define X6(x...) X4(x), X2(x)
185 #define X7(x...) X4(x), X3(x)
186 #define X8(x...) X4(x), X4(x)
187 #define X16(x...) X8(x), X8(x)
189 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
190 #define FASTOP_SIZE 8
193 * fastop functions have a special calling convention:
198 * flags: rflags (in/out)
199 * ex: rsi (in:fastop pointer, out:zero if exception)
201 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
202 * different operand sizes can be reached by calculation, rather than a jump
203 * table (which would be bigger than the code).
205 * fastop functions are declared as taking a never-defined fastop parameter,
206 * so they can't be called from C directly.
215 int (*execute)(struct x86_emulate_ctxt *ctxt);
216 const struct opcode *group;
217 const struct group_dual *gdual;
218 const struct gprefix *gprefix;
219 const struct escape *esc;
220 const struct instr_dual *idual;
221 const struct mode_dual *mdual;
222 void (*fastop)(struct fastop *fake);
224 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
228 struct opcode mod012[8];
229 struct opcode mod3[8];
233 struct opcode pfx_no;
234 struct opcode pfx_66;
235 struct opcode pfx_f2;
236 struct opcode pfx_f3;
241 struct opcode high[64];
245 struct opcode mod012;
250 struct opcode mode32;
251 struct opcode mode64;
254 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
256 enum x86_transfer_type {
258 X86_TRANSFER_CALL_JMP,
260 X86_TRANSFER_TASK_SWITCH,
263 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 if (!(ctxt->regs_valid & (1 << nr))) {
266 ctxt->regs_valid |= 1 << nr;
267 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
269 return ctxt->_regs[nr];
272 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
274 ctxt->regs_valid |= 1 << nr;
275 ctxt->regs_dirty |= 1 << nr;
276 return &ctxt->_regs[nr];
279 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 return reg_write(ctxt, nr);
285 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
289 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
290 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
293 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
295 ctxt->regs_dirty = 0;
296 ctxt->regs_valid = 0;
300 * These EFLAGS bits are restored from saved value during emulation, and
301 * any changes are written back to the saved value after emulation.
303 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
304 X86_EFLAGS_PF|X86_EFLAGS_CF)
312 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
314 #define FOP_FUNC(name) \
315 ".align " __stringify(FASTOP_SIZE) " \n\t" \
316 ".type " name ", @function \n\t" \
319 #define FOP_RET "ret \n\t"
321 #define FOP_START(op) \
322 extern void em_##op(struct fastop *fake); \
323 asm(".pushsection .text, \"ax\" \n\t" \
324 ".global em_" #op " \n\t" \
331 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
334 #define FOP1E(op, dst) \
335 FOP_FUNC(#op "_" #dst) \
336 "10: " #op " %" #dst " \n\t" FOP_RET
338 #define FOP1EEX(op, dst) \
339 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
341 #define FASTOP1(op) \
346 ON64(FOP1E(op##q, rax)) \
349 /* 1-operand, using src2 (for MUL/DIV r/m) */
350 #define FASTOP1SRC2(op, name) \
355 ON64(FOP1E(op, rcx)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
359 #define FASTOP1SRC2EX(op, name) \
364 ON64(FOP1EEX(op, rcx)) \
367 #define FOP2E(op, dst, src) \
368 FOP_FUNC(#op "_" #dst "_" #src) \
369 #op " %" #src ", %" #dst " \n\t" FOP_RET
371 #define FASTOP2(op) \
373 FOP2E(op##b, al, dl) \
374 FOP2E(op##w, ax, dx) \
375 FOP2E(op##l, eax, edx) \
376 ON64(FOP2E(op##q, rax, rdx)) \
379 /* 2 operand, word only */
380 #define FASTOP2W(op) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, src is CL */
389 #define FASTOP2CL(op) \
391 FOP2E(op##b, al, cl) \
392 FOP2E(op##w, ax, cl) \
393 FOP2E(op##l, eax, cl) \
394 ON64(FOP2E(op##q, rax, cl)) \
397 /* 2 operand, src and dest are reversed */
398 #define FASTOP2R(op, name) \
400 FOP2E(op##b, dl, al) \
401 FOP2E(op##w, dx, ax) \
402 FOP2E(op##l, edx, eax) \
403 ON64(FOP2E(op##q, rdx, rax)) \
406 #define FOP3E(op, dst, src, src2) \
407 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
408 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
410 /* 3-operand, word-only, src2=cl */
411 #define FASTOP3WCL(op) \
414 FOP3E(op##w, ax, dx, cl) \
415 FOP3E(op##l, eax, edx, cl) \
416 ON64(FOP3E(op##q, rax, rdx, cl)) \
419 /* Special case for SETcc - 1 instruction per cc */
420 #define FOP_SETCC(op) \
422 ".type " #op ", @function \n\t" \
427 asm(".global kvm_fastop_exception \n"
428 "kvm_fastop_exception: xor %esi, %esi; ret");
449 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
453 * XXX: inoutclob user must know where the argument is being expanded.
454 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
456 #define asm_safe(insn, inoutclob...) \
460 asm volatile("1:" insn "\n" \
462 ".pushsection .fixup, \"ax\"\n" \
463 "3: movl $1, %[_fault]\n" \
466 _ASM_EXTABLE(1b, 3b) \
467 : [_fault] "+qm"(_fault) inoutclob ); \
469 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
472 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
473 enum x86_intercept intercept,
474 enum x86_intercept_stage stage)
476 struct x86_instruction_info info = {
477 .intercept = intercept,
478 .rep_prefix = ctxt->rep_prefix,
479 .modrm_mod = ctxt->modrm_mod,
480 .modrm_reg = ctxt->modrm_reg,
481 .modrm_rm = ctxt->modrm_rm,
482 .src_val = ctxt->src.val64,
483 .dst_val = ctxt->dst.val64,
484 .src_bytes = ctxt->src.bytes,
485 .dst_bytes = ctxt->dst.bytes,
486 .ad_bytes = ctxt->ad_bytes,
487 .next_rip = ctxt->eip,
490 return ctxt->ops->intercept(ctxt, &info, stage);
493 static void assign_masked(ulong *dest, ulong src, ulong mask)
495 *dest = (*dest & ~mask) | (src & mask);
498 static void assign_register(unsigned long *reg, u64 val, int bytes)
500 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
503 *(u8 *)reg = (u8)val;
506 *(u16 *)reg = (u16)val;
510 break; /* 64b: zero-extend */
517 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
519 return (1UL << (ctxt->ad_bytes << 3)) - 1;
522 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
525 struct desc_struct ss;
527 if (ctxt->mode == X86EMUL_MODE_PROT64)
529 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
530 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
533 static int stack_size(struct x86_emulate_ctxt *ctxt)
535 return (__fls(stack_mask(ctxt)) + 1) >> 3;
538 /* Access/update address held in a register, based on addressing mode. */
539 static inline unsigned long
540 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
542 if (ctxt->ad_bytes == sizeof(unsigned long))
545 return reg & ad_mask(ctxt);
548 static inline unsigned long
549 register_address(struct x86_emulate_ctxt *ctxt, int reg)
551 return address_mask(ctxt, reg_read(ctxt, reg));
554 static void masked_increment(ulong *reg, ulong mask, int inc)
556 assign_masked(reg, *reg + inc, mask);
560 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
562 ulong *preg = reg_rmw(ctxt, reg);
564 assign_register(preg, *preg + inc, ctxt->ad_bytes);
567 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
569 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
572 static u32 desc_limit_scaled(struct desc_struct *desc)
574 u32 limit = get_desc_limit(desc);
576 return desc->g ? (limit << 12) | 0xfff : limit;
579 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
581 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
584 return ctxt->ops->get_cached_segment_base(ctxt, seg);
587 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
588 u32 error, bool valid)
591 ctxt->exception.vector = vec;
592 ctxt->exception.error_code = error;
593 ctxt->exception.error_code_valid = valid;
594 return X86EMUL_PROPAGATE_FAULT;
597 static int emulate_db(struct x86_emulate_ctxt *ctxt)
599 return emulate_exception(ctxt, DB_VECTOR, 0, false);
602 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
604 return emulate_exception(ctxt, GP_VECTOR, err, true);
607 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
609 return emulate_exception(ctxt, SS_VECTOR, err, true);
612 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
614 return emulate_exception(ctxt, UD_VECTOR, 0, false);
617 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
619 return emulate_exception(ctxt, TS_VECTOR, err, true);
622 static int emulate_de(struct x86_emulate_ctxt *ctxt)
624 return emulate_exception(ctxt, DE_VECTOR, 0, false);
627 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
629 return emulate_exception(ctxt, NM_VECTOR, 0, false);
632 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
635 struct desc_struct desc;
637 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
641 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
646 struct desc_struct desc;
648 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
649 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
653 * x86 defines three classes of vector instructions: explicitly
654 * aligned, explicitly unaligned, and the rest, which change behaviour
655 * depending on whether they're AVX encoded or not.
657 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
658 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
659 * 512 bytes of data must be aligned to a 16 byte boundary.
661 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
663 if (likely(size < 16))
666 if (ctxt->d & Aligned)
668 else if (ctxt->d & Unaligned)
670 else if (ctxt->d & Avx)
672 else if (ctxt->d & Aligned16)
678 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
679 struct segmented_address addr,
680 unsigned *max_size, unsigned size,
681 bool write, bool fetch,
682 enum x86emul_mode mode, ulong *linear)
684 struct desc_struct desc;
690 la = seg_base(ctxt, addr.seg) + addr.ea;
693 case X86EMUL_MODE_PROT64:
695 if (is_noncanonical_address(la))
698 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
699 if (size > *max_size)
703 *linear = la = (u32)la;
704 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
708 /* code segment in protected mode or read-only data segment */
709 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
710 || !(desc.type & 2)) && write)
712 /* unreadable code segment */
713 if (!fetch && (desc.type & 8) && !(desc.type & 2))
715 lim = desc_limit_scaled(&desc);
716 if (!(desc.type & 8) && (desc.type & 4)) {
717 /* expand-down segment */
720 lim = desc.d ? 0xffffffff : 0xffff;
724 if (lim == 0xffffffff)
727 *max_size = (u64)lim + 1 - addr.ea;
728 if (size > *max_size)
733 if (la & (insn_alignment(ctxt, size) - 1))
734 return emulate_gp(ctxt, 0);
735 return X86EMUL_CONTINUE;
737 if (addr.seg == VCPU_SREG_SS)
738 return emulate_ss(ctxt, 0);
740 return emulate_gp(ctxt, 0);
743 static int linearize(struct x86_emulate_ctxt *ctxt,
744 struct segmented_address addr,
745 unsigned size, bool write,
749 return __linearize(ctxt, addr, &max_size, size, write, false,
753 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
754 enum x86emul_mode mode)
759 struct segmented_address addr = { .seg = VCPU_SREG_CS,
762 if (ctxt->op_bytes != sizeof(unsigned long))
763 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
764 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
765 if (rc == X86EMUL_CONTINUE)
766 ctxt->_eip = addr.ea;
770 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
772 return assign_eip(ctxt, dst, ctxt->mode);
775 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
776 const struct desc_struct *cs_desc)
778 enum x86emul_mode mode = ctxt->mode;
782 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
786 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
788 mode = X86EMUL_MODE_PROT64;
790 mode = X86EMUL_MODE_PROT32; /* temporary value */
793 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
794 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
795 rc = assign_eip(ctxt, dst, mode);
796 if (rc == X86EMUL_CONTINUE)
801 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
803 return assign_eip_near(ctxt, ctxt->_eip + rel);
806 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
807 void *data, unsigned size)
809 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
812 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
813 ulong linear, void *data,
816 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
819 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
820 struct segmented_address addr,
827 rc = linearize(ctxt, addr, size, false, &linear);
828 if (rc != X86EMUL_CONTINUE)
830 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
833 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
834 struct segmented_address addr,
841 rc = linearize(ctxt, addr, size, true, &linear);
842 if (rc != X86EMUL_CONTINUE)
844 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
848 * Prefetch the remaining bytes of the instruction without crossing page
849 * boundary if they are not in fetch_cache yet.
851 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
854 unsigned size, max_size;
855 unsigned long linear;
856 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
857 struct segmented_address addr = { .seg = VCPU_SREG_CS,
858 .ea = ctxt->eip + cur_size };
861 * We do not know exactly how many bytes will be needed, and
862 * __linearize is expensive, so fetch as much as possible. We
863 * just have to avoid going beyond the 15 byte limit, the end
864 * of the segment, or the end of the page.
866 * __linearize is called with size 0 so that it does not do any
867 * boundary check itself. Instead, we use max_size to check
870 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
872 if (unlikely(rc != X86EMUL_CONTINUE))
875 size = min_t(unsigned, 15UL ^ cur_size, max_size);
876 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
879 * One instruction can only straddle two pages,
880 * and one has been loaded at the beginning of
881 * x86_decode_insn. So, if not enough bytes
882 * still, we must have hit the 15-byte boundary.
884 if (unlikely(size < op_size))
885 return emulate_gp(ctxt, 0);
887 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
888 size, &ctxt->exception);
889 if (unlikely(rc != X86EMUL_CONTINUE))
891 ctxt->fetch.end += size;
892 return X86EMUL_CONTINUE;
895 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
898 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
900 if (unlikely(done_size < size))
901 return __do_insn_fetch_bytes(ctxt, size - done_size);
903 return X86EMUL_CONTINUE;
906 /* Fetch next part of the instruction being emulated. */
907 #define insn_fetch(_type, _ctxt) \
910 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
911 if (rc != X86EMUL_CONTINUE) \
913 ctxt->_eip += sizeof(_type); \
914 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
915 ctxt->fetch.ptr += sizeof(_type); \
919 #define insn_fetch_arr(_arr, _size, _ctxt) \
921 rc = do_insn_fetch_bytes(_ctxt, _size); \
922 if (rc != X86EMUL_CONTINUE) \
924 ctxt->_eip += (_size); \
925 memcpy(_arr, ctxt->fetch.ptr, _size); \
926 ctxt->fetch.ptr += (_size); \
930 * Given the 'reg' portion of a ModRM byte, and a register block, return a
931 * pointer into the block that addresses the relevant register.
932 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
934 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
938 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
940 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
941 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
943 p = reg_rmw(ctxt, modrm_reg);
947 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
948 struct segmented_address addr,
949 u16 *size, unsigned long *address, int op_bytes)
956 rc = segmented_read_std(ctxt, addr, size, 2);
957 if (rc != X86EMUL_CONTINUE)
960 rc = segmented_read_std(ctxt, addr, address, op_bytes);
974 FASTOP1SRC2(mul, mul_ex);
975 FASTOP1SRC2(imul, imul_ex);
976 FASTOP1SRC2EX(div, div_ex);
977 FASTOP1SRC2EX(idiv, idiv_ex);
1006 FASTOP2R(cmp, cmp_r);
1008 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1010 /* If src is zero, do not writeback, but update flags */
1011 if (ctxt->src.val == 0)
1012 ctxt->dst.type = OP_NONE;
1013 return fastop(ctxt, em_bsf);
1016 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1018 /* If src is zero, do not writeback, but update flags */
1019 if (ctxt->src.val == 0)
1020 ctxt->dst.type = OP_NONE;
1021 return fastop(ctxt, em_bsr);
1024 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1027 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1029 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1030 asm("push %[flags]; popf; " CALL_NOSPEC
1031 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1035 static void fetch_register_operand(struct operand *op)
1037 switch (op->bytes) {
1039 op->val = *(u8 *)op->addr.reg;
1042 op->val = *(u16 *)op->addr.reg;
1045 op->val = *(u32 *)op->addr.reg;
1048 op->val = *(u64 *)op->addr.reg;
1053 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1055 ctxt->ops->get_fpu(ctxt);
1057 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1058 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1059 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1060 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1061 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1062 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1063 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1064 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1065 #ifdef CONFIG_X86_64
1066 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1067 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1068 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1069 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1070 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1071 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1072 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1073 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1077 ctxt->ops->put_fpu(ctxt);
1080 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1083 ctxt->ops->get_fpu(ctxt);
1085 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1086 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1087 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1088 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1089 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1090 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1091 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1092 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1093 #ifdef CONFIG_X86_64
1094 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1095 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1096 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1097 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1098 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1099 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1100 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1101 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1105 ctxt->ops->put_fpu(ctxt);
1108 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1110 ctxt->ops->get_fpu(ctxt);
1112 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1113 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1114 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1115 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1116 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1117 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1118 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1119 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1122 ctxt->ops->put_fpu(ctxt);
1125 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1127 ctxt->ops->get_fpu(ctxt);
1129 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1130 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1131 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1132 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1133 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1134 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1135 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1136 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1139 ctxt->ops->put_fpu(ctxt);
1142 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1144 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1145 return emulate_nm(ctxt);
1147 ctxt->ops->get_fpu(ctxt);
1148 asm volatile("fninit");
1149 ctxt->ops->put_fpu(ctxt);
1150 return X86EMUL_CONTINUE;
1153 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1157 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1158 return emulate_nm(ctxt);
1160 ctxt->ops->get_fpu(ctxt);
1161 asm volatile("fnstcw %0": "+m"(fcw));
1162 ctxt->ops->put_fpu(ctxt);
1164 ctxt->dst.val = fcw;
1166 return X86EMUL_CONTINUE;
1169 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1173 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1174 return emulate_nm(ctxt);
1176 ctxt->ops->get_fpu(ctxt);
1177 asm volatile("fnstsw %0": "+m"(fsw));
1178 ctxt->ops->put_fpu(ctxt);
1180 ctxt->dst.val = fsw;
1182 return X86EMUL_CONTINUE;
1185 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1188 unsigned reg = ctxt->modrm_reg;
1190 if (!(ctxt->d & ModRM))
1191 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1193 if (ctxt->d & Sse) {
1197 read_sse_reg(ctxt, &op->vec_val, reg);
1200 if (ctxt->d & Mmx) {
1209 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1210 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1212 fetch_register_operand(op);
1213 op->orig_val = op->val;
1216 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1218 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1219 ctxt->modrm_seg = VCPU_SREG_SS;
1222 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1226 int index_reg, base_reg, scale;
1227 int rc = X86EMUL_CONTINUE;
1230 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1231 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1232 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1234 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1235 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1236 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1237 ctxt->modrm_seg = VCPU_SREG_DS;
1239 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1241 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1242 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1244 if (ctxt->d & Sse) {
1247 op->addr.xmm = ctxt->modrm_rm;
1248 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1251 if (ctxt->d & Mmx) {
1254 op->addr.mm = ctxt->modrm_rm & 7;
1257 fetch_register_operand(op);
1263 if (ctxt->ad_bytes == 2) {
1264 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1265 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1266 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1267 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1269 /* 16-bit ModR/M decode. */
1270 switch (ctxt->modrm_mod) {
1272 if (ctxt->modrm_rm == 6)
1273 modrm_ea += insn_fetch(u16, ctxt);
1276 modrm_ea += insn_fetch(s8, ctxt);
1279 modrm_ea += insn_fetch(u16, ctxt);
1282 switch (ctxt->modrm_rm) {
1284 modrm_ea += bx + si;
1287 modrm_ea += bx + di;
1290 modrm_ea += bp + si;
1293 modrm_ea += bp + di;
1302 if (ctxt->modrm_mod != 0)
1309 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1310 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1311 ctxt->modrm_seg = VCPU_SREG_SS;
1312 modrm_ea = (u16)modrm_ea;
1314 /* 32/64-bit ModR/M decode. */
1315 if ((ctxt->modrm_rm & 7) == 4) {
1316 sib = insn_fetch(u8, ctxt);
1317 index_reg |= (sib >> 3) & 7;
1318 base_reg |= sib & 7;
1321 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1322 modrm_ea += insn_fetch(s32, ctxt);
1324 modrm_ea += reg_read(ctxt, base_reg);
1325 adjust_modrm_seg(ctxt, base_reg);
1326 /* Increment ESP on POP [ESP] */
1327 if ((ctxt->d & IncSP) &&
1328 base_reg == VCPU_REGS_RSP)
1329 modrm_ea += ctxt->op_bytes;
1332 modrm_ea += reg_read(ctxt, index_reg) << scale;
1333 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1334 modrm_ea += insn_fetch(s32, ctxt);
1335 if (ctxt->mode == X86EMUL_MODE_PROT64)
1336 ctxt->rip_relative = 1;
1338 base_reg = ctxt->modrm_rm;
1339 modrm_ea += reg_read(ctxt, base_reg);
1340 adjust_modrm_seg(ctxt, base_reg);
1342 switch (ctxt->modrm_mod) {
1344 modrm_ea += insn_fetch(s8, ctxt);
1347 modrm_ea += insn_fetch(s32, ctxt);
1351 op->addr.mem.ea = modrm_ea;
1352 if (ctxt->ad_bytes != 8)
1353 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1359 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1362 int rc = X86EMUL_CONTINUE;
1365 switch (ctxt->ad_bytes) {
1367 op->addr.mem.ea = insn_fetch(u16, ctxt);
1370 op->addr.mem.ea = insn_fetch(u32, ctxt);
1373 op->addr.mem.ea = insn_fetch(u64, ctxt);
1380 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1384 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1385 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1387 if (ctxt->src.bytes == 2)
1388 sv = (s16)ctxt->src.val & (s16)mask;
1389 else if (ctxt->src.bytes == 4)
1390 sv = (s32)ctxt->src.val & (s32)mask;
1392 sv = (s64)ctxt->src.val & (s64)mask;
1394 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1395 ctxt->dst.addr.mem.ea + (sv >> 3));
1398 /* only subword offset */
1399 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1402 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1403 unsigned long addr, void *dest, unsigned size)
1406 struct read_cache *mc = &ctxt->mem_read;
1408 if (mc->pos < mc->end)
1411 WARN_ON((mc->end + size) >= sizeof(mc->data));
1413 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1415 if (rc != X86EMUL_CONTINUE)
1421 memcpy(dest, mc->data + mc->pos, size);
1423 return X86EMUL_CONTINUE;
1426 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1427 struct segmented_address addr,
1434 rc = linearize(ctxt, addr, size, false, &linear);
1435 if (rc != X86EMUL_CONTINUE)
1437 return read_emulated(ctxt, linear, data, size);
1440 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1441 struct segmented_address addr,
1448 rc = linearize(ctxt, addr, size, true, &linear);
1449 if (rc != X86EMUL_CONTINUE)
1451 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1455 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1456 struct segmented_address addr,
1457 const void *orig_data, const void *data,
1463 rc = linearize(ctxt, addr, size, true, &linear);
1464 if (rc != X86EMUL_CONTINUE)
1466 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1467 size, &ctxt->exception);
1470 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1471 unsigned int size, unsigned short port,
1474 struct read_cache *rc = &ctxt->io_read;
1476 if (rc->pos == rc->end) { /* refill pio read ahead */
1477 unsigned int in_page, n;
1478 unsigned int count = ctxt->rep_prefix ?
1479 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1480 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1481 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1482 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1483 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1486 rc->pos = rc->end = 0;
1487 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1492 if (ctxt->rep_prefix && (ctxt->d & String) &&
1493 !(ctxt->eflags & X86_EFLAGS_DF)) {
1494 ctxt->dst.data = rc->data + rc->pos;
1495 ctxt->dst.type = OP_MEM_STR;
1496 ctxt->dst.count = (rc->end - rc->pos) / size;
1499 memcpy(dest, rc->data + rc->pos, size);
1505 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1506 u16 index, struct desc_struct *desc)
1511 ctxt->ops->get_idt(ctxt, &dt);
1513 if (dt.size < index * 8 + 7)
1514 return emulate_gp(ctxt, index << 3 | 0x2);
1516 addr = dt.address + index * 8;
1517 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1520 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1521 u16 selector, struct desc_ptr *dt)
1523 const struct x86_emulate_ops *ops = ctxt->ops;
1526 if (selector & 1 << 2) {
1527 struct desc_struct desc;
1530 memset (dt, 0, sizeof *dt);
1531 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1535 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1536 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1538 ops->get_gdt(ctxt, dt);
1541 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1542 u16 selector, ulong *desc_addr_p)
1545 u16 index = selector >> 3;
1548 get_descriptor_table_ptr(ctxt, selector, &dt);
1550 if (dt.size < index * 8 + 7)
1551 return emulate_gp(ctxt, selector & 0xfffc);
1553 addr = dt.address + index * 8;
1555 #ifdef CONFIG_X86_64
1556 if (addr >> 32 != 0) {
1559 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1560 if (!(efer & EFER_LMA))
1565 *desc_addr_p = addr;
1566 return X86EMUL_CONTINUE;
1569 /* allowed just for 8 bytes segments */
1570 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1571 u16 selector, struct desc_struct *desc,
1576 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1577 if (rc != X86EMUL_CONTINUE)
1580 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1583 /* allowed just for 8 bytes segments */
1584 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1585 u16 selector, struct desc_struct *desc)
1590 rc = get_descriptor_ptr(ctxt, selector, &addr);
1591 if (rc != X86EMUL_CONTINUE)
1594 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1597 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1598 u16 selector, int seg, u8 cpl,
1599 enum x86_transfer_type transfer,
1600 struct desc_struct *desc)
1602 struct desc_struct seg_desc, old_desc;
1604 unsigned err_vec = GP_VECTOR;
1606 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1612 memset(&seg_desc, 0, sizeof seg_desc);
1614 if (ctxt->mode == X86EMUL_MODE_REAL) {
1615 /* set real mode segment descriptor (keep limit etc. for
1617 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1618 set_desc_base(&seg_desc, selector << 4);
1620 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1621 /* VM86 needs a clean new segment descriptor */
1622 set_desc_base(&seg_desc, selector << 4);
1623 set_desc_limit(&seg_desc, 0xffff);
1633 /* TR should be in GDT only */
1634 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1637 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1638 if (null_selector) {
1639 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1642 if (seg == VCPU_SREG_SS) {
1643 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1647 * ctxt->ops->set_segment expects the CPL to be in
1648 * SS.DPL, so fake an expand-up 32-bit data segment.
1658 /* Skip all following checks */
1662 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1663 if (ret != X86EMUL_CONTINUE)
1666 err_code = selector & 0xfffc;
1667 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1670 /* can't load system descriptor into segment selector */
1671 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1672 if (transfer == X86_TRANSFER_CALL_JMP)
1673 return X86EMUL_UNHANDLEABLE;
1682 * segment is not a writable data segment or segment
1683 * selector's RPL != CPL or segment selector's RPL != CPL
1685 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1689 if (!(seg_desc.type & 8))
1692 if (seg_desc.type & 4) {
1698 if (rpl > cpl || dpl != cpl)
1701 /* in long-mode d/b must be clear if l is set */
1702 if (seg_desc.d && seg_desc.l) {
1705 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1706 if (efer & EFER_LMA)
1710 /* CS(RPL) <- CPL */
1711 selector = (selector & 0xfffc) | cpl;
1714 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1717 err_vec = NP_VECTOR;
1720 old_desc = seg_desc;
1721 seg_desc.type |= 2; /* busy */
1722 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1723 sizeof(seg_desc), &ctxt->exception);
1724 if (ret != X86EMUL_CONTINUE)
1727 case VCPU_SREG_LDTR:
1728 if (seg_desc.s || seg_desc.type != 2)
1731 default: /* DS, ES, FS, or GS */
1733 * segment is not a data or readable code segment or
1734 * ((segment is a data or nonconforming code segment)
1735 * and (both RPL and CPL > DPL))
1737 if ((seg_desc.type & 0xa) == 0x8 ||
1738 (((seg_desc.type & 0xc) != 0xc) &&
1739 (rpl > dpl && cpl > dpl)))
1745 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1750 /* mark segment as accessed */
1751 if (!(seg_desc.type & 1)) {
1753 ret = write_segment_descriptor(ctxt, selector,
1755 if (ret != X86EMUL_CONTINUE)
1758 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1759 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1760 if (ret != X86EMUL_CONTINUE)
1762 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1763 ((u64)base3 << 32)))
1764 return emulate_gp(ctxt, 0);
1767 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1770 return X86EMUL_CONTINUE;
1772 return emulate_exception(ctxt, err_vec, err_code, true);
1775 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1776 u16 selector, int seg)
1778 u8 cpl = ctxt->ops->cpl(ctxt);
1781 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1782 * they can load it at CPL<3 (Intel's manual says only LSS can,
1785 * However, the Intel manual says that putting IST=1/DPL=3 in
1786 * an interrupt gate will result in SS=3 (the AMD manual instead
1787 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1788 * and only forbid it here.
1790 if (seg == VCPU_SREG_SS && selector == 3 &&
1791 ctxt->mode == X86EMUL_MODE_PROT64)
1792 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1794 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1795 X86_TRANSFER_NONE, NULL);
1798 static void write_register_operand(struct operand *op)
1800 return assign_register(op->addr.reg, op->val, op->bytes);
1803 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1807 write_register_operand(op);
1810 if (ctxt->lock_prefix)
1811 return segmented_cmpxchg(ctxt,
1817 return segmented_write(ctxt,
1823 return segmented_write(ctxt,
1826 op->bytes * op->count);
1829 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1832 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1840 return X86EMUL_CONTINUE;
1843 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1845 struct segmented_address addr;
1847 rsp_increment(ctxt, -bytes);
1848 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1849 addr.seg = VCPU_SREG_SS;
1851 return segmented_write(ctxt, addr, data, bytes);
1854 static int em_push(struct x86_emulate_ctxt *ctxt)
1856 /* Disable writeback. */
1857 ctxt->dst.type = OP_NONE;
1858 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1861 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1862 void *dest, int len)
1865 struct segmented_address addr;
1867 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1868 addr.seg = VCPU_SREG_SS;
1869 rc = segmented_read(ctxt, addr, dest, len);
1870 if (rc != X86EMUL_CONTINUE)
1873 rsp_increment(ctxt, len);
1877 static int em_pop(struct x86_emulate_ctxt *ctxt)
1879 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1882 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1883 void *dest, int len)
1886 unsigned long val, change_mask;
1887 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1888 int cpl = ctxt->ops->cpl(ctxt);
1890 rc = emulate_pop(ctxt, &val, len);
1891 if (rc != X86EMUL_CONTINUE)
1894 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1895 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1896 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1897 X86_EFLAGS_AC | X86_EFLAGS_ID;
1899 switch(ctxt->mode) {
1900 case X86EMUL_MODE_PROT64:
1901 case X86EMUL_MODE_PROT32:
1902 case X86EMUL_MODE_PROT16:
1904 change_mask |= X86_EFLAGS_IOPL;
1906 change_mask |= X86_EFLAGS_IF;
1908 case X86EMUL_MODE_VM86:
1910 return emulate_gp(ctxt, 0);
1911 change_mask |= X86_EFLAGS_IF;
1913 default: /* real mode */
1914 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1918 *(unsigned long *)dest =
1919 (ctxt->eflags & ~change_mask) | (val & change_mask);
1924 static int em_popf(struct x86_emulate_ctxt *ctxt)
1926 ctxt->dst.type = OP_REG;
1927 ctxt->dst.addr.reg = &ctxt->eflags;
1928 ctxt->dst.bytes = ctxt->op_bytes;
1929 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1932 static int em_enter(struct x86_emulate_ctxt *ctxt)
1935 unsigned frame_size = ctxt->src.val;
1936 unsigned nesting_level = ctxt->src2.val & 31;
1940 return X86EMUL_UNHANDLEABLE;
1942 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1943 rc = push(ctxt, &rbp, stack_size(ctxt));
1944 if (rc != X86EMUL_CONTINUE)
1946 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1948 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1949 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1951 return X86EMUL_CONTINUE;
1954 static int em_leave(struct x86_emulate_ctxt *ctxt)
1956 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1958 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1961 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1963 int seg = ctxt->src2.val;
1965 ctxt->src.val = get_segment_selector(ctxt, seg);
1966 if (ctxt->op_bytes == 4) {
1967 rsp_increment(ctxt, -2);
1971 return em_push(ctxt);
1974 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1976 int seg = ctxt->src2.val;
1977 unsigned long selector;
1980 rc = emulate_pop(ctxt, &selector, 2);
1981 if (rc != X86EMUL_CONTINUE)
1984 if (ctxt->modrm_reg == VCPU_SREG_SS)
1985 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1986 if (ctxt->op_bytes > 2)
1987 rsp_increment(ctxt, ctxt->op_bytes - 2);
1989 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1993 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1995 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1996 int rc = X86EMUL_CONTINUE;
1997 int reg = VCPU_REGS_RAX;
1999 while (reg <= VCPU_REGS_RDI) {
2000 (reg == VCPU_REGS_RSP) ?
2001 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2004 if (rc != X86EMUL_CONTINUE)
2013 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2015 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2016 return em_push(ctxt);
2019 static int em_popa(struct x86_emulate_ctxt *ctxt)
2021 int rc = X86EMUL_CONTINUE;
2022 int reg = VCPU_REGS_RDI;
2025 while (reg >= VCPU_REGS_RAX) {
2026 if (reg == VCPU_REGS_RSP) {
2027 rsp_increment(ctxt, ctxt->op_bytes);
2031 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2032 if (rc != X86EMUL_CONTINUE)
2034 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2040 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2042 const struct x86_emulate_ops *ops = ctxt->ops;
2049 /* TODO: Add limit checks */
2050 ctxt->src.val = ctxt->eflags;
2052 if (rc != X86EMUL_CONTINUE)
2055 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2057 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2059 if (rc != X86EMUL_CONTINUE)
2062 ctxt->src.val = ctxt->_eip;
2064 if (rc != X86EMUL_CONTINUE)
2067 ops->get_idt(ctxt, &dt);
2069 eip_addr = dt.address + (irq << 2);
2070 cs_addr = dt.address + (irq << 2) + 2;
2072 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2073 if (rc != X86EMUL_CONTINUE)
2076 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2077 if (rc != X86EMUL_CONTINUE)
2080 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2081 if (rc != X86EMUL_CONTINUE)
2089 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2093 invalidate_registers(ctxt);
2094 rc = __emulate_int_real(ctxt, irq);
2095 if (rc == X86EMUL_CONTINUE)
2096 writeback_registers(ctxt);
2100 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2102 switch(ctxt->mode) {
2103 case X86EMUL_MODE_REAL:
2104 return __emulate_int_real(ctxt, irq);
2105 case X86EMUL_MODE_VM86:
2106 case X86EMUL_MODE_PROT16:
2107 case X86EMUL_MODE_PROT32:
2108 case X86EMUL_MODE_PROT64:
2110 /* Protected mode interrupts unimplemented yet */
2111 return X86EMUL_UNHANDLEABLE;
2115 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2117 int rc = X86EMUL_CONTINUE;
2118 unsigned long temp_eip = 0;
2119 unsigned long temp_eflags = 0;
2120 unsigned long cs = 0;
2121 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2122 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2123 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2124 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2125 X86_EFLAGS_AC | X86_EFLAGS_ID |
2127 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2130 /* TODO: Add stack limit check */
2132 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2134 if (rc != X86EMUL_CONTINUE)
2137 if (temp_eip & ~0xffff)
2138 return emulate_gp(ctxt, 0);
2140 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2142 if (rc != X86EMUL_CONTINUE)
2145 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2147 if (rc != X86EMUL_CONTINUE)
2150 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2152 if (rc != X86EMUL_CONTINUE)
2155 ctxt->_eip = temp_eip;
2157 if (ctxt->op_bytes == 4)
2158 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2159 else if (ctxt->op_bytes == 2) {
2160 ctxt->eflags &= ~0xffff;
2161 ctxt->eflags |= temp_eflags;
2164 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2165 ctxt->eflags |= X86_EFLAGS_FIXED;
2166 ctxt->ops->set_nmi_mask(ctxt, false);
2171 static int em_iret(struct x86_emulate_ctxt *ctxt)
2173 switch(ctxt->mode) {
2174 case X86EMUL_MODE_REAL:
2175 return emulate_iret_real(ctxt);
2176 case X86EMUL_MODE_VM86:
2177 case X86EMUL_MODE_PROT16:
2178 case X86EMUL_MODE_PROT32:
2179 case X86EMUL_MODE_PROT64:
2181 /* iret from protected mode unimplemented yet */
2182 return X86EMUL_UNHANDLEABLE;
2186 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2190 struct desc_struct new_desc;
2191 u8 cpl = ctxt->ops->cpl(ctxt);
2193 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2195 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2196 X86_TRANSFER_CALL_JMP,
2198 if (rc != X86EMUL_CONTINUE)
2201 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2202 /* Error handling is not implemented. */
2203 if (rc != X86EMUL_CONTINUE)
2204 return X86EMUL_UNHANDLEABLE;
2209 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2211 return assign_eip_near(ctxt, ctxt->src.val);
2214 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2219 old_eip = ctxt->_eip;
2220 rc = assign_eip_near(ctxt, ctxt->src.val);
2221 if (rc != X86EMUL_CONTINUE)
2223 ctxt->src.val = old_eip;
2228 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2230 u64 old = ctxt->dst.orig_val64;
2232 if (ctxt->dst.bytes == 16)
2233 return X86EMUL_UNHANDLEABLE;
2235 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2236 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2237 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2238 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2239 ctxt->eflags &= ~X86_EFLAGS_ZF;
2241 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2242 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2244 ctxt->eflags |= X86_EFLAGS_ZF;
2246 return X86EMUL_CONTINUE;
2249 static int em_ret(struct x86_emulate_ctxt *ctxt)
2254 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2255 if (rc != X86EMUL_CONTINUE)
2258 return assign_eip_near(ctxt, eip);
2261 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2264 unsigned long eip, cs;
2265 int cpl = ctxt->ops->cpl(ctxt);
2266 struct desc_struct new_desc;
2268 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2269 if (rc != X86EMUL_CONTINUE)
2271 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2272 if (rc != X86EMUL_CONTINUE)
2274 /* Outer-privilege level return is not implemented */
2275 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2276 return X86EMUL_UNHANDLEABLE;
2277 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2280 if (rc != X86EMUL_CONTINUE)
2282 rc = assign_eip_far(ctxt, eip, &new_desc);
2283 /* Error handling is not implemented. */
2284 if (rc != X86EMUL_CONTINUE)
2285 return X86EMUL_UNHANDLEABLE;
2290 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2294 rc = em_ret_far(ctxt);
2295 if (rc != X86EMUL_CONTINUE)
2297 rsp_increment(ctxt, ctxt->src.val);
2298 return X86EMUL_CONTINUE;
2301 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2303 /* Save real source value, then compare EAX against destination. */
2304 ctxt->dst.orig_val = ctxt->dst.val;
2305 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2306 ctxt->src.orig_val = ctxt->src.val;
2307 ctxt->src.val = ctxt->dst.orig_val;
2308 fastop(ctxt, em_cmp);
2310 if (ctxt->eflags & X86_EFLAGS_ZF) {
2311 /* Success: write back to memory; no update of EAX */
2312 ctxt->src.type = OP_NONE;
2313 ctxt->dst.val = ctxt->src.orig_val;
2315 /* Failure: write the value we saw to EAX. */
2316 ctxt->src.type = OP_REG;
2317 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2318 ctxt->src.val = ctxt->dst.orig_val;
2319 /* Create write-cycle to dest by writing the same value */
2320 ctxt->dst.val = ctxt->dst.orig_val;
2322 return X86EMUL_CONTINUE;
2325 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2327 int seg = ctxt->src2.val;
2331 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2333 rc = load_segment_descriptor(ctxt, sel, seg);
2334 if (rc != X86EMUL_CONTINUE)
2337 ctxt->dst.val = ctxt->src.val;
2341 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2343 u32 eax, ebx, ecx, edx;
2347 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2348 return edx & bit(X86_FEATURE_LM);
2351 #define GET_SMSTATE(type, smbase, offset) \
2354 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2356 if (r != X86EMUL_CONTINUE) \
2357 return X86EMUL_UNHANDLEABLE; \
2361 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2363 desc->g = (flags >> 23) & 1;
2364 desc->d = (flags >> 22) & 1;
2365 desc->l = (flags >> 21) & 1;
2366 desc->avl = (flags >> 20) & 1;
2367 desc->p = (flags >> 15) & 1;
2368 desc->dpl = (flags >> 13) & 3;
2369 desc->s = (flags >> 12) & 1;
2370 desc->type = (flags >> 8) & 15;
2373 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2375 struct desc_struct desc;
2379 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2382 offset = 0x7f84 + n * 12;
2384 offset = 0x7f2c + (n - 3) * 12;
2386 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2387 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2388 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2389 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2390 return X86EMUL_CONTINUE;
2393 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2395 struct desc_struct desc;
2400 offset = 0x7e00 + n * 16;
2402 selector = GET_SMSTATE(u16, smbase, offset);
2403 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2404 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2405 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2406 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2408 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2409 return X86EMUL_CONTINUE;
2412 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2413 u64 cr0, u64 cr3, u64 cr4)
2418 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2420 if (cr4 & X86_CR4_PCIDE) {
2425 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2427 return X86EMUL_UNHANDLEABLE;
2430 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2431 * Then enable protected mode. However, PCID cannot be enabled
2432 * if EFER.LMA=0, so set it separately.
2434 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2436 return X86EMUL_UNHANDLEABLE;
2438 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2440 return X86EMUL_UNHANDLEABLE;
2442 if (cr4 & X86_CR4_PCIDE) {
2443 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2445 return X86EMUL_UNHANDLEABLE;
2447 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2449 return X86EMUL_UNHANDLEABLE;
2454 return X86EMUL_CONTINUE;
2457 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2459 struct desc_struct desc;
2462 u32 val, cr0, cr3, cr4;
2465 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2466 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2467 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2468 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2470 for (i = 0; i < 8; i++)
2471 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2473 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2474 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2475 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2476 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2478 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2479 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2480 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2481 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2482 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2484 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2485 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2486 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2487 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2488 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2490 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2491 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2492 ctxt->ops->set_gdt(ctxt, &dt);
2494 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2495 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2496 ctxt->ops->set_idt(ctxt, &dt);
2498 for (i = 0; i < 6; i++) {
2499 int r = rsm_load_seg_32(ctxt, smbase, i);
2500 if (r != X86EMUL_CONTINUE)
2504 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2506 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2508 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2511 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2513 struct desc_struct desc;
2515 u64 val, cr0, cr3, cr4;
2520 for (i = 0; i < 16; i++)
2521 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2523 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2524 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2526 val = GET_SMSTATE(u32, smbase, 0x7f68);
2527 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2528 val = GET_SMSTATE(u32, smbase, 0x7f60);
2529 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2531 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2532 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2533 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2534 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2535 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2536 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2538 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2539 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2540 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2541 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2542 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2543 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2545 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2546 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2547 ctxt->ops->set_idt(ctxt, &dt);
2549 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2550 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2551 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2552 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2553 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2554 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2556 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2557 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2558 ctxt->ops->set_gdt(ctxt, &dt);
2560 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2561 if (r != X86EMUL_CONTINUE)
2564 for (i = 0; i < 6; i++) {
2565 r = rsm_load_seg_64(ctxt, smbase, i);
2566 if (r != X86EMUL_CONTINUE)
2570 return X86EMUL_CONTINUE;
2573 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2575 unsigned long cr0, cr4, efer;
2579 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2580 return emulate_ud(ctxt);
2583 * Get back to real mode, to prepare a safe state in which to load
2584 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2585 * supports long mode.
2587 if (emulator_has_longmode(ctxt)) {
2588 struct desc_struct cs_desc;
2590 /* Zero CR4.PCIDE before CR0.PG. */
2591 cr4 = ctxt->ops->get_cr(ctxt, 4);
2592 if (cr4 & X86_CR4_PCIDE)
2593 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2595 /* A 32-bit code segment is required to clear EFER.LMA. */
2596 memset(&cs_desc, 0, sizeof(cs_desc));
2598 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2599 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2602 /* For the 64-bit case, this will clear EFER.LMA. */
2603 cr0 = ctxt->ops->get_cr(ctxt, 0);
2604 if (cr0 & X86_CR0_PE)
2605 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2607 if (emulator_has_longmode(ctxt)) {
2608 /* Clear CR4.PAE before clearing EFER.LME. */
2609 cr4 = ctxt->ops->get_cr(ctxt, 4);
2610 if (cr4 & X86_CR4_PAE)
2611 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2613 /* And finally go back to 32-bit mode. */
2615 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2618 smbase = ctxt->ops->get_smbase(ctxt);
2619 if (emulator_has_longmode(ctxt))
2620 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2622 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2624 if (ret != X86EMUL_CONTINUE) {
2625 /* FIXME: should triple fault */
2626 return X86EMUL_UNHANDLEABLE;
2629 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2630 ctxt->ops->set_nmi_mask(ctxt, false);
2632 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2633 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2634 return X86EMUL_CONTINUE;
2638 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2639 struct desc_struct *cs, struct desc_struct *ss)
2641 cs->l = 0; /* will be adjusted later */
2642 set_desc_base(cs, 0); /* flat segment */
2643 cs->g = 1; /* 4kb granularity */
2644 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2645 cs->type = 0x0b; /* Read, Execute, Accessed */
2647 cs->dpl = 0; /* will be adjusted later */
2652 set_desc_base(ss, 0); /* flat segment */
2653 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2654 ss->g = 1; /* 4kb granularity */
2656 ss->type = 0x03; /* Read/Write, Accessed */
2657 ss->d = 1; /* 32bit stack segment */
2664 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2666 u32 eax, ebx, ecx, edx;
2669 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2670 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2671 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2672 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2675 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2677 const struct x86_emulate_ops *ops = ctxt->ops;
2678 u32 eax, ebx, ecx, edx;
2681 * syscall should always be enabled in longmode - so only become
2682 * vendor specific (cpuid) if other modes are active...
2684 if (ctxt->mode == X86EMUL_MODE_PROT64)
2689 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2691 * Intel ("GenuineIntel")
2692 * remark: Intel CPUs only support "syscall" in 64bit
2693 * longmode. Also an 64bit guest with a
2694 * 32bit compat-app running will #UD !! While this
2695 * behaviour can be fixed (by emulating) into AMD
2696 * response - CPUs of AMD can't behave like Intel.
2698 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2699 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2700 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2703 /* AMD ("AuthenticAMD") */
2704 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2705 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2706 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2709 /* AMD ("AMDisbetter!") */
2710 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2711 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2712 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2715 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2719 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2721 const struct x86_emulate_ops *ops = ctxt->ops;
2722 struct desc_struct cs, ss;
2727 /* syscall is not available in real mode */
2728 if (ctxt->mode == X86EMUL_MODE_REAL ||
2729 ctxt->mode == X86EMUL_MODE_VM86)
2730 return emulate_ud(ctxt);
2732 if (!(em_syscall_is_enabled(ctxt)))
2733 return emulate_ud(ctxt);
2735 ops->get_msr(ctxt, MSR_EFER, &efer);
2736 setup_syscalls_segments(ctxt, &cs, &ss);
2738 if (!(efer & EFER_SCE))
2739 return emulate_ud(ctxt);
2741 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2743 cs_sel = (u16)(msr_data & 0xfffc);
2744 ss_sel = (u16)(msr_data + 8);
2746 if (efer & EFER_LMA) {
2750 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2751 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2753 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2754 if (efer & EFER_LMA) {
2755 #ifdef CONFIG_X86_64
2756 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2759 ctxt->mode == X86EMUL_MODE_PROT64 ?
2760 MSR_LSTAR : MSR_CSTAR, &msr_data);
2761 ctxt->_eip = msr_data;
2763 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2764 ctxt->eflags &= ~msr_data;
2765 ctxt->eflags |= X86_EFLAGS_FIXED;
2769 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2770 ctxt->_eip = (u32)msr_data;
2772 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2775 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2776 return X86EMUL_CONTINUE;
2779 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2781 const struct x86_emulate_ops *ops = ctxt->ops;
2782 struct desc_struct cs, ss;
2787 ops->get_msr(ctxt, MSR_EFER, &efer);
2788 /* inject #GP if in real mode */
2789 if (ctxt->mode == X86EMUL_MODE_REAL)
2790 return emulate_gp(ctxt, 0);
2793 * Not recognized on AMD in compat mode (but is recognized in legacy
2796 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2797 && !vendor_intel(ctxt))
2798 return emulate_ud(ctxt);
2800 /* sysenter/sysexit have not been tested in 64bit mode. */
2801 if (ctxt->mode == X86EMUL_MODE_PROT64)
2802 return X86EMUL_UNHANDLEABLE;
2804 setup_syscalls_segments(ctxt, &cs, &ss);
2806 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2807 if ((msr_data & 0xfffc) == 0x0)
2808 return emulate_gp(ctxt, 0);
2810 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2811 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2812 ss_sel = cs_sel + 8;
2813 if (efer & EFER_LMA) {
2818 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2819 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2821 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2822 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2824 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2825 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2828 return X86EMUL_CONTINUE;
2831 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2833 const struct x86_emulate_ops *ops = ctxt->ops;
2834 struct desc_struct cs, ss;
2835 u64 msr_data, rcx, rdx;
2837 u16 cs_sel = 0, ss_sel = 0;
2839 /* inject #GP if in real mode or Virtual 8086 mode */
2840 if (ctxt->mode == X86EMUL_MODE_REAL ||
2841 ctxt->mode == X86EMUL_MODE_VM86)
2842 return emulate_gp(ctxt, 0);
2844 setup_syscalls_segments(ctxt, &cs, &ss);
2846 if ((ctxt->rex_prefix & 0x8) != 0x0)
2847 usermode = X86EMUL_MODE_PROT64;
2849 usermode = X86EMUL_MODE_PROT32;
2851 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2852 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2856 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2858 case X86EMUL_MODE_PROT32:
2859 cs_sel = (u16)(msr_data + 16);
2860 if ((msr_data & 0xfffc) == 0x0)
2861 return emulate_gp(ctxt, 0);
2862 ss_sel = (u16)(msr_data + 24);
2866 case X86EMUL_MODE_PROT64:
2867 cs_sel = (u16)(msr_data + 32);
2868 if (msr_data == 0x0)
2869 return emulate_gp(ctxt, 0);
2870 ss_sel = cs_sel + 8;
2873 if (is_noncanonical_address(rcx) ||
2874 is_noncanonical_address(rdx))
2875 return emulate_gp(ctxt, 0);
2878 cs_sel |= SEGMENT_RPL_MASK;
2879 ss_sel |= SEGMENT_RPL_MASK;
2881 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2882 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2885 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2887 return X86EMUL_CONTINUE;
2890 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2893 if (ctxt->mode == X86EMUL_MODE_REAL)
2895 if (ctxt->mode == X86EMUL_MODE_VM86)
2897 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2898 return ctxt->ops->cpl(ctxt) > iopl;
2901 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2904 const struct x86_emulate_ops *ops = ctxt->ops;
2905 struct desc_struct tr_seg;
2908 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2909 unsigned mask = (1 << len) - 1;
2912 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2915 if (desc_limit_scaled(&tr_seg) < 103)
2917 base = get_desc_base(&tr_seg);
2918 #ifdef CONFIG_X86_64
2919 base |= ((u64)base3) << 32;
2921 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2922 if (r != X86EMUL_CONTINUE)
2924 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2926 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2927 if (r != X86EMUL_CONTINUE)
2929 if ((perm >> bit_idx) & mask)
2934 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2940 if (emulator_bad_iopl(ctxt))
2941 if (!emulator_io_port_access_allowed(ctxt, port, len))
2944 ctxt->perm_ok = true;
2949 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2952 * Intel CPUs mask the counter and pointers in quite strange
2953 * manner when ECX is zero due to REP-string optimizations.
2955 #ifdef CONFIG_X86_64
2956 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2959 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2962 case 0xa4: /* movsb */
2963 case 0xa5: /* movsd/w */
2964 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2966 case 0xaa: /* stosb */
2967 case 0xab: /* stosd/w */
2968 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2973 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2974 struct tss_segment_16 *tss)
2976 tss->ip = ctxt->_eip;
2977 tss->flag = ctxt->eflags;
2978 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2979 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2980 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2981 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2982 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2983 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2984 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2985 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2987 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2988 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2989 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2990 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2991 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2994 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2995 struct tss_segment_16 *tss)
3000 ctxt->_eip = tss->ip;
3001 ctxt->eflags = tss->flag | 2;
3002 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3003 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3004 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3005 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3006 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3007 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3008 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3009 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3012 * SDM says that segment selectors are loaded before segment
3015 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3016 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3017 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3018 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3019 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3024 * Now load segment descriptors. If fault happens at this stage
3025 * it is handled in a context of new task
3027 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3028 X86_TRANSFER_TASK_SWITCH, NULL);
3029 if (ret != X86EMUL_CONTINUE)
3031 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3032 X86_TRANSFER_TASK_SWITCH, NULL);
3033 if (ret != X86EMUL_CONTINUE)
3035 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3036 X86_TRANSFER_TASK_SWITCH, NULL);
3037 if (ret != X86EMUL_CONTINUE)
3039 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3040 X86_TRANSFER_TASK_SWITCH, NULL);
3041 if (ret != X86EMUL_CONTINUE)
3043 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3044 X86_TRANSFER_TASK_SWITCH, NULL);
3045 if (ret != X86EMUL_CONTINUE)
3048 return X86EMUL_CONTINUE;
3051 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3052 u16 tss_selector, u16 old_tss_sel,
3053 ulong old_tss_base, struct desc_struct *new_desc)
3055 struct tss_segment_16 tss_seg;
3057 u32 new_tss_base = get_desc_base(new_desc);
3059 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3060 if (ret != X86EMUL_CONTINUE)
3063 save_state_to_tss16(ctxt, &tss_seg);
3065 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3066 if (ret != X86EMUL_CONTINUE)
3069 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3070 if (ret != X86EMUL_CONTINUE)
3073 if (old_tss_sel != 0xffff) {
3074 tss_seg.prev_task_link = old_tss_sel;
3076 ret = linear_write_system(ctxt, new_tss_base,
3077 &tss_seg.prev_task_link,
3078 sizeof tss_seg.prev_task_link);
3079 if (ret != X86EMUL_CONTINUE)
3083 return load_state_from_tss16(ctxt, &tss_seg);
3086 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3087 struct tss_segment_32 *tss)
3089 /* CR3 and ldt selector are not saved intentionally */
3090 tss->eip = ctxt->_eip;
3091 tss->eflags = ctxt->eflags;
3092 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3093 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3094 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3095 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3096 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3097 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3098 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3099 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3101 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3102 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3103 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3104 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3105 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3106 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3109 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3110 struct tss_segment_32 *tss)
3115 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3116 return emulate_gp(ctxt, 0);
3117 ctxt->_eip = tss->eip;
3118 ctxt->eflags = tss->eflags | 2;
3120 /* General purpose registers */
3121 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3122 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3123 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3124 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3125 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3126 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3127 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3128 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3131 * SDM says that segment selectors are loaded before segment
3132 * descriptors. This is important because CPL checks will
3135 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3136 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3137 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3138 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3139 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3140 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3141 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3144 * If we're switching between Protected Mode and VM86, we need to make
3145 * sure to update the mode before loading the segment descriptors so
3146 * that the selectors are interpreted correctly.
3148 if (ctxt->eflags & X86_EFLAGS_VM) {
3149 ctxt->mode = X86EMUL_MODE_VM86;
3152 ctxt->mode = X86EMUL_MODE_PROT32;
3157 * Now load segment descriptors. If fault happenes at this stage
3158 * it is handled in a context of new task
3160 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3161 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3162 if (ret != X86EMUL_CONTINUE)
3164 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3165 X86_TRANSFER_TASK_SWITCH, NULL);
3166 if (ret != X86EMUL_CONTINUE)
3168 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3169 X86_TRANSFER_TASK_SWITCH, NULL);
3170 if (ret != X86EMUL_CONTINUE)
3172 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3173 X86_TRANSFER_TASK_SWITCH, NULL);
3174 if (ret != X86EMUL_CONTINUE)
3176 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3177 X86_TRANSFER_TASK_SWITCH, NULL);
3178 if (ret != X86EMUL_CONTINUE)
3180 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3181 X86_TRANSFER_TASK_SWITCH, NULL);
3182 if (ret != X86EMUL_CONTINUE)
3184 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3185 X86_TRANSFER_TASK_SWITCH, NULL);
3190 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3191 u16 tss_selector, u16 old_tss_sel,
3192 ulong old_tss_base, struct desc_struct *new_desc)
3194 struct tss_segment_32 tss_seg;
3196 u32 new_tss_base = get_desc_base(new_desc);
3197 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3198 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3200 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3201 if (ret != X86EMUL_CONTINUE)
3204 save_state_to_tss32(ctxt, &tss_seg);
3206 /* Only GP registers and segment selectors are saved */
3207 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3208 ldt_sel_offset - eip_offset);
3209 if (ret != X86EMUL_CONTINUE)
3212 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3213 if (ret != X86EMUL_CONTINUE)
3216 if (old_tss_sel != 0xffff) {
3217 tss_seg.prev_task_link = old_tss_sel;
3219 ret = linear_write_system(ctxt, new_tss_base,
3220 &tss_seg.prev_task_link,
3221 sizeof tss_seg.prev_task_link);
3222 if (ret != X86EMUL_CONTINUE)
3226 return load_state_from_tss32(ctxt, &tss_seg);
3229 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3230 u16 tss_selector, int idt_index, int reason,
3231 bool has_error_code, u32 error_code)
3233 const struct x86_emulate_ops *ops = ctxt->ops;
3234 struct desc_struct curr_tss_desc, next_tss_desc;
3236 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3237 ulong old_tss_base =
3238 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3240 ulong desc_addr, dr7;
3242 /* FIXME: old_tss_base == ~0 ? */
3244 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3245 if (ret != X86EMUL_CONTINUE)
3247 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3248 if (ret != X86EMUL_CONTINUE)
3251 /* FIXME: check that next_tss_desc is tss */
3254 * Check privileges. The three cases are task switch caused by...
3256 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3257 * 2. Exception/IRQ/iret: No check is performed
3258 * 3. jmp/call to TSS/task-gate: No check is performed since the
3259 * hardware checks it before exiting.
3261 if (reason == TASK_SWITCH_GATE) {
3262 if (idt_index != -1) {
3263 /* Software interrupts */
3264 struct desc_struct task_gate_desc;
3267 ret = read_interrupt_descriptor(ctxt, idt_index,
3269 if (ret != X86EMUL_CONTINUE)
3272 dpl = task_gate_desc.dpl;
3273 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3274 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3278 desc_limit = desc_limit_scaled(&next_tss_desc);
3279 if (!next_tss_desc.p ||
3280 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3281 desc_limit < 0x2b)) {
3282 return emulate_ts(ctxt, tss_selector & 0xfffc);
3285 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3286 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3287 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3290 if (reason == TASK_SWITCH_IRET)
3291 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3293 /* set back link to prev task only if NT bit is set in eflags
3294 note that old_tss_sel is not used after this point */
3295 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3296 old_tss_sel = 0xffff;
3298 if (next_tss_desc.type & 8)
3299 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3300 old_tss_base, &next_tss_desc);
3302 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3303 old_tss_base, &next_tss_desc);
3304 if (ret != X86EMUL_CONTINUE)
3307 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3308 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3310 if (reason != TASK_SWITCH_IRET) {
3311 next_tss_desc.type |= (1 << 1); /* set busy flag */
3312 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3315 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3316 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3318 if (has_error_code) {
3319 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3320 ctxt->lock_prefix = 0;
3321 ctxt->src.val = (unsigned long) error_code;
3322 ret = em_push(ctxt);
3325 ops->get_dr(ctxt, 7, &dr7);
3326 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3331 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3332 u16 tss_selector, int idt_index, int reason,
3333 bool has_error_code, u32 error_code)
3337 invalidate_registers(ctxt);
3338 ctxt->_eip = ctxt->eip;
3339 ctxt->dst.type = OP_NONE;
3341 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3342 has_error_code, error_code);
3344 if (rc == X86EMUL_CONTINUE) {
3345 ctxt->eip = ctxt->_eip;
3346 writeback_registers(ctxt);
3349 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3352 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3355 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3357 register_address_increment(ctxt, reg, df * op->bytes);
3358 op->addr.mem.ea = register_address(ctxt, reg);
3361 static int em_das(struct x86_emulate_ctxt *ctxt)
3364 bool af, cf, old_cf;
3366 cf = ctxt->eflags & X86_EFLAGS_CF;
3372 af = ctxt->eflags & X86_EFLAGS_AF;
3373 if ((al & 0x0f) > 9 || af) {
3375 cf = old_cf | (al >= 250);
3380 if (old_al > 0x99 || old_cf) {
3386 /* Set PF, ZF, SF */
3387 ctxt->src.type = OP_IMM;
3389 ctxt->src.bytes = 1;
3390 fastop(ctxt, em_or);
3391 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3393 ctxt->eflags |= X86_EFLAGS_CF;
3395 ctxt->eflags |= X86_EFLAGS_AF;
3396 return X86EMUL_CONTINUE;
3399 static int em_aam(struct x86_emulate_ctxt *ctxt)
3403 if (ctxt->src.val == 0)
3404 return emulate_de(ctxt);
3406 al = ctxt->dst.val & 0xff;
3407 ah = al / ctxt->src.val;
3408 al %= ctxt->src.val;
3410 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3412 /* Set PF, ZF, SF */
3413 ctxt->src.type = OP_IMM;
3415 ctxt->src.bytes = 1;
3416 fastop(ctxt, em_or);
3418 return X86EMUL_CONTINUE;
3421 static int em_aad(struct x86_emulate_ctxt *ctxt)
3423 u8 al = ctxt->dst.val & 0xff;
3424 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3426 al = (al + (ah * ctxt->src.val)) & 0xff;
3428 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3430 /* Set PF, ZF, SF */
3431 ctxt->src.type = OP_IMM;
3433 ctxt->src.bytes = 1;
3434 fastop(ctxt, em_or);
3436 return X86EMUL_CONTINUE;
3439 static int em_call(struct x86_emulate_ctxt *ctxt)
3442 long rel = ctxt->src.val;
3444 ctxt->src.val = (unsigned long)ctxt->_eip;
3445 rc = jmp_rel(ctxt, rel);
3446 if (rc != X86EMUL_CONTINUE)
3448 return em_push(ctxt);
3451 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3456 struct desc_struct old_desc, new_desc;
3457 const struct x86_emulate_ops *ops = ctxt->ops;
3458 int cpl = ctxt->ops->cpl(ctxt);
3459 enum x86emul_mode prev_mode = ctxt->mode;
3461 old_eip = ctxt->_eip;
3462 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3464 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3465 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3466 X86_TRANSFER_CALL_JMP, &new_desc);
3467 if (rc != X86EMUL_CONTINUE)
3470 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3471 if (rc != X86EMUL_CONTINUE)
3474 ctxt->src.val = old_cs;
3476 if (rc != X86EMUL_CONTINUE)
3479 ctxt->src.val = old_eip;
3481 /* If we failed, we tainted the memory, but the very least we should
3483 if (rc != X86EMUL_CONTINUE) {
3484 pr_warn_once("faulting far call emulation tainted memory\n");
3489 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3490 ctxt->mode = prev_mode;
3495 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3500 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3501 if (rc != X86EMUL_CONTINUE)
3503 rc = assign_eip_near(ctxt, eip);
3504 if (rc != X86EMUL_CONTINUE)
3506 rsp_increment(ctxt, ctxt->src.val);
3507 return X86EMUL_CONTINUE;
3510 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3512 /* Write back the register source. */
3513 ctxt->src.val = ctxt->dst.val;
3514 write_register_operand(&ctxt->src);
3516 /* Write back the memory destination with implicit LOCK prefix. */
3517 ctxt->dst.val = ctxt->src.orig_val;
3518 ctxt->lock_prefix = 1;
3519 return X86EMUL_CONTINUE;
3522 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3524 ctxt->dst.val = ctxt->src2.val;
3525 return fastop(ctxt, em_imul);
3528 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3530 ctxt->dst.type = OP_REG;
3531 ctxt->dst.bytes = ctxt->src.bytes;
3532 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3533 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3535 return X86EMUL_CONTINUE;
3538 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3542 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3543 return emulate_ud(ctxt);
3544 ctxt->dst.val = tsc_aux;
3545 return X86EMUL_CONTINUE;
3548 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3552 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3553 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3554 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3555 return X86EMUL_CONTINUE;
3558 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3562 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3563 return emulate_gp(ctxt, 0);
3564 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3565 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3566 return X86EMUL_CONTINUE;
3569 static int em_mov(struct x86_emulate_ctxt *ctxt)
3571 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3572 return X86EMUL_CONTINUE;
3575 #define FFL(x) bit(X86_FEATURE_##x)
3577 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3579 u32 ebx, ecx, edx, eax = 1;
3583 * Check MOVBE is set in the guest-visible CPUID leaf.
3585 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3586 if (!(ecx & FFL(MOVBE)))
3587 return emulate_ud(ctxt);
3589 switch (ctxt->op_bytes) {
3592 * From MOVBE definition: "...When the operand size is 16 bits,
3593 * the upper word of the destination register remains unchanged
3596 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3597 * rules so we have to do the operation almost per hand.
3599 tmp = (u16)ctxt->src.val;
3600 ctxt->dst.val &= ~0xffffUL;
3601 ctxt->dst.val |= (unsigned long)swab16(tmp);
3604 ctxt->dst.val = swab32((u32)ctxt->src.val);
3607 ctxt->dst.val = swab64(ctxt->src.val);
3612 return X86EMUL_CONTINUE;
3615 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3617 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3618 return emulate_gp(ctxt, 0);
3620 /* Disable writeback. */
3621 ctxt->dst.type = OP_NONE;
3622 return X86EMUL_CONTINUE;
3625 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3629 if (ctxt->mode == X86EMUL_MODE_PROT64)
3630 val = ctxt->src.val & ~0ULL;
3632 val = ctxt->src.val & ~0U;
3634 /* #UD condition is already handled. */
3635 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3636 return emulate_gp(ctxt, 0);
3638 /* Disable writeback. */
3639 ctxt->dst.type = OP_NONE;
3640 return X86EMUL_CONTINUE;
3643 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3647 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3648 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3649 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3650 return emulate_gp(ctxt, 0);
3652 return X86EMUL_CONTINUE;
3655 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3659 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3660 return emulate_gp(ctxt, 0);
3662 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3663 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3664 return X86EMUL_CONTINUE;
3667 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3669 if (ctxt->modrm_reg > VCPU_SREG_GS)
3670 return emulate_ud(ctxt);
3672 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3673 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3674 ctxt->dst.bytes = 2;
3675 return X86EMUL_CONTINUE;
3678 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3680 u16 sel = ctxt->src.val;
3682 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3683 return emulate_ud(ctxt);
3685 if (ctxt->modrm_reg == VCPU_SREG_SS)
3686 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3688 /* Disable writeback. */
3689 ctxt->dst.type = OP_NONE;
3690 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3693 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3695 u16 sel = ctxt->src.val;
3697 /* Disable writeback. */
3698 ctxt->dst.type = OP_NONE;
3699 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3702 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3704 u16 sel = ctxt->src.val;
3706 /* Disable writeback. */
3707 ctxt->dst.type = OP_NONE;
3708 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3711 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3716 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3717 if (rc == X86EMUL_CONTINUE)
3718 ctxt->ops->invlpg(ctxt, linear);
3719 /* Disable writeback. */
3720 ctxt->dst.type = OP_NONE;
3721 return X86EMUL_CONTINUE;
3724 static int em_clts(struct x86_emulate_ctxt *ctxt)
3728 cr0 = ctxt->ops->get_cr(ctxt, 0);
3730 ctxt->ops->set_cr(ctxt, 0, cr0);
3731 return X86EMUL_CONTINUE;
3734 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3736 int rc = ctxt->ops->fix_hypercall(ctxt);
3738 if (rc != X86EMUL_CONTINUE)
3741 /* Let the processor re-execute the fixed hypercall */
3742 ctxt->_eip = ctxt->eip;
3743 /* Disable writeback. */
3744 ctxt->dst.type = OP_NONE;
3745 return X86EMUL_CONTINUE;
3748 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3749 void (*get)(struct x86_emulate_ctxt *ctxt,
3750 struct desc_ptr *ptr))
3752 struct desc_ptr desc_ptr;
3754 if (ctxt->mode == X86EMUL_MODE_PROT64)
3756 get(ctxt, &desc_ptr);
3757 if (ctxt->op_bytes == 2) {
3759 desc_ptr.address &= 0x00ffffff;
3761 /* Disable writeback. */
3762 ctxt->dst.type = OP_NONE;
3763 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3764 &desc_ptr, 2 + ctxt->op_bytes);
3767 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3769 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3772 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3774 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3777 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3779 struct desc_ptr desc_ptr;
3782 if (ctxt->mode == X86EMUL_MODE_PROT64)
3784 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3785 &desc_ptr.size, &desc_ptr.address,
3787 if (rc != X86EMUL_CONTINUE)
3789 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3790 is_noncanonical_address(desc_ptr.address))
3791 return emulate_gp(ctxt, 0);
3793 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3795 ctxt->ops->set_idt(ctxt, &desc_ptr);
3796 /* Disable writeback. */
3797 ctxt->dst.type = OP_NONE;
3798 return X86EMUL_CONTINUE;
3801 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3803 return em_lgdt_lidt(ctxt, true);
3806 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3808 return em_lgdt_lidt(ctxt, false);
3811 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3813 if (ctxt->dst.type == OP_MEM)
3814 ctxt->dst.bytes = 2;
3815 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3816 return X86EMUL_CONTINUE;
3819 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3821 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3822 | (ctxt->src.val & 0x0f));
3823 ctxt->dst.type = OP_NONE;
3824 return X86EMUL_CONTINUE;
3827 static int em_loop(struct x86_emulate_ctxt *ctxt)
3829 int rc = X86EMUL_CONTINUE;
3831 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3832 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3833 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3834 rc = jmp_rel(ctxt, ctxt->src.val);
3839 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3841 int rc = X86EMUL_CONTINUE;
3843 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3844 rc = jmp_rel(ctxt, ctxt->src.val);
3849 static int em_in(struct x86_emulate_ctxt *ctxt)
3851 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3853 return X86EMUL_IO_NEEDED;
3855 return X86EMUL_CONTINUE;
3858 static int em_out(struct x86_emulate_ctxt *ctxt)
3860 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3862 /* Disable writeback. */
3863 ctxt->dst.type = OP_NONE;
3864 return X86EMUL_CONTINUE;
3867 static int em_cli(struct x86_emulate_ctxt *ctxt)
3869 if (emulator_bad_iopl(ctxt))
3870 return emulate_gp(ctxt, 0);
3872 ctxt->eflags &= ~X86_EFLAGS_IF;
3873 return X86EMUL_CONTINUE;
3876 static int em_sti(struct x86_emulate_ctxt *ctxt)
3878 if (emulator_bad_iopl(ctxt))
3879 return emulate_gp(ctxt, 0);
3881 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3882 ctxt->eflags |= X86_EFLAGS_IF;
3883 return X86EMUL_CONTINUE;
3886 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3888 u32 eax, ebx, ecx, edx;
3890 eax = reg_read(ctxt, VCPU_REGS_RAX);
3891 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3892 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3893 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3894 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3895 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3896 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3897 return X86EMUL_CONTINUE;
3900 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3904 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3906 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3908 ctxt->eflags &= ~0xffUL;
3909 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3910 return X86EMUL_CONTINUE;
3913 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3915 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3916 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3917 return X86EMUL_CONTINUE;
3920 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3922 switch (ctxt->op_bytes) {
3923 #ifdef CONFIG_X86_64
3925 asm("bswap %0" : "+r"(ctxt->dst.val));
3929 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3932 return X86EMUL_CONTINUE;
3935 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3937 /* emulating clflush regardless of cpuid */
3938 return X86EMUL_CONTINUE;
3941 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3943 /* emulating clflushopt regardless of cpuid */
3944 return X86EMUL_CONTINUE;
3947 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3949 ctxt->dst.val = (s32) ctxt->src.val;
3950 return X86EMUL_CONTINUE;
3953 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3955 u32 eax = 1, ebx, ecx = 0, edx;
3957 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3958 if (!(edx & FFL(FXSR)))
3959 return emulate_ud(ctxt);
3961 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3962 return emulate_nm(ctxt);
3965 * Don't emulate a case that should never be hit, instead of working
3966 * around a lack of fxsave64/fxrstor64 on old compilers.
3968 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3969 return X86EMUL_UNHANDLEABLE;
3971 return X86EMUL_CONTINUE;
3975 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3978 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3979 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3981 * 3) 64-bit mode with REX.W prefix
3982 * - like (2), but XMM 8-15 are being saved and restored
3983 * 4) 64-bit mode without REX.W prefix
3984 * - like (3), but FIP and FDP are 64 bit
3986 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3987 * desired result. (4) is not emulated.
3989 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3990 * and FPU DS) should match.
3992 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
3994 struct fxregs_state fx_state;
3998 rc = check_fxsr(ctxt);
3999 if (rc != X86EMUL_CONTINUE)
4002 ctxt->ops->get_fpu(ctxt);
4004 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4006 ctxt->ops->put_fpu(ctxt);
4008 if (rc != X86EMUL_CONTINUE)
4011 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
4012 size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
4014 size = offsetof(struct fxregs_state, xmm_space[0]);
4016 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4019 static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
4020 struct fxregs_state *new)
4022 int rc = X86EMUL_CONTINUE;
4023 struct fxregs_state old;
4025 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
4026 if (rc != X86EMUL_CONTINUE)
4030 * 64 bit host will restore XMM 8-15, which is not correct on non-64
4031 * bit guests. Load the current values in order to preserve 64 bit
4032 * XMMs after fxrstor.
4034 #ifdef CONFIG_X86_64
4035 /* XXX: accessing XMM 8-15 very awkwardly */
4036 memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
4040 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
4041 * does save and restore MXCSR.
4043 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
4044 memcpy(new->xmm_space, old.xmm_space, 8 * 16);
4049 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4051 struct fxregs_state fx_state;
4054 rc = check_fxsr(ctxt);
4055 if (rc != X86EMUL_CONTINUE)
4058 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
4059 if (rc != X86EMUL_CONTINUE)
4062 if (fx_state.mxcsr >> 16)
4063 return emulate_gp(ctxt, 0);
4065 ctxt->ops->get_fpu(ctxt);
4067 if (ctxt->mode < X86EMUL_MODE_PROT64)
4068 rc = fxrstor_fixup(ctxt, &fx_state);
4070 if (rc == X86EMUL_CONTINUE)
4071 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4073 ctxt->ops->put_fpu(ctxt);
4078 static bool valid_cr(int nr)
4090 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4092 if (!valid_cr(ctxt->modrm_reg))
4093 return emulate_ud(ctxt);
4095 return X86EMUL_CONTINUE;
4098 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4100 u64 new_val = ctxt->src.val64;
4101 int cr = ctxt->modrm_reg;
4104 static u64 cr_reserved_bits[] = {
4105 0xffffffff00000000ULL,
4106 0, 0, 0, /* CR3 checked later */
4113 return emulate_ud(ctxt);
4115 if (new_val & cr_reserved_bits[cr])
4116 return emulate_gp(ctxt, 0);
4121 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4122 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4123 return emulate_gp(ctxt, 0);
4125 cr4 = ctxt->ops->get_cr(ctxt, 4);
4126 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4128 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4129 !(cr4 & X86_CR4_PAE))
4130 return emulate_gp(ctxt, 0);
4137 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4138 if (efer & EFER_LMA)
4139 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
4142 return emulate_gp(ctxt, 0);
4147 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4149 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4150 return emulate_gp(ctxt, 0);
4156 return X86EMUL_CONTINUE;
4159 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4163 ctxt->ops->get_dr(ctxt, 7, &dr7);
4165 /* Check if DR7.Global_Enable is set */
4166 return dr7 & (1 << 13);
4169 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4171 int dr = ctxt->modrm_reg;
4175 return emulate_ud(ctxt);
4177 cr4 = ctxt->ops->get_cr(ctxt, 4);
4178 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4179 return emulate_ud(ctxt);
4181 if (check_dr7_gd(ctxt)) {
4184 ctxt->ops->get_dr(ctxt, 6, &dr6);
4186 dr6 |= DR6_BD | DR6_RTM;
4187 ctxt->ops->set_dr(ctxt, 6, dr6);
4188 return emulate_db(ctxt);
4191 return X86EMUL_CONTINUE;
4194 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4196 u64 new_val = ctxt->src.val64;
4197 int dr = ctxt->modrm_reg;
4199 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4200 return emulate_gp(ctxt, 0);
4202 return check_dr_read(ctxt);
4205 static int check_svme(struct x86_emulate_ctxt *ctxt)
4209 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4211 if (!(efer & EFER_SVME))
4212 return emulate_ud(ctxt);
4214 return X86EMUL_CONTINUE;
4217 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4219 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4221 /* Valid physical address? */
4222 if (rax & 0xffff000000000000ULL)
4223 return emulate_gp(ctxt, 0);
4225 return check_svme(ctxt);
4228 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4230 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4232 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4233 return emulate_ud(ctxt);
4235 return X86EMUL_CONTINUE;
4238 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4240 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4241 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4243 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4244 ctxt->ops->check_pmc(ctxt, rcx))
4245 return emulate_gp(ctxt, 0);
4247 return X86EMUL_CONTINUE;
4250 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4252 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4253 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4254 return emulate_gp(ctxt, 0);
4256 return X86EMUL_CONTINUE;
4259 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4261 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4262 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4263 return emulate_gp(ctxt, 0);
4265 return X86EMUL_CONTINUE;
4268 #define D(_y) { .flags = (_y) }
4269 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4270 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4271 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4272 #define N D(NotImpl)
4273 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4274 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4275 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4276 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4277 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4278 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4279 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4280 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4281 #define II(_f, _e, _i) \
4282 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4283 #define IIP(_f, _e, _i, _p) \
4284 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4285 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4286 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4288 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4289 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4290 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4291 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4292 #define I2bvIP(_f, _e, _i, _p) \
4293 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4295 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4296 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4297 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4299 static const struct opcode group7_rm0[] = {
4301 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4305 static const struct opcode group7_rm1[] = {
4306 DI(SrcNone | Priv, monitor),
4307 DI(SrcNone | Priv, mwait),
4311 static const struct opcode group7_rm3[] = {
4312 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4313 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4314 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4315 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4316 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4317 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4318 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4319 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4322 static const struct opcode group7_rm7[] = {
4324 DIP(SrcNone, rdtscp, check_rdtsc),
4328 static const struct opcode group1[] = {
4330 F(Lock | PageTable, em_or),
4333 F(Lock | PageTable, em_and),
4339 static const struct opcode group1A[] = {
4340 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4343 static const struct opcode group2[] = {
4344 F(DstMem | ModRM, em_rol),
4345 F(DstMem | ModRM, em_ror),
4346 F(DstMem | ModRM, em_rcl),
4347 F(DstMem | ModRM, em_rcr),
4348 F(DstMem | ModRM, em_shl),
4349 F(DstMem | ModRM, em_shr),
4350 F(DstMem | ModRM, em_shl),
4351 F(DstMem | ModRM, em_sar),
4354 static const struct opcode group3[] = {
4355 F(DstMem | SrcImm | NoWrite, em_test),
4356 F(DstMem | SrcImm | NoWrite, em_test),
4357 F(DstMem | SrcNone | Lock, em_not),
4358 F(DstMem | SrcNone | Lock, em_neg),
4359 F(DstXacc | Src2Mem, em_mul_ex),
4360 F(DstXacc | Src2Mem, em_imul_ex),
4361 F(DstXacc | Src2Mem, em_div_ex),
4362 F(DstXacc | Src2Mem, em_idiv_ex),
4365 static const struct opcode group4[] = {
4366 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4367 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4371 static const struct opcode group5[] = {
4372 F(DstMem | SrcNone | Lock, em_inc),
4373 F(DstMem | SrcNone | Lock, em_dec),
4374 I(SrcMem | NearBranch, em_call_near_abs),
4375 I(SrcMemFAddr | ImplicitOps, em_call_far),
4376 I(SrcMem | NearBranch, em_jmp_abs),
4377 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4378 I(SrcMem | Stack, em_push), D(Undefined),
4381 static const struct opcode group6[] = {
4382 DI(Prot | DstMem, sldt),
4383 DI(Prot | DstMem, str),
4384 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4385 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4389 static const struct group_dual group7 = { {
4390 II(Mov | DstMem, em_sgdt, sgdt),
4391 II(Mov | DstMem, em_sidt, sidt),
4392 II(SrcMem | Priv, em_lgdt, lgdt),
4393 II(SrcMem | Priv, em_lidt, lidt),
4394 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4395 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4396 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4400 N, EXT(0, group7_rm3),
4401 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4402 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4406 static const struct opcode group8[] = {
4408 F(DstMem | SrcImmByte | NoWrite, em_bt),
4409 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4410 F(DstMem | SrcImmByte | Lock, em_btr),
4411 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4415 * The "memory" destination is actually always a register, since we come
4416 * from the register case of group9.
4418 static const struct gprefix pfx_0f_c7_7 = {
4419 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4423 static const struct group_dual group9 = { {
4424 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4426 N, N, N, N, N, N, N,
4427 GP(0, &pfx_0f_c7_7),
4430 static const struct opcode group11[] = {
4431 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4435 static const struct gprefix pfx_0f_ae_7 = {
4436 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4439 static const struct group_dual group15 = { {
4440 I(ModRM | Aligned16, em_fxsave),
4441 I(ModRM | Aligned16, em_fxrstor),
4442 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4444 N, N, N, N, N, N, N, N,
4447 static const struct gprefix pfx_0f_6f_0f_7f = {
4448 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4451 static const struct instr_dual instr_dual_0f_2b = {
4455 static const struct gprefix pfx_0f_2b = {
4456 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4459 static const struct gprefix pfx_0f_28_0f_29 = {
4460 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4463 static const struct gprefix pfx_0f_e7 = {
4464 N, I(Sse, em_mov), N, N,
4467 static const struct escape escape_d9 = { {
4468 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4471 N, N, N, N, N, N, N, N,
4473 N, N, N, N, N, N, N, N,
4475 N, N, N, N, N, N, N, N,
4477 N, N, N, N, N, N, N, N,
4479 N, N, N, N, N, N, N, N,
4481 N, N, N, N, N, N, N, N,
4483 N, N, N, N, N, N, N, N,
4485 N, N, N, N, N, N, N, N,
4488 static const struct escape escape_db = { {
4489 N, N, N, N, N, N, N, N,
4492 N, N, N, N, N, N, N, N,
4494 N, N, N, N, N, N, N, N,
4496 N, N, N, N, N, N, N, N,
4498 N, N, N, N, N, N, N, N,
4500 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4502 N, N, N, N, N, N, N, N,
4504 N, N, N, N, N, N, N, N,
4506 N, N, N, N, N, N, N, N,
4509 static const struct escape escape_dd = { {
4510 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4513 N, N, N, N, N, N, N, N,
4515 N, N, N, N, N, N, N, N,
4517 N, N, N, N, N, N, N, N,
4519 N, N, N, N, N, N, N, N,
4521 N, N, N, N, N, N, N, N,
4523 N, N, N, N, N, N, N, N,
4525 N, N, N, N, N, N, N, N,
4527 N, N, N, N, N, N, N, N,
4530 static const struct instr_dual instr_dual_0f_c3 = {
4531 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4534 static const struct mode_dual mode_dual_63 = {
4535 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4538 static const struct opcode opcode_table[256] = {
4540 F6ALU(Lock, em_add),
4541 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4542 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4544 F6ALU(Lock | PageTable, em_or),
4545 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4548 F6ALU(Lock, em_adc),
4549 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4550 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4552 F6ALU(Lock, em_sbb),
4553 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4554 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4556 F6ALU(Lock | PageTable, em_and), N, N,
4558 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4560 F6ALU(Lock, em_xor), N, N,
4562 F6ALU(NoWrite, em_cmp), N, N,
4564 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4566 X8(I(SrcReg | Stack, em_push)),
4568 X8(I(DstReg | Stack, em_pop)),
4570 I(ImplicitOps | Stack | No64, em_pusha),
4571 I(ImplicitOps | Stack | No64, em_popa),
4572 N, MD(ModRM, &mode_dual_63),
4575 I(SrcImm | Mov | Stack, em_push),
4576 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4577 I(SrcImmByte | Mov | Stack, em_push),
4578 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4579 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4580 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4582 X16(D(SrcImmByte | NearBranch)),
4584 G(ByteOp | DstMem | SrcImm, group1),
4585 G(DstMem | SrcImm, group1),
4586 G(ByteOp | DstMem | SrcImm | No64, group1),
4587 G(DstMem | SrcImmByte, group1),
4588 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4589 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4591 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4592 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4593 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4594 D(ModRM | SrcMem | NoAccess | DstReg),
4595 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4598 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4600 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4601 I(SrcImmFAddr | No64, em_call_far), N,
4602 II(ImplicitOps | Stack, em_pushf, pushf),
4603 II(ImplicitOps | Stack, em_popf, popf),
4604 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4606 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4607 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4608 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4609 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4611 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4612 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4613 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4614 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4616 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4618 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4620 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4621 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4622 I(ImplicitOps | NearBranch, em_ret),
4623 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4624 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4625 G(ByteOp, group11), G(0, group11),
4627 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4628 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4629 I(ImplicitOps, em_ret_far),
4630 D(ImplicitOps), DI(SrcImmByte, intn),
4631 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4633 G(Src2One | ByteOp, group2), G(Src2One, group2),
4634 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4635 I(DstAcc | SrcImmUByte | No64, em_aam),
4636 I(DstAcc | SrcImmUByte | No64, em_aad),
4637 F(DstAcc | ByteOp | No64, em_salc),
4638 I(DstAcc | SrcXLat | ByteOp, em_mov),
4640 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4642 X3(I(SrcImmByte | NearBranch, em_loop)),
4643 I(SrcImmByte | NearBranch, em_jcxz),
4644 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4645 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4647 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4648 I(SrcImmFAddr | No64, em_jmp_far),
4649 D(SrcImmByte | ImplicitOps | NearBranch),
4650 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4651 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4653 N, DI(ImplicitOps, icebp), N, N,
4654 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4655 G(ByteOp, group3), G(0, group3),
4657 D(ImplicitOps), D(ImplicitOps),
4658 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4659 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4662 static const struct opcode twobyte_table[256] = {
4664 G(0, group6), GD(0, &group7), N, N,
4665 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4666 II(ImplicitOps | Priv, em_clts, clts), N,
4667 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4668 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4670 N, N, N, N, N, N, N, N,
4671 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4672 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4674 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4675 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4676 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4678 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4681 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4682 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4683 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4686 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4687 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4688 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4689 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4690 I(ImplicitOps | EmulateOnUD, em_sysenter),
4691 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4693 N, N, N, N, N, N, N, N,
4695 X16(D(DstReg | SrcMem | ModRM)),
4697 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4702 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4707 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4709 X16(D(SrcImm | NearBranch)),
4711 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4713 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4714 II(ImplicitOps, em_cpuid, cpuid),
4715 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4716 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4717 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4719 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4720 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4721 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4722 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4723 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4724 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4726 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4727 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4728 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4729 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4730 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4731 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4735 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4736 I(DstReg | SrcMem | ModRM, em_bsf_c),
4737 I(DstReg | SrcMem | ModRM, em_bsr_c),
4738 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4740 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4741 N, ID(0, &instr_dual_0f_c3),
4742 N, N, N, GD(0, &group9),
4744 X8(I(DstReg, em_bswap)),
4746 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4748 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4749 N, N, N, N, N, N, N, N,
4751 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4754 static const struct instr_dual instr_dual_0f_38_f0 = {
4755 I(DstReg | SrcMem | Mov, em_movbe), N
4758 static const struct instr_dual instr_dual_0f_38_f1 = {
4759 I(DstMem | SrcReg | Mov, em_movbe), N
4762 static const struct gprefix three_byte_0f_38_f0 = {
4763 ID(0, &instr_dual_0f_38_f0), N, N, N
4766 static const struct gprefix three_byte_0f_38_f1 = {
4767 ID(0, &instr_dual_0f_38_f1), N, N, N
4771 * Insns below are selected by the prefix which indexed by the third opcode
4774 static const struct opcode opcode_map_0f_38[256] = {
4776 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4778 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4780 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4781 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4802 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4806 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4812 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4813 unsigned size, bool sign_extension)
4815 int rc = X86EMUL_CONTINUE;
4819 op->addr.mem.ea = ctxt->_eip;
4820 /* NB. Immediates are sign-extended as necessary. */
4821 switch (op->bytes) {
4823 op->val = insn_fetch(s8, ctxt);
4826 op->val = insn_fetch(s16, ctxt);
4829 op->val = insn_fetch(s32, ctxt);
4832 op->val = insn_fetch(s64, ctxt);
4835 if (!sign_extension) {
4836 switch (op->bytes) {
4844 op->val &= 0xffffffff;
4852 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4855 int rc = X86EMUL_CONTINUE;
4859 decode_register_operand(ctxt, op);
4862 rc = decode_imm(ctxt, op, 1, false);
4865 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4869 if (ctxt->d & BitOp)
4870 fetch_bit_operand(ctxt);
4871 op->orig_val = op->val;
4874 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4878 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4879 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4880 fetch_register_operand(op);
4881 op->orig_val = op->val;
4885 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4886 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4887 fetch_register_operand(op);
4888 op->orig_val = op->val;
4891 if (ctxt->d & ByteOp) {
4896 op->bytes = ctxt->op_bytes;
4897 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4898 fetch_register_operand(op);
4899 op->orig_val = op->val;
4903 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4905 register_address(ctxt, VCPU_REGS_RDI);
4906 op->addr.mem.seg = VCPU_SREG_ES;
4913 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4914 fetch_register_operand(op);
4919 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4922 rc = decode_imm(ctxt, op, 1, true);
4930 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4933 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4936 ctxt->memop.bytes = 1;
4937 if (ctxt->memop.type == OP_REG) {
4938 ctxt->memop.addr.reg = decode_register(ctxt,
4939 ctxt->modrm_rm, true);
4940 fetch_register_operand(&ctxt->memop);
4944 ctxt->memop.bytes = 2;
4947 ctxt->memop.bytes = 4;
4950 rc = decode_imm(ctxt, op, 2, false);
4953 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4957 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4959 register_address(ctxt, VCPU_REGS_RSI);
4960 op->addr.mem.seg = ctxt->seg_override;
4966 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4969 reg_read(ctxt, VCPU_REGS_RBX) +
4970 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4971 op->addr.mem.seg = ctxt->seg_override;
4976 op->addr.mem.ea = ctxt->_eip;
4977 op->bytes = ctxt->op_bytes + 2;
4978 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4981 ctxt->memop.bytes = ctxt->op_bytes + 2;
4985 op->val = VCPU_SREG_ES;
4989 op->val = VCPU_SREG_CS;
4993 op->val = VCPU_SREG_SS;
4997 op->val = VCPU_SREG_DS;
5001 op->val = VCPU_SREG_FS;
5005 op->val = VCPU_SREG_GS;
5008 /* Special instructions do their own operand decoding. */
5010 op->type = OP_NONE; /* Disable writeback. */
5018 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5020 int rc = X86EMUL_CONTINUE;
5021 int mode = ctxt->mode;
5022 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5023 bool op_prefix = false;
5024 bool has_seg_override = false;
5025 struct opcode opcode;
5027 struct desc_struct desc;
5029 ctxt->memop.type = OP_NONE;
5030 ctxt->memopp = NULL;
5031 ctxt->_eip = ctxt->eip;
5032 ctxt->fetch.ptr = ctxt->fetch.data;
5033 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5034 ctxt->opcode_len = 1;
5035 ctxt->intercept = x86_intercept_none;
5037 memcpy(ctxt->fetch.data, insn, insn_len);
5039 rc = __do_insn_fetch_bytes(ctxt, 1);
5040 if (rc != X86EMUL_CONTINUE)
5045 case X86EMUL_MODE_REAL:
5046 case X86EMUL_MODE_VM86:
5047 def_op_bytes = def_ad_bytes = 2;
5048 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5050 def_op_bytes = def_ad_bytes = 4;
5052 case X86EMUL_MODE_PROT16:
5053 def_op_bytes = def_ad_bytes = 2;
5055 case X86EMUL_MODE_PROT32:
5056 def_op_bytes = def_ad_bytes = 4;
5058 #ifdef CONFIG_X86_64
5059 case X86EMUL_MODE_PROT64:
5065 return EMULATION_FAILED;
5068 ctxt->op_bytes = def_op_bytes;
5069 ctxt->ad_bytes = def_ad_bytes;
5071 /* Legacy prefixes. */
5073 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5074 case 0x66: /* operand-size override */
5076 /* switch between 2/4 bytes */
5077 ctxt->op_bytes = def_op_bytes ^ 6;
5079 case 0x67: /* address-size override */
5080 if (mode == X86EMUL_MODE_PROT64)
5081 /* switch between 4/8 bytes */
5082 ctxt->ad_bytes = def_ad_bytes ^ 12;
5084 /* switch between 2/4 bytes */
5085 ctxt->ad_bytes = def_ad_bytes ^ 6;
5087 case 0x26: /* ES override */
5088 has_seg_override = true;
5089 ctxt->seg_override = VCPU_SREG_ES;
5091 case 0x2e: /* CS override */
5092 has_seg_override = true;
5093 ctxt->seg_override = VCPU_SREG_CS;
5095 case 0x36: /* SS override */
5096 has_seg_override = true;
5097 ctxt->seg_override = VCPU_SREG_SS;
5099 case 0x3e: /* DS override */
5100 has_seg_override = true;
5101 ctxt->seg_override = VCPU_SREG_DS;
5103 case 0x64: /* FS override */
5104 has_seg_override = true;
5105 ctxt->seg_override = VCPU_SREG_FS;
5107 case 0x65: /* GS override */
5108 has_seg_override = true;
5109 ctxt->seg_override = VCPU_SREG_GS;
5111 case 0x40 ... 0x4f: /* REX */
5112 if (mode != X86EMUL_MODE_PROT64)
5114 ctxt->rex_prefix = ctxt->b;
5116 case 0xf0: /* LOCK */
5117 ctxt->lock_prefix = 1;
5119 case 0xf2: /* REPNE/REPNZ */
5120 case 0xf3: /* REP/REPE/REPZ */
5121 ctxt->rep_prefix = ctxt->b;
5127 /* Any legacy prefix after a REX prefix nullifies its effect. */
5129 ctxt->rex_prefix = 0;
5135 if (ctxt->rex_prefix & 8)
5136 ctxt->op_bytes = 8; /* REX.W */
5138 /* Opcode byte(s). */
5139 opcode = opcode_table[ctxt->b];
5140 /* Two-byte opcode? */
5141 if (ctxt->b == 0x0f) {
5142 ctxt->opcode_len = 2;
5143 ctxt->b = insn_fetch(u8, ctxt);
5144 opcode = twobyte_table[ctxt->b];
5146 /* 0F_38 opcode map */
5147 if (ctxt->b == 0x38) {
5148 ctxt->opcode_len = 3;
5149 ctxt->b = insn_fetch(u8, ctxt);
5150 opcode = opcode_map_0f_38[ctxt->b];
5153 ctxt->d = opcode.flags;
5155 if (ctxt->d & ModRM)
5156 ctxt->modrm = insn_fetch(u8, ctxt);
5158 /* vex-prefix instructions are not implemented */
5159 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5160 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5164 while (ctxt->d & GroupMask) {
5165 switch (ctxt->d & GroupMask) {
5167 goffset = (ctxt->modrm >> 3) & 7;
5168 opcode = opcode.u.group[goffset];
5171 goffset = (ctxt->modrm >> 3) & 7;
5172 if ((ctxt->modrm >> 6) == 3)
5173 opcode = opcode.u.gdual->mod3[goffset];
5175 opcode = opcode.u.gdual->mod012[goffset];
5178 goffset = ctxt->modrm & 7;
5179 opcode = opcode.u.group[goffset];
5182 if (ctxt->rep_prefix && op_prefix)
5183 return EMULATION_FAILED;
5184 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5185 switch (simd_prefix) {
5186 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5187 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5188 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5189 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5193 if (ctxt->modrm > 0xbf) {
5194 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5195 u32 index = array_index_nospec(
5196 ctxt->modrm - 0xc0, size);
5198 opcode = opcode.u.esc->high[index];
5200 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5204 if ((ctxt->modrm >> 6) == 3)
5205 opcode = opcode.u.idual->mod3;
5207 opcode = opcode.u.idual->mod012;
5210 if (ctxt->mode == X86EMUL_MODE_PROT64)
5211 opcode = opcode.u.mdual->mode64;
5213 opcode = opcode.u.mdual->mode32;
5216 return EMULATION_FAILED;
5219 ctxt->d &= ~(u64)GroupMask;
5220 ctxt->d |= opcode.flags;
5225 return EMULATION_FAILED;
5227 ctxt->execute = opcode.u.execute;
5229 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5230 return EMULATION_FAILED;
5232 if (unlikely(ctxt->d &
5233 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5236 * These are copied unconditionally here, and checked unconditionally
5237 * in x86_emulate_insn.
5239 ctxt->check_perm = opcode.check_perm;
5240 ctxt->intercept = opcode.intercept;
5242 if (ctxt->d & NotImpl)
5243 return EMULATION_FAILED;
5245 if (mode == X86EMUL_MODE_PROT64) {
5246 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5248 else if (ctxt->d & NearBranch)
5252 if (ctxt->d & Op3264) {
5253 if (mode == X86EMUL_MODE_PROT64)
5259 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5263 ctxt->op_bytes = 16;
5264 else if (ctxt->d & Mmx)
5268 /* ModRM and SIB bytes. */
5269 if (ctxt->d & ModRM) {
5270 rc = decode_modrm(ctxt, &ctxt->memop);
5271 if (!has_seg_override) {
5272 has_seg_override = true;
5273 ctxt->seg_override = ctxt->modrm_seg;
5275 } else if (ctxt->d & MemAbs)
5276 rc = decode_abs(ctxt, &ctxt->memop);
5277 if (rc != X86EMUL_CONTINUE)
5280 if (!has_seg_override)
5281 ctxt->seg_override = VCPU_SREG_DS;
5283 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5286 * Decode and fetch the source operand: register, memory
5289 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5290 if (rc != X86EMUL_CONTINUE)
5294 * Decode and fetch the second source operand: register, memory
5297 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5298 if (rc != X86EMUL_CONTINUE)
5301 /* Decode and fetch the destination operand: register or memory. */
5302 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5304 if (ctxt->rip_relative && likely(ctxt->memopp))
5305 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5306 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5309 if (rc == X86EMUL_PROPAGATE_FAULT)
5310 ctxt->have_exception = true;
5311 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5314 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5316 return ctxt->d & PageTable;
5319 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5321 /* The second termination condition only applies for REPE
5322 * and REPNE. Test if the repeat string operation prefix is
5323 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5324 * corresponding termination condition according to:
5325 * - if REPE/REPZ and ZF = 0 then done
5326 * - if REPNE/REPNZ and ZF = 1 then done
5328 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5329 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5330 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5331 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5332 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5333 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5339 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5343 ctxt->ops->get_fpu(ctxt);
5344 rc = asm_safe("fwait");
5345 ctxt->ops->put_fpu(ctxt);
5347 if (unlikely(rc != X86EMUL_CONTINUE))
5348 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5350 return X86EMUL_CONTINUE;
5353 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5356 if (op->type == OP_MM)
5357 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5360 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5362 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5364 if (!(ctxt->d & ByteOp))
5365 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5367 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5368 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5369 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5370 : "c"(ctxt->src2.val));
5372 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5373 if (!fop) /* exception is returned in fop variable */
5374 return emulate_de(ctxt);
5375 return X86EMUL_CONTINUE;
5378 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5380 memset(&ctxt->rip_relative, 0,
5381 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5383 ctxt->io_read.pos = 0;
5384 ctxt->io_read.end = 0;
5385 ctxt->mem_read.end = 0;
5388 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5390 const struct x86_emulate_ops *ops = ctxt->ops;
5391 int rc = X86EMUL_CONTINUE;
5392 int saved_dst_type = ctxt->dst.type;
5393 unsigned emul_flags;
5395 ctxt->mem_read.pos = 0;
5397 /* LOCK prefix is allowed only with some instructions */
5398 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5399 rc = emulate_ud(ctxt);
5403 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5404 rc = emulate_ud(ctxt);
5408 emul_flags = ctxt->ops->get_hflags(ctxt);
5409 if (unlikely(ctxt->d &
5410 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5411 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5412 (ctxt->d & Undefined)) {
5413 rc = emulate_ud(ctxt);
5417 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5418 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5419 rc = emulate_ud(ctxt);
5423 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5424 rc = emulate_nm(ctxt);
5428 if (ctxt->d & Mmx) {
5429 rc = flush_pending_x87_faults(ctxt);
5430 if (rc != X86EMUL_CONTINUE)
5433 * Now that we know the fpu is exception safe, we can fetch
5436 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5437 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5438 if (!(ctxt->d & Mov))
5439 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5442 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5443 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5444 X86_ICPT_PRE_EXCEPT);
5445 if (rc != X86EMUL_CONTINUE)
5449 /* Instruction can only be executed in protected mode */
5450 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5451 rc = emulate_ud(ctxt);
5455 /* Privileged instruction can be executed only in CPL=0 */
5456 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5457 if (ctxt->d & PrivUD)
5458 rc = emulate_ud(ctxt);
5460 rc = emulate_gp(ctxt, 0);
5464 /* Do instruction specific permission checks */
5465 if (ctxt->d & CheckPerm) {
5466 rc = ctxt->check_perm(ctxt);
5467 if (rc != X86EMUL_CONTINUE)
5471 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5472 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5473 X86_ICPT_POST_EXCEPT);
5474 if (rc != X86EMUL_CONTINUE)
5478 if (ctxt->rep_prefix && (ctxt->d & String)) {
5479 /* All REP prefixes have the same first termination condition */
5480 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5481 string_registers_quirk(ctxt);
5482 ctxt->eip = ctxt->_eip;
5483 ctxt->eflags &= ~X86_EFLAGS_RF;
5489 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5490 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5491 ctxt->src.valptr, ctxt->src.bytes);
5492 if (rc != X86EMUL_CONTINUE)
5494 ctxt->src.orig_val64 = ctxt->src.val64;
5497 if (ctxt->src2.type == OP_MEM) {
5498 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5499 &ctxt->src2.val, ctxt->src2.bytes);
5500 if (rc != X86EMUL_CONTINUE)
5504 if ((ctxt->d & DstMask) == ImplicitOps)
5508 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5509 /* optimisation - avoid slow emulated read if Mov */
5510 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5511 &ctxt->dst.val, ctxt->dst.bytes);
5512 if (rc != X86EMUL_CONTINUE) {
5513 if (!(ctxt->d & NoWrite) &&
5514 rc == X86EMUL_PROPAGATE_FAULT &&
5515 ctxt->exception.vector == PF_VECTOR)
5516 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5520 /* Copy full 64-bit value for CMPXCHG8B. */
5521 ctxt->dst.orig_val64 = ctxt->dst.val64;
5525 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5526 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5527 X86_ICPT_POST_MEMACCESS);
5528 if (rc != X86EMUL_CONTINUE)
5532 if (ctxt->rep_prefix && (ctxt->d & String))
5533 ctxt->eflags |= X86_EFLAGS_RF;
5535 ctxt->eflags &= ~X86_EFLAGS_RF;
5537 if (ctxt->execute) {
5538 if (ctxt->d & Fastop) {
5539 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5540 rc = fastop(ctxt, fop);
5541 if (rc != X86EMUL_CONTINUE)
5545 rc = ctxt->execute(ctxt);
5546 if (rc != X86EMUL_CONTINUE)
5551 if (ctxt->opcode_len == 2)
5553 else if (ctxt->opcode_len == 3)
5554 goto threebyte_insn;
5557 case 0x70 ... 0x7f: /* jcc (short) */
5558 if (test_cc(ctxt->b, ctxt->eflags))
5559 rc = jmp_rel(ctxt, ctxt->src.val);
5561 case 0x8d: /* lea r16/r32, m */
5562 ctxt->dst.val = ctxt->src.addr.mem.ea;
5564 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5565 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5566 ctxt->dst.type = OP_NONE;
5570 case 0x98: /* cbw/cwde/cdqe */
5571 switch (ctxt->op_bytes) {
5572 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5573 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5574 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5577 case 0xcc: /* int3 */
5578 rc = emulate_int(ctxt, 3);
5580 case 0xcd: /* int n */
5581 rc = emulate_int(ctxt, ctxt->src.val);
5583 case 0xce: /* into */
5584 if (ctxt->eflags & X86_EFLAGS_OF)
5585 rc = emulate_int(ctxt, 4);
5587 case 0xe9: /* jmp rel */
5588 case 0xeb: /* jmp rel short */
5589 rc = jmp_rel(ctxt, ctxt->src.val);
5590 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5592 case 0xf4: /* hlt */
5593 ctxt->ops->halt(ctxt);
5595 case 0xf5: /* cmc */
5596 /* complement carry flag from eflags reg */
5597 ctxt->eflags ^= X86_EFLAGS_CF;
5599 case 0xf8: /* clc */
5600 ctxt->eflags &= ~X86_EFLAGS_CF;
5602 case 0xf9: /* stc */
5603 ctxt->eflags |= X86_EFLAGS_CF;
5605 case 0xfc: /* cld */
5606 ctxt->eflags &= ~X86_EFLAGS_DF;
5608 case 0xfd: /* std */
5609 ctxt->eflags |= X86_EFLAGS_DF;
5612 goto cannot_emulate;
5615 if (rc != X86EMUL_CONTINUE)
5619 if (ctxt->d & SrcWrite) {
5620 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5621 rc = writeback(ctxt, &ctxt->src);
5622 if (rc != X86EMUL_CONTINUE)
5625 if (!(ctxt->d & NoWrite)) {
5626 rc = writeback(ctxt, &ctxt->dst);
5627 if (rc != X86EMUL_CONTINUE)
5632 * restore dst type in case the decoding will be reused
5633 * (happens for string instruction )
5635 ctxt->dst.type = saved_dst_type;
5637 if ((ctxt->d & SrcMask) == SrcSI)
5638 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5640 if ((ctxt->d & DstMask) == DstDI)
5641 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5643 if (ctxt->rep_prefix && (ctxt->d & String)) {
5645 struct read_cache *r = &ctxt->io_read;
5646 if ((ctxt->d & SrcMask) == SrcSI)
5647 count = ctxt->src.count;
5649 count = ctxt->dst.count;
5650 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5652 if (!string_insn_completed(ctxt)) {
5654 * Re-enter guest when pio read ahead buffer is empty
5655 * or, if it is not used, after each 1024 iteration.
5657 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5658 (r->end == 0 || r->end != r->pos)) {
5660 * Reset read cache. Usually happens before
5661 * decode, but since instruction is restarted
5662 * we have to do it here.
5664 ctxt->mem_read.end = 0;
5665 writeback_registers(ctxt);
5666 return EMULATION_RESTART;
5668 goto done; /* skip rip writeback */
5670 ctxt->eflags &= ~X86_EFLAGS_RF;
5673 ctxt->eip = ctxt->_eip;
5676 if (rc == X86EMUL_PROPAGATE_FAULT) {
5677 WARN_ON(ctxt->exception.vector > 0x1f);
5678 ctxt->have_exception = true;
5680 if (rc == X86EMUL_INTERCEPTED)
5681 return EMULATION_INTERCEPTED;
5683 if (rc == X86EMUL_CONTINUE)
5684 writeback_registers(ctxt);
5686 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5690 case 0x09: /* wbinvd */
5691 (ctxt->ops->wbinvd)(ctxt);
5693 case 0x08: /* invd */
5694 case 0x0d: /* GrpP (prefetch) */
5695 case 0x18: /* Grp16 (prefetch/nop) */
5696 case 0x1f: /* nop */
5698 case 0x20: /* mov cr, reg */
5699 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5701 case 0x21: /* mov from dr to reg */
5702 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5704 case 0x40 ... 0x4f: /* cmov */
5705 if (test_cc(ctxt->b, ctxt->eflags))
5706 ctxt->dst.val = ctxt->src.val;
5707 else if (ctxt->op_bytes != 4)
5708 ctxt->dst.type = OP_NONE; /* no writeback */
5710 case 0x80 ... 0x8f: /* jnz rel, etc*/
5711 if (test_cc(ctxt->b, ctxt->eflags))
5712 rc = jmp_rel(ctxt, ctxt->src.val);
5714 case 0x90 ... 0x9f: /* setcc r/m8 */
5715 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5717 case 0xb6 ... 0xb7: /* movzx */
5718 ctxt->dst.bytes = ctxt->op_bytes;
5719 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5720 : (u16) ctxt->src.val;
5722 case 0xbe ... 0xbf: /* movsx */
5723 ctxt->dst.bytes = ctxt->op_bytes;
5724 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5725 (s16) ctxt->src.val;
5728 goto cannot_emulate;
5733 if (rc != X86EMUL_CONTINUE)
5739 return EMULATION_FAILED;
5742 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5744 invalidate_registers(ctxt);
5747 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5749 writeback_registers(ctxt);