1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include <linux/nospec.h>
25 #include "kvm_cache_regs.h"
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
28 #include <asm/debugreg.h>
29 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
164 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
165 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
166 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
167 #define NoWrite ((u64)1 << 45) /* No writeback */
168 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
169 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
170 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
171 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
172 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
173 #define NearBranch ((u64)1 << 52) /* Near branches */
174 #define No16 ((u64)1 << 53) /* No 16 bit operand */
175 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
176 #define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
178 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
180 #define X2(x...) x, x
181 #define X3(x...) X2(x), x
182 #define X4(x...) X2(x), X2(x)
183 #define X5(x...) X4(x), x
184 #define X6(x...) X4(x), X2(x)
185 #define X7(x...) X4(x), X3(x)
186 #define X8(x...) X4(x), X4(x)
187 #define X16(x...) X8(x), X8(x)
189 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
190 #define FASTOP_SIZE 8
193 * fastop functions have a special calling convention:
198 * flags: rflags (in/out)
199 * ex: rsi (in:fastop pointer, out:zero if exception)
201 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
202 * different operand sizes can be reached by calculation, rather than a jump
203 * table (which would be bigger than the code).
205 * fastop functions are declared as taking a never-defined fastop parameter,
206 * so they can't be called from C directly.
215 int (*execute)(struct x86_emulate_ctxt *ctxt);
216 const struct opcode *group;
217 const struct group_dual *gdual;
218 const struct gprefix *gprefix;
219 const struct escape *esc;
220 const struct instr_dual *idual;
221 const struct mode_dual *mdual;
222 void (*fastop)(struct fastop *fake);
224 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
228 struct opcode mod012[8];
229 struct opcode mod3[8];
233 struct opcode pfx_no;
234 struct opcode pfx_66;
235 struct opcode pfx_f2;
236 struct opcode pfx_f3;
241 struct opcode high[64];
245 struct opcode mod012;
250 struct opcode mode32;
251 struct opcode mode64;
254 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
256 enum x86_transfer_type {
258 X86_TRANSFER_CALL_JMP,
260 X86_TRANSFER_TASK_SWITCH,
263 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 if (!(ctxt->regs_valid & (1 << nr))) {
266 ctxt->regs_valid |= 1 << nr;
267 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
269 return ctxt->_regs[nr];
272 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
274 ctxt->regs_valid |= 1 << nr;
275 ctxt->regs_dirty |= 1 << nr;
276 return &ctxt->_regs[nr];
279 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 return reg_write(ctxt, nr);
285 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
289 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
290 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
293 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
295 ctxt->regs_dirty = 0;
296 ctxt->regs_valid = 0;
300 * These EFLAGS bits are restored from saved value during emulation, and
301 * any changes are written back to the saved value after emulation.
303 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
304 X86_EFLAGS_PF|X86_EFLAGS_CF)
312 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
314 #define FOP_FUNC(name) \
315 ".align " __stringify(FASTOP_SIZE) " \n\t" \
316 ".type " name ", @function \n\t" \
319 #define FOP_RET "ret \n\t"
321 #define FOP_START(op) \
322 extern void em_##op(struct fastop *fake); \
323 asm(".pushsection .text, \"ax\" \n\t" \
324 ".global em_" #op " \n\t" \
331 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
334 #define FOP1E(op, dst) \
335 FOP_FUNC(#op "_" #dst) \
336 "10: " #op " %" #dst " \n\t" FOP_RET
338 #define FOP1EEX(op, dst) \
339 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
341 #define FASTOP1(op) \
346 ON64(FOP1E(op##q, rax)) \
349 /* 1-operand, using src2 (for MUL/DIV r/m) */
350 #define FASTOP1SRC2(op, name) \
355 ON64(FOP1E(op, rcx)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
359 #define FASTOP1SRC2EX(op, name) \
364 ON64(FOP1EEX(op, rcx)) \
367 #define FOP2E(op, dst, src) \
368 FOP_FUNC(#op "_" #dst "_" #src) \
369 #op " %" #src ", %" #dst " \n\t" FOP_RET
371 #define FASTOP2(op) \
373 FOP2E(op##b, al, dl) \
374 FOP2E(op##w, ax, dx) \
375 FOP2E(op##l, eax, edx) \
376 ON64(FOP2E(op##q, rax, rdx)) \
379 /* 2 operand, word only */
380 #define FASTOP2W(op) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, src is CL */
389 #define FASTOP2CL(op) \
391 FOP2E(op##b, al, cl) \
392 FOP2E(op##w, ax, cl) \
393 FOP2E(op##l, eax, cl) \
394 ON64(FOP2E(op##q, rax, cl)) \
397 /* 2 operand, src and dest are reversed */
398 #define FASTOP2R(op, name) \
400 FOP2E(op##b, dl, al) \
401 FOP2E(op##w, dx, ax) \
402 FOP2E(op##l, edx, eax) \
403 ON64(FOP2E(op##q, rdx, rax)) \
406 #define FOP3E(op, dst, src, src2) \
407 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
408 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
410 /* 3-operand, word-only, src2=cl */
411 #define FASTOP3WCL(op) \
414 FOP3E(op##w, ax, dx, cl) \
415 FOP3E(op##l, eax, edx, cl) \
416 ON64(FOP3E(op##q, rax, rdx, cl)) \
419 /* Special case for SETcc - 1 instruction per cc */
420 #define FOP_SETCC(op) \
422 ".type " #op ", @function \n\t" \
427 asm(".global kvm_fastop_exception \n"
428 "kvm_fastop_exception: xor %esi, %esi; ret");
449 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
453 * XXX: inoutclob user must know where the argument is being expanded.
454 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
456 #define asm_safe(insn, inoutclob...) \
460 asm volatile("1:" insn "\n" \
462 ".pushsection .fixup, \"ax\"\n" \
463 "3: movl $1, %[_fault]\n" \
466 _ASM_EXTABLE(1b, 3b) \
467 : [_fault] "+qm"(_fault) inoutclob ); \
469 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
472 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
473 enum x86_intercept intercept,
474 enum x86_intercept_stage stage)
476 struct x86_instruction_info info = {
477 .intercept = intercept,
478 .rep_prefix = ctxt->rep_prefix,
479 .modrm_mod = ctxt->modrm_mod,
480 .modrm_reg = ctxt->modrm_reg,
481 .modrm_rm = ctxt->modrm_rm,
482 .src_val = ctxt->src.val64,
483 .dst_val = ctxt->dst.val64,
484 .src_bytes = ctxt->src.bytes,
485 .dst_bytes = ctxt->dst.bytes,
486 .ad_bytes = ctxt->ad_bytes,
487 .next_rip = ctxt->eip,
490 return ctxt->ops->intercept(ctxt, &info, stage);
493 static void assign_masked(ulong *dest, ulong src, ulong mask)
495 *dest = (*dest & ~mask) | (src & mask);
498 static void assign_register(unsigned long *reg, u64 val, int bytes)
500 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
503 *(u8 *)reg = (u8)val;
506 *(u16 *)reg = (u16)val;
510 break; /* 64b: zero-extend */
517 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
519 return (1UL << (ctxt->ad_bytes << 3)) - 1;
522 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
525 struct desc_struct ss;
527 if (ctxt->mode == X86EMUL_MODE_PROT64)
529 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
530 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
533 static int stack_size(struct x86_emulate_ctxt *ctxt)
535 return (__fls(stack_mask(ctxt)) + 1) >> 3;
538 /* Access/update address held in a register, based on addressing mode. */
539 static inline unsigned long
540 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
542 if (ctxt->ad_bytes == sizeof(unsigned long))
545 return reg & ad_mask(ctxt);
548 static inline unsigned long
549 register_address(struct x86_emulate_ctxt *ctxt, int reg)
551 return address_mask(ctxt, reg_read(ctxt, reg));
554 static void masked_increment(ulong *reg, ulong mask, int inc)
556 assign_masked(reg, *reg + inc, mask);
560 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
562 ulong *preg = reg_rmw(ctxt, reg);
564 assign_register(preg, *preg + inc, ctxt->ad_bytes);
567 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
569 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
572 static u32 desc_limit_scaled(struct desc_struct *desc)
574 u32 limit = get_desc_limit(desc);
576 return desc->g ? (limit << 12) | 0xfff : limit;
579 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
581 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
584 return ctxt->ops->get_cached_segment_base(ctxt, seg);
587 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
588 u32 error, bool valid)
591 ctxt->exception.vector = vec;
592 ctxt->exception.error_code = error;
593 ctxt->exception.error_code_valid = valid;
594 return X86EMUL_PROPAGATE_FAULT;
597 static int emulate_db(struct x86_emulate_ctxt *ctxt)
599 return emulate_exception(ctxt, DB_VECTOR, 0, false);
602 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
604 return emulate_exception(ctxt, GP_VECTOR, err, true);
607 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
609 return emulate_exception(ctxt, SS_VECTOR, err, true);
612 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
614 return emulate_exception(ctxt, UD_VECTOR, 0, false);
617 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
619 return emulate_exception(ctxt, TS_VECTOR, err, true);
622 static int emulate_de(struct x86_emulate_ctxt *ctxt)
624 return emulate_exception(ctxt, DE_VECTOR, 0, false);
627 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
629 return emulate_exception(ctxt, NM_VECTOR, 0, false);
632 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
635 struct desc_struct desc;
637 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
641 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
646 struct desc_struct desc;
648 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
649 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
653 * x86 defines three classes of vector instructions: explicitly
654 * aligned, explicitly unaligned, and the rest, which change behaviour
655 * depending on whether they're AVX encoded or not.
657 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
658 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
659 * 512 bytes of data must be aligned to a 16 byte boundary.
661 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
663 if (likely(size < 16))
666 if (ctxt->d & Aligned)
668 else if (ctxt->d & Unaligned)
670 else if (ctxt->d & Avx)
672 else if (ctxt->d & Aligned16)
678 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
679 struct segmented_address addr,
680 unsigned *max_size, unsigned size,
681 bool write, bool fetch,
682 enum x86emul_mode mode, ulong *linear)
684 struct desc_struct desc;
690 la = seg_base(ctxt, addr.seg) + addr.ea;
693 case X86EMUL_MODE_PROT64:
695 if (is_noncanonical_address(la))
698 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
699 if (size > *max_size)
703 *linear = la = (u32)la;
704 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
708 /* code segment in protected mode or read-only data segment */
709 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
710 || !(desc.type & 2)) && write)
712 /* unreadable code segment */
713 if (!fetch && (desc.type & 8) && !(desc.type & 2))
715 lim = desc_limit_scaled(&desc);
716 if (!(desc.type & 8) && (desc.type & 4)) {
717 /* expand-down segment */
720 lim = desc.d ? 0xffffffff : 0xffff;
724 if (lim == 0xffffffff)
727 *max_size = (u64)lim + 1 - addr.ea;
728 if (size > *max_size)
733 if (la & (insn_alignment(ctxt, size) - 1))
734 return emulate_gp(ctxt, 0);
735 return X86EMUL_CONTINUE;
737 if (addr.seg == VCPU_SREG_SS)
738 return emulate_ss(ctxt, 0);
740 return emulate_gp(ctxt, 0);
743 static int linearize(struct x86_emulate_ctxt *ctxt,
744 struct segmented_address addr,
745 unsigned size, bool write,
749 return __linearize(ctxt, addr, &max_size, size, write, false,
753 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
754 enum x86emul_mode mode)
759 struct segmented_address addr = { .seg = VCPU_SREG_CS,
762 if (ctxt->op_bytes != sizeof(unsigned long))
763 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
764 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
765 if (rc == X86EMUL_CONTINUE)
766 ctxt->_eip = addr.ea;
770 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
772 return assign_eip(ctxt, dst, ctxt->mode);
775 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
776 const struct desc_struct *cs_desc)
778 enum x86emul_mode mode = ctxt->mode;
782 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
786 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
788 mode = X86EMUL_MODE_PROT64;
790 mode = X86EMUL_MODE_PROT32; /* temporary value */
793 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
794 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
795 rc = assign_eip(ctxt, dst, mode);
796 if (rc == X86EMUL_CONTINUE)
801 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
803 return assign_eip_near(ctxt, ctxt->_eip + rel);
806 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
807 void *data, unsigned size)
809 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
812 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
813 ulong linear, void *data,
816 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
819 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
820 struct segmented_address addr,
827 rc = linearize(ctxt, addr, size, false, &linear);
828 if (rc != X86EMUL_CONTINUE)
830 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
833 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
834 struct segmented_address addr,
841 rc = linearize(ctxt, addr, size, true, &linear);
842 if (rc != X86EMUL_CONTINUE)
844 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
848 * Prefetch the remaining bytes of the instruction without crossing page
849 * boundary if they are not in fetch_cache yet.
851 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
854 unsigned size, max_size;
855 unsigned long linear;
856 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
857 struct segmented_address addr = { .seg = VCPU_SREG_CS,
858 .ea = ctxt->eip + cur_size };
861 * We do not know exactly how many bytes will be needed, and
862 * __linearize is expensive, so fetch as much as possible. We
863 * just have to avoid going beyond the 15 byte limit, the end
864 * of the segment, or the end of the page.
866 * __linearize is called with size 0 so that it does not do any
867 * boundary check itself. Instead, we use max_size to check
870 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
872 if (unlikely(rc != X86EMUL_CONTINUE))
875 size = min_t(unsigned, 15UL ^ cur_size, max_size);
876 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
879 * One instruction can only straddle two pages,
880 * and one has been loaded at the beginning of
881 * x86_decode_insn. So, if not enough bytes
882 * still, we must have hit the 15-byte boundary.
884 if (unlikely(size < op_size))
885 return emulate_gp(ctxt, 0);
887 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
888 size, &ctxt->exception);
889 if (unlikely(rc != X86EMUL_CONTINUE))
891 ctxt->fetch.end += size;
892 return X86EMUL_CONTINUE;
895 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
898 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
900 if (unlikely(done_size < size))
901 return __do_insn_fetch_bytes(ctxt, size - done_size);
903 return X86EMUL_CONTINUE;
906 /* Fetch next part of the instruction being emulated. */
907 #define insn_fetch(_type, _ctxt) \
910 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
911 if (rc != X86EMUL_CONTINUE) \
913 ctxt->_eip += sizeof(_type); \
914 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
915 ctxt->fetch.ptr += sizeof(_type); \
919 #define insn_fetch_arr(_arr, _size, _ctxt) \
921 rc = do_insn_fetch_bytes(_ctxt, _size); \
922 if (rc != X86EMUL_CONTINUE) \
924 ctxt->_eip += (_size); \
925 memcpy(_arr, ctxt->fetch.ptr, _size); \
926 ctxt->fetch.ptr += (_size); \
930 * Given the 'reg' portion of a ModRM byte, and a register block, return a
931 * pointer into the block that addresses the relevant register.
932 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
934 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
938 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
940 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
941 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
943 p = reg_rmw(ctxt, modrm_reg);
947 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
948 struct segmented_address addr,
949 u16 *size, unsigned long *address, int op_bytes)
956 rc = segmented_read_std(ctxt, addr, size, 2);
957 if (rc != X86EMUL_CONTINUE)
960 rc = segmented_read_std(ctxt, addr, address, op_bytes);
974 FASTOP1SRC2(mul, mul_ex);
975 FASTOP1SRC2(imul, imul_ex);
976 FASTOP1SRC2EX(div, div_ex);
977 FASTOP1SRC2EX(idiv, idiv_ex);
1006 FASTOP2R(cmp, cmp_r);
1008 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1010 /* If src is zero, do not writeback, but update flags */
1011 if (ctxt->src.val == 0)
1012 ctxt->dst.type = OP_NONE;
1013 return fastop(ctxt, em_bsf);
1016 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1018 /* If src is zero, do not writeback, but update flags */
1019 if (ctxt->src.val == 0)
1020 ctxt->dst.type = OP_NONE;
1021 return fastop(ctxt, em_bsr);
1024 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1027 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1029 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1030 asm("push %[flags]; popf; " CALL_NOSPEC
1031 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1035 static void fetch_register_operand(struct operand *op)
1037 switch (op->bytes) {
1039 op->val = *(u8 *)op->addr.reg;
1042 op->val = *(u16 *)op->addr.reg;
1045 op->val = *(u32 *)op->addr.reg;
1048 op->val = *(u64 *)op->addr.reg;
1053 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1055 ctxt->ops->get_fpu(ctxt);
1057 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1058 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1059 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1060 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1061 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1062 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1063 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1064 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1065 #ifdef CONFIG_X86_64
1066 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1067 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1068 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1069 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1070 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1071 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1072 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1073 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1077 ctxt->ops->put_fpu(ctxt);
1080 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1083 ctxt->ops->get_fpu(ctxt);
1085 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1086 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1087 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1088 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1089 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1090 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1091 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1092 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1093 #ifdef CONFIG_X86_64
1094 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1095 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1096 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1097 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1098 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1099 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1100 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1101 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1105 ctxt->ops->put_fpu(ctxt);
1108 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1110 ctxt->ops->get_fpu(ctxt);
1112 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1113 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1114 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1115 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1116 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1117 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1118 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1119 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1122 ctxt->ops->put_fpu(ctxt);
1125 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1127 ctxt->ops->get_fpu(ctxt);
1129 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1130 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1131 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1132 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1133 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1134 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1135 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1136 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1139 ctxt->ops->put_fpu(ctxt);
1142 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1144 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1145 return emulate_nm(ctxt);
1147 ctxt->ops->get_fpu(ctxt);
1148 asm volatile("fninit");
1149 ctxt->ops->put_fpu(ctxt);
1150 return X86EMUL_CONTINUE;
1153 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1157 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1158 return emulate_nm(ctxt);
1160 ctxt->ops->get_fpu(ctxt);
1161 asm volatile("fnstcw %0": "+m"(fcw));
1162 ctxt->ops->put_fpu(ctxt);
1164 ctxt->dst.val = fcw;
1166 return X86EMUL_CONTINUE;
1169 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1173 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1174 return emulate_nm(ctxt);
1176 ctxt->ops->get_fpu(ctxt);
1177 asm volatile("fnstsw %0": "+m"(fsw));
1178 ctxt->ops->put_fpu(ctxt);
1180 ctxt->dst.val = fsw;
1182 return X86EMUL_CONTINUE;
1185 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1188 unsigned reg = ctxt->modrm_reg;
1190 if (!(ctxt->d & ModRM))
1191 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1193 if (ctxt->d & Sse) {
1197 read_sse_reg(ctxt, &op->vec_val, reg);
1200 if (ctxt->d & Mmx) {
1209 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1210 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1212 fetch_register_operand(op);
1213 op->orig_val = op->val;
1216 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1218 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1219 ctxt->modrm_seg = VCPU_SREG_SS;
1222 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1226 int index_reg, base_reg, scale;
1227 int rc = X86EMUL_CONTINUE;
1230 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1231 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1232 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1234 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1235 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1236 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1237 ctxt->modrm_seg = VCPU_SREG_DS;
1239 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1241 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1242 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1244 if (ctxt->d & Sse) {
1247 op->addr.xmm = ctxt->modrm_rm;
1248 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1251 if (ctxt->d & Mmx) {
1254 op->addr.mm = ctxt->modrm_rm & 7;
1257 fetch_register_operand(op);
1263 if (ctxt->ad_bytes == 2) {
1264 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1265 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1266 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1267 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1269 /* 16-bit ModR/M decode. */
1270 switch (ctxt->modrm_mod) {
1272 if (ctxt->modrm_rm == 6)
1273 modrm_ea += insn_fetch(u16, ctxt);
1276 modrm_ea += insn_fetch(s8, ctxt);
1279 modrm_ea += insn_fetch(u16, ctxt);
1282 switch (ctxt->modrm_rm) {
1284 modrm_ea += bx + si;
1287 modrm_ea += bx + di;
1290 modrm_ea += bp + si;
1293 modrm_ea += bp + di;
1302 if (ctxt->modrm_mod != 0)
1309 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1310 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1311 ctxt->modrm_seg = VCPU_SREG_SS;
1312 modrm_ea = (u16)modrm_ea;
1314 /* 32/64-bit ModR/M decode. */
1315 if ((ctxt->modrm_rm & 7) == 4) {
1316 sib = insn_fetch(u8, ctxt);
1317 index_reg |= (sib >> 3) & 7;
1318 base_reg |= sib & 7;
1321 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1322 modrm_ea += insn_fetch(s32, ctxt);
1324 modrm_ea += reg_read(ctxt, base_reg);
1325 adjust_modrm_seg(ctxt, base_reg);
1326 /* Increment ESP on POP [ESP] */
1327 if ((ctxt->d & IncSP) &&
1328 base_reg == VCPU_REGS_RSP)
1329 modrm_ea += ctxt->op_bytes;
1332 modrm_ea += reg_read(ctxt, index_reg) << scale;
1333 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1334 modrm_ea += insn_fetch(s32, ctxt);
1335 if (ctxt->mode == X86EMUL_MODE_PROT64)
1336 ctxt->rip_relative = 1;
1338 base_reg = ctxt->modrm_rm;
1339 modrm_ea += reg_read(ctxt, base_reg);
1340 adjust_modrm_seg(ctxt, base_reg);
1342 switch (ctxt->modrm_mod) {
1344 modrm_ea += insn_fetch(s8, ctxt);
1347 modrm_ea += insn_fetch(s32, ctxt);
1351 op->addr.mem.ea = modrm_ea;
1352 if (ctxt->ad_bytes != 8)
1353 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1359 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1362 int rc = X86EMUL_CONTINUE;
1365 switch (ctxt->ad_bytes) {
1367 op->addr.mem.ea = insn_fetch(u16, ctxt);
1370 op->addr.mem.ea = insn_fetch(u32, ctxt);
1373 op->addr.mem.ea = insn_fetch(u64, ctxt);
1380 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1384 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1385 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1387 if (ctxt->src.bytes == 2)
1388 sv = (s16)ctxt->src.val & (s16)mask;
1389 else if (ctxt->src.bytes == 4)
1390 sv = (s32)ctxt->src.val & (s32)mask;
1392 sv = (s64)ctxt->src.val & (s64)mask;
1394 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1395 ctxt->dst.addr.mem.ea + (sv >> 3));
1398 /* only subword offset */
1399 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1402 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1403 unsigned long addr, void *dest, unsigned size)
1406 struct read_cache *mc = &ctxt->mem_read;
1408 if (mc->pos < mc->end)
1411 WARN_ON((mc->end + size) >= sizeof(mc->data));
1413 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1415 if (rc != X86EMUL_CONTINUE)
1421 memcpy(dest, mc->data + mc->pos, size);
1423 return X86EMUL_CONTINUE;
1426 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1427 struct segmented_address addr,
1434 rc = linearize(ctxt, addr, size, false, &linear);
1435 if (rc != X86EMUL_CONTINUE)
1437 return read_emulated(ctxt, linear, data, size);
1440 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1441 struct segmented_address addr,
1448 rc = linearize(ctxt, addr, size, true, &linear);
1449 if (rc != X86EMUL_CONTINUE)
1451 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1455 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1456 struct segmented_address addr,
1457 const void *orig_data, const void *data,
1463 rc = linearize(ctxt, addr, size, true, &linear);
1464 if (rc != X86EMUL_CONTINUE)
1466 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1467 size, &ctxt->exception);
1470 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1471 unsigned int size, unsigned short port,
1474 struct read_cache *rc = &ctxt->io_read;
1476 if (rc->pos == rc->end) { /* refill pio read ahead */
1477 unsigned int in_page, n;
1478 unsigned int count = ctxt->rep_prefix ?
1479 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1480 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1481 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1482 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1483 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1486 rc->pos = rc->end = 0;
1487 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1492 if (ctxt->rep_prefix && (ctxt->d & String) &&
1493 !(ctxt->eflags & X86_EFLAGS_DF)) {
1494 ctxt->dst.data = rc->data + rc->pos;
1495 ctxt->dst.type = OP_MEM_STR;
1496 ctxt->dst.count = (rc->end - rc->pos) / size;
1499 memcpy(dest, rc->data + rc->pos, size);
1505 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1506 u16 index, struct desc_struct *desc)
1511 ctxt->ops->get_idt(ctxt, &dt);
1513 if (dt.size < index * 8 + 7)
1514 return emulate_gp(ctxt, index << 3 | 0x2);
1516 addr = dt.address + index * 8;
1517 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1520 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1521 u16 selector, struct desc_ptr *dt)
1523 const struct x86_emulate_ops *ops = ctxt->ops;
1526 if (selector & 1 << 2) {
1527 struct desc_struct desc;
1530 memset (dt, 0, sizeof *dt);
1531 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1535 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1536 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1538 ops->get_gdt(ctxt, dt);
1541 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1542 u16 selector, ulong *desc_addr_p)
1545 u16 index = selector >> 3;
1548 get_descriptor_table_ptr(ctxt, selector, &dt);
1550 if (dt.size < index * 8 + 7)
1551 return emulate_gp(ctxt, selector & 0xfffc);
1553 addr = dt.address + index * 8;
1555 #ifdef CONFIG_X86_64
1556 if (addr >> 32 != 0) {
1559 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1560 if (!(efer & EFER_LMA))
1565 *desc_addr_p = addr;
1566 return X86EMUL_CONTINUE;
1569 /* allowed just for 8 bytes segments */
1570 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1571 u16 selector, struct desc_struct *desc,
1576 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1577 if (rc != X86EMUL_CONTINUE)
1580 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1583 /* allowed just for 8 bytes segments */
1584 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1585 u16 selector, struct desc_struct *desc)
1590 rc = get_descriptor_ptr(ctxt, selector, &addr);
1591 if (rc != X86EMUL_CONTINUE)
1594 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1597 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1598 u16 selector, int seg, u8 cpl,
1599 enum x86_transfer_type transfer,
1600 struct desc_struct *desc)
1602 struct desc_struct seg_desc, old_desc;
1604 unsigned err_vec = GP_VECTOR;
1606 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1612 memset(&seg_desc, 0, sizeof seg_desc);
1614 if (ctxt->mode == X86EMUL_MODE_REAL) {
1615 /* set real mode segment descriptor (keep limit etc. for
1617 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1618 set_desc_base(&seg_desc, selector << 4);
1620 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1621 /* VM86 needs a clean new segment descriptor */
1622 set_desc_base(&seg_desc, selector << 4);
1623 set_desc_limit(&seg_desc, 0xffff);
1633 /* TR should be in GDT only */
1634 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1637 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1638 if (null_selector) {
1639 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1642 if (seg == VCPU_SREG_SS) {
1643 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1647 * ctxt->ops->set_segment expects the CPL to be in
1648 * SS.DPL, so fake an expand-up 32-bit data segment.
1658 /* Skip all following checks */
1662 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1663 if (ret != X86EMUL_CONTINUE)
1666 err_code = selector & 0xfffc;
1667 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1670 /* can't load system descriptor into segment selector */
1671 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1672 if (transfer == X86_TRANSFER_CALL_JMP)
1673 return X86EMUL_UNHANDLEABLE;
1682 * segment is not a writable data segment or segment
1683 * selector's RPL != CPL or segment selector's RPL != CPL
1685 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1689 if (!(seg_desc.type & 8))
1692 if (seg_desc.type & 4) {
1698 if (rpl > cpl || dpl != cpl)
1701 /* in long-mode d/b must be clear if l is set */
1702 if (seg_desc.d && seg_desc.l) {
1705 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1706 if (efer & EFER_LMA)
1710 /* CS(RPL) <- CPL */
1711 selector = (selector & 0xfffc) | cpl;
1714 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1717 case VCPU_SREG_LDTR:
1718 if (seg_desc.s || seg_desc.type != 2)
1721 default: /* DS, ES, FS, or GS */
1723 * segment is not a data or readable code segment or
1724 * ((segment is a data or nonconforming code segment)
1725 * and (both RPL and CPL > DPL))
1727 if ((seg_desc.type & 0xa) == 0x8 ||
1728 (((seg_desc.type & 0xc) != 0xc) &&
1729 (rpl > dpl && cpl > dpl)))
1735 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1740 /* mark segment as accessed */
1741 if (!(seg_desc.type & 1)) {
1743 ret = write_segment_descriptor(ctxt, selector,
1745 if (ret != X86EMUL_CONTINUE)
1748 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1749 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1750 if (ret != X86EMUL_CONTINUE)
1752 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1753 ((u64)base3 << 32)))
1754 return emulate_gp(ctxt, 0);
1757 if (seg == VCPU_SREG_TR) {
1758 old_desc = seg_desc;
1759 seg_desc.type |= 2; /* busy */
1760 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1761 sizeof(seg_desc), &ctxt->exception);
1762 if (ret != X86EMUL_CONTINUE)
1766 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1769 return X86EMUL_CONTINUE;
1771 return emulate_exception(ctxt, err_vec, err_code, true);
1774 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1775 u16 selector, int seg)
1777 u8 cpl = ctxt->ops->cpl(ctxt);
1780 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1781 * they can load it at CPL<3 (Intel's manual says only LSS can,
1784 * However, the Intel manual says that putting IST=1/DPL=3 in
1785 * an interrupt gate will result in SS=3 (the AMD manual instead
1786 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1787 * and only forbid it here.
1789 if (seg == VCPU_SREG_SS && selector == 3 &&
1790 ctxt->mode == X86EMUL_MODE_PROT64)
1791 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1793 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1794 X86_TRANSFER_NONE, NULL);
1797 static void write_register_operand(struct operand *op)
1799 return assign_register(op->addr.reg, op->val, op->bytes);
1802 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1806 write_register_operand(op);
1809 if (ctxt->lock_prefix)
1810 return segmented_cmpxchg(ctxt,
1816 return segmented_write(ctxt,
1822 return segmented_write(ctxt,
1825 op->bytes * op->count);
1828 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1831 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1839 return X86EMUL_CONTINUE;
1842 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1844 struct segmented_address addr;
1846 rsp_increment(ctxt, -bytes);
1847 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1848 addr.seg = VCPU_SREG_SS;
1850 return segmented_write(ctxt, addr, data, bytes);
1853 static int em_push(struct x86_emulate_ctxt *ctxt)
1855 /* Disable writeback. */
1856 ctxt->dst.type = OP_NONE;
1857 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1860 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1861 void *dest, int len)
1864 struct segmented_address addr;
1866 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1867 addr.seg = VCPU_SREG_SS;
1868 rc = segmented_read(ctxt, addr, dest, len);
1869 if (rc != X86EMUL_CONTINUE)
1872 rsp_increment(ctxt, len);
1876 static int em_pop(struct x86_emulate_ctxt *ctxt)
1878 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1881 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1882 void *dest, int len)
1885 unsigned long val, change_mask;
1886 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1887 int cpl = ctxt->ops->cpl(ctxt);
1889 rc = emulate_pop(ctxt, &val, len);
1890 if (rc != X86EMUL_CONTINUE)
1893 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1894 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1895 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1896 X86_EFLAGS_AC | X86_EFLAGS_ID;
1898 switch(ctxt->mode) {
1899 case X86EMUL_MODE_PROT64:
1900 case X86EMUL_MODE_PROT32:
1901 case X86EMUL_MODE_PROT16:
1903 change_mask |= X86_EFLAGS_IOPL;
1905 change_mask |= X86_EFLAGS_IF;
1907 case X86EMUL_MODE_VM86:
1909 return emulate_gp(ctxt, 0);
1910 change_mask |= X86_EFLAGS_IF;
1912 default: /* real mode */
1913 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1917 *(unsigned long *)dest =
1918 (ctxt->eflags & ~change_mask) | (val & change_mask);
1923 static int em_popf(struct x86_emulate_ctxt *ctxt)
1925 ctxt->dst.type = OP_REG;
1926 ctxt->dst.addr.reg = &ctxt->eflags;
1927 ctxt->dst.bytes = ctxt->op_bytes;
1928 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1931 static int em_enter(struct x86_emulate_ctxt *ctxt)
1934 unsigned frame_size = ctxt->src.val;
1935 unsigned nesting_level = ctxt->src2.val & 31;
1939 return X86EMUL_UNHANDLEABLE;
1941 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1942 rc = push(ctxt, &rbp, stack_size(ctxt));
1943 if (rc != X86EMUL_CONTINUE)
1945 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1947 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1948 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1950 return X86EMUL_CONTINUE;
1953 static int em_leave(struct x86_emulate_ctxt *ctxt)
1955 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1957 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1960 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1962 int seg = ctxt->src2.val;
1964 ctxt->src.val = get_segment_selector(ctxt, seg);
1965 if (ctxt->op_bytes == 4) {
1966 rsp_increment(ctxt, -2);
1970 return em_push(ctxt);
1973 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1975 int seg = ctxt->src2.val;
1976 unsigned long selector;
1979 rc = emulate_pop(ctxt, &selector, 2);
1980 if (rc != X86EMUL_CONTINUE)
1983 if (seg == VCPU_SREG_SS)
1984 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1985 if (ctxt->op_bytes > 2)
1986 rsp_increment(ctxt, ctxt->op_bytes - 2);
1988 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1992 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1994 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1995 int rc = X86EMUL_CONTINUE;
1996 int reg = VCPU_REGS_RAX;
1998 while (reg <= VCPU_REGS_RDI) {
1999 (reg == VCPU_REGS_RSP) ?
2000 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2003 if (rc != X86EMUL_CONTINUE)
2012 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2014 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2015 return em_push(ctxt);
2018 static int em_popa(struct x86_emulate_ctxt *ctxt)
2020 int rc = X86EMUL_CONTINUE;
2021 int reg = VCPU_REGS_RDI;
2024 while (reg >= VCPU_REGS_RAX) {
2025 if (reg == VCPU_REGS_RSP) {
2026 rsp_increment(ctxt, ctxt->op_bytes);
2030 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2031 if (rc != X86EMUL_CONTINUE)
2033 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2039 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2041 const struct x86_emulate_ops *ops = ctxt->ops;
2048 /* TODO: Add limit checks */
2049 ctxt->src.val = ctxt->eflags;
2051 if (rc != X86EMUL_CONTINUE)
2054 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2056 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2058 if (rc != X86EMUL_CONTINUE)
2061 ctxt->src.val = ctxt->_eip;
2063 if (rc != X86EMUL_CONTINUE)
2066 ops->get_idt(ctxt, &dt);
2068 eip_addr = dt.address + (irq << 2);
2069 cs_addr = dt.address + (irq << 2) + 2;
2071 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2072 if (rc != X86EMUL_CONTINUE)
2075 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2076 if (rc != X86EMUL_CONTINUE)
2079 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2080 if (rc != X86EMUL_CONTINUE)
2088 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2092 invalidate_registers(ctxt);
2093 rc = __emulate_int_real(ctxt, irq);
2094 if (rc == X86EMUL_CONTINUE)
2095 writeback_registers(ctxt);
2099 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2101 switch(ctxt->mode) {
2102 case X86EMUL_MODE_REAL:
2103 return __emulate_int_real(ctxt, irq);
2104 case X86EMUL_MODE_VM86:
2105 case X86EMUL_MODE_PROT16:
2106 case X86EMUL_MODE_PROT32:
2107 case X86EMUL_MODE_PROT64:
2109 /* Protected mode interrupts unimplemented yet */
2110 return X86EMUL_UNHANDLEABLE;
2114 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2116 int rc = X86EMUL_CONTINUE;
2117 unsigned long temp_eip = 0;
2118 unsigned long temp_eflags = 0;
2119 unsigned long cs = 0;
2120 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2121 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2122 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2123 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2124 X86_EFLAGS_AC | X86_EFLAGS_ID |
2126 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2129 /* TODO: Add stack limit check */
2131 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2133 if (rc != X86EMUL_CONTINUE)
2136 if (temp_eip & ~0xffff)
2137 return emulate_gp(ctxt, 0);
2139 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2141 if (rc != X86EMUL_CONTINUE)
2144 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2146 if (rc != X86EMUL_CONTINUE)
2149 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2151 if (rc != X86EMUL_CONTINUE)
2154 ctxt->_eip = temp_eip;
2156 if (ctxt->op_bytes == 4)
2157 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2158 else if (ctxt->op_bytes == 2) {
2159 ctxt->eflags &= ~0xffff;
2160 ctxt->eflags |= temp_eflags;
2163 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2164 ctxt->eflags |= X86_EFLAGS_FIXED;
2165 ctxt->ops->set_nmi_mask(ctxt, false);
2170 static int em_iret(struct x86_emulate_ctxt *ctxt)
2172 switch(ctxt->mode) {
2173 case X86EMUL_MODE_REAL:
2174 return emulate_iret_real(ctxt);
2175 case X86EMUL_MODE_VM86:
2176 case X86EMUL_MODE_PROT16:
2177 case X86EMUL_MODE_PROT32:
2178 case X86EMUL_MODE_PROT64:
2180 /* iret from protected mode unimplemented yet */
2181 return X86EMUL_UNHANDLEABLE;
2185 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2189 struct desc_struct new_desc;
2190 u8 cpl = ctxt->ops->cpl(ctxt);
2192 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2194 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2195 X86_TRANSFER_CALL_JMP,
2197 if (rc != X86EMUL_CONTINUE)
2200 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2201 /* Error handling is not implemented. */
2202 if (rc != X86EMUL_CONTINUE)
2203 return X86EMUL_UNHANDLEABLE;
2208 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2210 return assign_eip_near(ctxt, ctxt->src.val);
2213 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2218 old_eip = ctxt->_eip;
2219 rc = assign_eip_near(ctxt, ctxt->src.val);
2220 if (rc != X86EMUL_CONTINUE)
2222 ctxt->src.val = old_eip;
2227 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2229 u64 old = ctxt->dst.orig_val64;
2231 if (ctxt->dst.bytes == 16)
2232 return X86EMUL_UNHANDLEABLE;
2234 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2235 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2236 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2237 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2238 ctxt->eflags &= ~X86_EFLAGS_ZF;
2240 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2241 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2243 ctxt->eflags |= X86_EFLAGS_ZF;
2245 return X86EMUL_CONTINUE;
2248 static int em_ret(struct x86_emulate_ctxt *ctxt)
2253 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2254 if (rc != X86EMUL_CONTINUE)
2257 return assign_eip_near(ctxt, eip);
2260 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2263 unsigned long eip, cs;
2264 int cpl = ctxt->ops->cpl(ctxt);
2265 struct desc_struct new_desc;
2267 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2268 if (rc != X86EMUL_CONTINUE)
2270 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2271 if (rc != X86EMUL_CONTINUE)
2273 /* Outer-privilege level return is not implemented */
2274 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2275 return X86EMUL_UNHANDLEABLE;
2276 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2279 if (rc != X86EMUL_CONTINUE)
2281 rc = assign_eip_far(ctxt, eip, &new_desc);
2282 /* Error handling is not implemented. */
2283 if (rc != X86EMUL_CONTINUE)
2284 return X86EMUL_UNHANDLEABLE;
2289 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2293 rc = em_ret_far(ctxt);
2294 if (rc != X86EMUL_CONTINUE)
2296 rsp_increment(ctxt, ctxt->src.val);
2297 return X86EMUL_CONTINUE;
2300 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2302 /* Save real source value, then compare EAX against destination. */
2303 ctxt->dst.orig_val = ctxt->dst.val;
2304 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2305 ctxt->src.orig_val = ctxt->src.val;
2306 ctxt->src.val = ctxt->dst.orig_val;
2307 fastop(ctxt, em_cmp);
2309 if (ctxt->eflags & X86_EFLAGS_ZF) {
2310 /* Success: write back to memory; no update of EAX */
2311 ctxt->src.type = OP_NONE;
2312 ctxt->dst.val = ctxt->src.orig_val;
2314 /* Failure: write the value we saw to EAX. */
2315 ctxt->src.type = OP_REG;
2316 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2317 ctxt->src.val = ctxt->dst.orig_val;
2318 /* Create write-cycle to dest by writing the same value */
2319 ctxt->dst.val = ctxt->dst.orig_val;
2321 return X86EMUL_CONTINUE;
2324 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2326 int seg = ctxt->src2.val;
2330 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2332 rc = load_segment_descriptor(ctxt, sel, seg);
2333 if (rc != X86EMUL_CONTINUE)
2336 ctxt->dst.val = ctxt->src.val;
2340 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2342 u32 eax, ebx, ecx, edx;
2346 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2347 return edx & bit(X86_FEATURE_LM);
2350 #define GET_SMSTATE(type, smbase, offset) \
2353 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2355 if (r != X86EMUL_CONTINUE) \
2356 return X86EMUL_UNHANDLEABLE; \
2360 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2362 desc->g = (flags >> 23) & 1;
2363 desc->d = (flags >> 22) & 1;
2364 desc->l = (flags >> 21) & 1;
2365 desc->avl = (flags >> 20) & 1;
2366 desc->p = (flags >> 15) & 1;
2367 desc->dpl = (flags >> 13) & 3;
2368 desc->s = (flags >> 12) & 1;
2369 desc->type = (flags >> 8) & 15;
2372 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2374 struct desc_struct desc;
2378 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2381 offset = 0x7f84 + n * 12;
2383 offset = 0x7f2c + (n - 3) * 12;
2385 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2386 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2387 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2388 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2389 return X86EMUL_CONTINUE;
2392 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2394 struct desc_struct desc;
2399 offset = 0x7e00 + n * 16;
2401 selector = GET_SMSTATE(u16, smbase, offset);
2402 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2403 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2404 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2405 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2407 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2408 return X86EMUL_CONTINUE;
2411 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2412 u64 cr0, u64 cr3, u64 cr4)
2417 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2419 if (cr4 & X86_CR4_PCIDE) {
2424 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2426 return X86EMUL_UNHANDLEABLE;
2429 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2430 * Then enable protected mode. However, PCID cannot be enabled
2431 * if EFER.LMA=0, so set it separately.
2433 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2435 return X86EMUL_UNHANDLEABLE;
2437 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2439 return X86EMUL_UNHANDLEABLE;
2441 if (cr4 & X86_CR4_PCIDE) {
2442 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2444 return X86EMUL_UNHANDLEABLE;
2446 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2448 return X86EMUL_UNHANDLEABLE;
2453 return X86EMUL_CONTINUE;
2456 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2458 struct desc_struct desc;
2461 u32 val, cr0, cr3, cr4;
2464 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2465 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2466 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2467 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2469 for (i = 0; i < 8; i++)
2470 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2472 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2473 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2474 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2475 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2477 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2478 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2479 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2480 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2481 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2483 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2484 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2485 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2486 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2487 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2489 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2490 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2491 ctxt->ops->set_gdt(ctxt, &dt);
2493 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2494 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2495 ctxt->ops->set_idt(ctxt, &dt);
2497 for (i = 0; i < 6; i++) {
2498 int r = rsm_load_seg_32(ctxt, smbase, i);
2499 if (r != X86EMUL_CONTINUE)
2503 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2505 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2507 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2510 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2512 struct desc_struct desc;
2514 u64 val, cr0, cr3, cr4;
2519 for (i = 0; i < 16; i++)
2520 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2522 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2523 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2525 val = GET_SMSTATE(u32, smbase, 0x7f68);
2526 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2527 val = GET_SMSTATE(u32, smbase, 0x7f60);
2528 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2530 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2531 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2532 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2533 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2534 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2535 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2537 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2538 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2539 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2540 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2541 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2542 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2544 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2545 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2546 ctxt->ops->set_idt(ctxt, &dt);
2548 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2549 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2550 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2551 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2552 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2553 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2555 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2556 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2557 ctxt->ops->set_gdt(ctxt, &dt);
2559 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2560 if (r != X86EMUL_CONTINUE)
2563 for (i = 0; i < 6; i++) {
2564 r = rsm_load_seg_64(ctxt, smbase, i);
2565 if (r != X86EMUL_CONTINUE)
2569 return X86EMUL_CONTINUE;
2572 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2574 unsigned long cr0, cr4, efer;
2578 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2579 return emulate_ud(ctxt);
2582 * Get back to real mode, to prepare a safe state in which to load
2583 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2584 * supports long mode.
2586 if (emulator_has_longmode(ctxt)) {
2587 struct desc_struct cs_desc;
2589 /* Zero CR4.PCIDE before CR0.PG. */
2590 cr4 = ctxt->ops->get_cr(ctxt, 4);
2591 if (cr4 & X86_CR4_PCIDE)
2592 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2594 /* A 32-bit code segment is required to clear EFER.LMA. */
2595 memset(&cs_desc, 0, sizeof(cs_desc));
2597 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2598 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2601 /* For the 64-bit case, this will clear EFER.LMA. */
2602 cr0 = ctxt->ops->get_cr(ctxt, 0);
2603 if (cr0 & X86_CR0_PE)
2604 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2606 if (emulator_has_longmode(ctxt)) {
2607 /* Clear CR4.PAE before clearing EFER.LME. */
2608 cr4 = ctxt->ops->get_cr(ctxt, 4);
2609 if (cr4 & X86_CR4_PAE)
2610 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2612 /* And finally go back to 32-bit mode. */
2614 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2617 smbase = ctxt->ops->get_smbase(ctxt);
2618 if (emulator_has_longmode(ctxt))
2619 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2621 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2623 if (ret != X86EMUL_CONTINUE) {
2624 /* FIXME: should triple fault */
2625 return X86EMUL_UNHANDLEABLE;
2628 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2629 ctxt->ops->set_nmi_mask(ctxt, false);
2631 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2632 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2633 return X86EMUL_CONTINUE;
2637 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2638 struct desc_struct *cs, struct desc_struct *ss)
2640 cs->l = 0; /* will be adjusted later */
2641 set_desc_base(cs, 0); /* flat segment */
2642 cs->g = 1; /* 4kb granularity */
2643 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2644 cs->type = 0x0b; /* Read, Execute, Accessed */
2646 cs->dpl = 0; /* will be adjusted later */
2651 set_desc_base(ss, 0); /* flat segment */
2652 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2653 ss->g = 1; /* 4kb granularity */
2655 ss->type = 0x03; /* Read/Write, Accessed */
2656 ss->d = 1; /* 32bit stack segment */
2663 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2665 u32 eax, ebx, ecx, edx;
2668 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2669 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2670 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2671 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2674 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2676 const struct x86_emulate_ops *ops = ctxt->ops;
2677 u32 eax, ebx, ecx, edx;
2680 * syscall should always be enabled in longmode - so only become
2681 * vendor specific (cpuid) if other modes are active...
2683 if (ctxt->mode == X86EMUL_MODE_PROT64)
2688 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2690 * Intel ("GenuineIntel")
2691 * remark: Intel CPUs only support "syscall" in 64bit
2692 * longmode. Also an 64bit guest with a
2693 * 32bit compat-app running will #UD !! While this
2694 * behaviour can be fixed (by emulating) into AMD
2695 * response - CPUs of AMD can't behave like Intel.
2697 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2698 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2699 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2702 /* AMD ("AuthenticAMD") */
2703 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2704 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2705 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2708 /* AMD ("AMDisbetter!") */
2709 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2710 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2711 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2714 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2718 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2720 const struct x86_emulate_ops *ops = ctxt->ops;
2721 struct desc_struct cs, ss;
2726 /* syscall is not available in real mode */
2727 if (ctxt->mode == X86EMUL_MODE_REAL ||
2728 ctxt->mode == X86EMUL_MODE_VM86)
2729 return emulate_ud(ctxt);
2731 if (!(em_syscall_is_enabled(ctxt)))
2732 return emulate_ud(ctxt);
2734 ops->get_msr(ctxt, MSR_EFER, &efer);
2735 setup_syscalls_segments(ctxt, &cs, &ss);
2737 if (!(efer & EFER_SCE))
2738 return emulate_ud(ctxt);
2740 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2742 cs_sel = (u16)(msr_data & 0xfffc);
2743 ss_sel = (u16)(msr_data + 8);
2745 if (efer & EFER_LMA) {
2749 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2750 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2752 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2753 if (efer & EFER_LMA) {
2754 #ifdef CONFIG_X86_64
2755 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2758 ctxt->mode == X86EMUL_MODE_PROT64 ?
2759 MSR_LSTAR : MSR_CSTAR, &msr_data);
2760 ctxt->_eip = msr_data;
2762 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2763 ctxt->eflags &= ~msr_data;
2764 ctxt->eflags |= X86_EFLAGS_FIXED;
2768 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2769 ctxt->_eip = (u32)msr_data;
2771 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2774 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2775 return X86EMUL_CONTINUE;
2778 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2780 const struct x86_emulate_ops *ops = ctxt->ops;
2781 struct desc_struct cs, ss;
2786 ops->get_msr(ctxt, MSR_EFER, &efer);
2787 /* inject #GP if in real mode */
2788 if (ctxt->mode == X86EMUL_MODE_REAL)
2789 return emulate_gp(ctxt, 0);
2792 * Not recognized on AMD in compat mode (but is recognized in legacy
2795 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2796 && !vendor_intel(ctxt))
2797 return emulate_ud(ctxt);
2799 /* sysenter/sysexit have not been tested in 64bit mode. */
2800 if (ctxt->mode == X86EMUL_MODE_PROT64)
2801 return X86EMUL_UNHANDLEABLE;
2803 setup_syscalls_segments(ctxt, &cs, &ss);
2805 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2806 if ((msr_data & 0xfffc) == 0x0)
2807 return emulate_gp(ctxt, 0);
2809 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2810 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2811 ss_sel = cs_sel + 8;
2812 if (efer & EFER_LMA) {
2817 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2818 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2820 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2821 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2823 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2824 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2827 return X86EMUL_CONTINUE;
2830 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2832 const struct x86_emulate_ops *ops = ctxt->ops;
2833 struct desc_struct cs, ss;
2834 u64 msr_data, rcx, rdx;
2836 u16 cs_sel = 0, ss_sel = 0;
2838 /* inject #GP if in real mode or Virtual 8086 mode */
2839 if (ctxt->mode == X86EMUL_MODE_REAL ||
2840 ctxt->mode == X86EMUL_MODE_VM86)
2841 return emulate_gp(ctxt, 0);
2843 setup_syscalls_segments(ctxt, &cs, &ss);
2845 if ((ctxt->rex_prefix & 0x8) != 0x0)
2846 usermode = X86EMUL_MODE_PROT64;
2848 usermode = X86EMUL_MODE_PROT32;
2850 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2851 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2855 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2857 case X86EMUL_MODE_PROT32:
2858 cs_sel = (u16)(msr_data + 16);
2859 if ((msr_data & 0xfffc) == 0x0)
2860 return emulate_gp(ctxt, 0);
2861 ss_sel = (u16)(msr_data + 24);
2865 case X86EMUL_MODE_PROT64:
2866 cs_sel = (u16)(msr_data + 32);
2867 if (msr_data == 0x0)
2868 return emulate_gp(ctxt, 0);
2869 ss_sel = cs_sel + 8;
2872 if (is_noncanonical_address(rcx) ||
2873 is_noncanonical_address(rdx))
2874 return emulate_gp(ctxt, 0);
2877 cs_sel |= SEGMENT_RPL_MASK;
2878 ss_sel |= SEGMENT_RPL_MASK;
2880 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2881 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2884 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2886 return X86EMUL_CONTINUE;
2889 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2892 if (ctxt->mode == X86EMUL_MODE_REAL)
2894 if (ctxt->mode == X86EMUL_MODE_VM86)
2896 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2897 return ctxt->ops->cpl(ctxt) > iopl;
2900 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2903 const struct x86_emulate_ops *ops = ctxt->ops;
2904 struct desc_struct tr_seg;
2907 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2908 unsigned mask = (1 << len) - 1;
2911 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2914 if (desc_limit_scaled(&tr_seg) < 103)
2916 base = get_desc_base(&tr_seg);
2917 #ifdef CONFIG_X86_64
2918 base |= ((u64)base3) << 32;
2920 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2921 if (r != X86EMUL_CONTINUE)
2923 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2925 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2926 if (r != X86EMUL_CONTINUE)
2928 if ((perm >> bit_idx) & mask)
2933 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2939 if (emulator_bad_iopl(ctxt))
2940 if (!emulator_io_port_access_allowed(ctxt, port, len))
2943 ctxt->perm_ok = true;
2948 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2951 * Intel CPUs mask the counter and pointers in quite strange
2952 * manner when ECX is zero due to REP-string optimizations.
2954 #ifdef CONFIG_X86_64
2955 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2958 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2961 case 0xa4: /* movsb */
2962 case 0xa5: /* movsd/w */
2963 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2965 case 0xaa: /* stosb */
2966 case 0xab: /* stosd/w */
2967 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2972 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2973 struct tss_segment_16 *tss)
2975 tss->ip = ctxt->_eip;
2976 tss->flag = ctxt->eflags;
2977 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2978 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2979 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2980 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2981 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2982 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2983 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2984 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2986 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2987 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2988 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2989 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2990 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2993 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2994 struct tss_segment_16 *tss)
2999 ctxt->_eip = tss->ip;
3000 ctxt->eflags = tss->flag | 2;
3001 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3002 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3003 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3004 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3005 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3006 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3007 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3008 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3011 * SDM says that segment selectors are loaded before segment
3014 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3015 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3016 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3017 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3018 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3023 * Now load segment descriptors. If fault happens at this stage
3024 * it is handled in a context of new task
3026 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3027 X86_TRANSFER_TASK_SWITCH, NULL);
3028 if (ret != X86EMUL_CONTINUE)
3030 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3031 X86_TRANSFER_TASK_SWITCH, NULL);
3032 if (ret != X86EMUL_CONTINUE)
3034 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3035 X86_TRANSFER_TASK_SWITCH, NULL);
3036 if (ret != X86EMUL_CONTINUE)
3038 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3039 X86_TRANSFER_TASK_SWITCH, NULL);
3040 if (ret != X86EMUL_CONTINUE)
3042 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3043 X86_TRANSFER_TASK_SWITCH, NULL);
3044 if (ret != X86EMUL_CONTINUE)
3047 return X86EMUL_CONTINUE;
3050 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3051 u16 tss_selector, u16 old_tss_sel,
3052 ulong old_tss_base, struct desc_struct *new_desc)
3054 struct tss_segment_16 tss_seg;
3056 u32 new_tss_base = get_desc_base(new_desc);
3058 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3059 if (ret != X86EMUL_CONTINUE)
3062 save_state_to_tss16(ctxt, &tss_seg);
3064 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3065 if (ret != X86EMUL_CONTINUE)
3068 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3069 if (ret != X86EMUL_CONTINUE)
3072 if (old_tss_sel != 0xffff) {
3073 tss_seg.prev_task_link = old_tss_sel;
3075 ret = linear_write_system(ctxt, new_tss_base,
3076 &tss_seg.prev_task_link,
3077 sizeof tss_seg.prev_task_link);
3078 if (ret != X86EMUL_CONTINUE)
3082 return load_state_from_tss16(ctxt, &tss_seg);
3085 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3086 struct tss_segment_32 *tss)
3088 /* CR3 and ldt selector are not saved intentionally */
3089 tss->eip = ctxt->_eip;
3090 tss->eflags = ctxt->eflags;
3091 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3092 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3093 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3094 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3095 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3096 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3097 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3098 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3100 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3101 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3102 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3103 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3104 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3105 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3108 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3109 struct tss_segment_32 *tss)
3114 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3115 return emulate_gp(ctxt, 0);
3116 ctxt->_eip = tss->eip;
3117 ctxt->eflags = tss->eflags | 2;
3119 /* General purpose registers */
3120 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3121 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3122 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3123 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3124 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3125 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3126 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3127 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3130 * SDM says that segment selectors are loaded before segment
3131 * descriptors. This is important because CPL checks will
3134 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3135 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3136 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3137 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3138 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3139 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3140 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3143 * If we're switching between Protected Mode and VM86, we need to make
3144 * sure to update the mode before loading the segment descriptors so
3145 * that the selectors are interpreted correctly.
3147 if (ctxt->eflags & X86_EFLAGS_VM) {
3148 ctxt->mode = X86EMUL_MODE_VM86;
3151 ctxt->mode = X86EMUL_MODE_PROT32;
3156 * Now load segment descriptors. If fault happenes at this stage
3157 * it is handled in a context of new task
3159 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3160 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3161 if (ret != X86EMUL_CONTINUE)
3163 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3164 X86_TRANSFER_TASK_SWITCH, NULL);
3165 if (ret != X86EMUL_CONTINUE)
3167 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3168 X86_TRANSFER_TASK_SWITCH, NULL);
3169 if (ret != X86EMUL_CONTINUE)
3171 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3172 X86_TRANSFER_TASK_SWITCH, NULL);
3173 if (ret != X86EMUL_CONTINUE)
3175 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3176 X86_TRANSFER_TASK_SWITCH, NULL);
3177 if (ret != X86EMUL_CONTINUE)
3179 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3180 X86_TRANSFER_TASK_SWITCH, NULL);
3181 if (ret != X86EMUL_CONTINUE)
3183 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3184 X86_TRANSFER_TASK_SWITCH, NULL);
3189 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3190 u16 tss_selector, u16 old_tss_sel,
3191 ulong old_tss_base, struct desc_struct *new_desc)
3193 struct tss_segment_32 tss_seg;
3195 u32 new_tss_base = get_desc_base(new_desc);
3196 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3197 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3199 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3200 if (ret != X86EMUL_CONTINUE)
3203 save_state_to_tss32(ctxt, &tss_seg);
3205 /* Only GP registers and segment selectors are saved */
3206 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3207 ldt_sel_offset - eip_offset);
3208 if (ret != X86EMUL_CONTINUE)
3211 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3212 if (ret != X86EMUL_CONTINUE)
3215 if (old_tss_sel != 0xffff) {
3216 tss_seg.prev_task_link = old_tss_sel;
3218 ret = linear_write_system(ctxt, new_tss_base,
3219 &tss_seg.prev_task_link,
3220 sizeof tss_seg.prev_task_link);
3221 if (ret != X86EMUL_CONTINUE)
3225 return load_state_from_tss32(ctxt, &tss_seg);
3228 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3229 u16 tss_selector, int idt_index, int reason,
3230 bool has_error_code, u32 error_code)
3232 const struct x86_emulate_ops *ops = ctxt->ops;
3233 struct desc_struct curr_tss_desc, next_tss_desc;
3235 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3236 ulong old_tss_base =
3237 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3239 ulong desc_addr, dr7;
3241 /* FIXME: old_tss_base == ~0 ? */
3243 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3244 if (ret != X86EMUL_CONTINUE)
3246 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3247 if (ret != X86EMUL_CONTINUE)
3250 /* FIXME: check that next_tss_desc is tss */
3253 * Check privileges. The three cases are task switch caused by...
3255 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3256 * 2. Exception/IRQ/iret: No check is performed
3257 * 3. jmp/call to TSS/task-gate: No check is performed since the
3258 * hardware checks it before exiting.
3260 if (reason == TASK_SWITCH_GATE) {
3261 if (idt_index != -1) {
3262 /* Software interrupts */
3263 struct desc_struct task_gate_desc;
3266 ret = read_interrupt_descriptor(ctxt, idt_index,
3268 if (ret != X86EMUL_CONTINUE)
3271 dpl = task_gate_desc.dpl;
3272 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3273 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3277 desc_limit = desc_limit_scaled(&next_tss_desc);
3278 if (!next_tss_desc.p ||
3279 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3280 desc_limit < 0x2b)) {
3281 return emulate_ts(ctxt, tss_selector & 0xfffc);
3284 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3285 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3286 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3289 if (reason == TASK_SWITCH_IRET)
3290 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3292 /* set back link to prev task only if NT bit is set in eflags
3293 note that old_tss_sel is not used after this point */
3294 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3295 old_tss_sel = 0xffff;
3297 if (next_tss_desc.type & 8)
3298 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3299 old_tss_base, &next_tss_desc);
3301 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3302 old_tss_base, &next_tss_desc);
3303 if (ret != X86EMUL_CONTINUE)
3306 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3307 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3309 if (reason != TASK_SWITCH_IRET) {
3310 next_tss_desc.type |= (1 << 1); /* set busy flag */
3311 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3314 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3315 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3317 if (has_error_code) {
3318 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3319 ctxt->lock_prefix = 0;
3320 ctxt->src.val = (unsigned long) error_code;
3321 ret = em_push(ctxt);
3324 ops->get_dr(ctxt, 7, &dr7);
3325 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3330 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3331 u16 tss_selector, int idt_index, int reason,
3332 bool has_error_code, u32 error_code)
3336 invalidate_registers(ctxt);
3337 ctxt->_eip = ctxt->eip;
3338 ctxt->dst.type = OP_NONE;
3340 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3341 has_error_code, error_code);
3343 if (rc == X86EMUL_CONTINUE) {
3344 ctxt->eip = ctxt->_eip;
3345 writeback_registers(ctxt);
3348 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3351 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3354 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3356 register_address_increment(ctxt, reg, df * op->bytes);
3357 op->addr.mem.ea = register_address(ctxt, reg);
3360 static int em_das(struct x86_emulate_ctxt *ctxt)
3363 bool af, cf, old_cf;
3365 cf = ctxt->eflags & X86_EFLAGS_CF;
3371 af = ctxt->eflags & X86_EFLAGS_AF;
3372 if ((al & 0x0f) > 9 || af) {
3374 cf = old_cf | (al >= 250);
3379 if (old_al > 0x99 || old_cf) {
3385 /* Set PF, ZF, SF */
3386 ctxt->src.type = OP_IMM;
3388 ctxt->src.bytes = 1;
3389 fastop(ctxt, em_or);
3390 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3392 ctxt->eflags |= X86_EFLAGS_CF;
3394 ctxt->eflags |= X86_EFLAGS_AF;
3395 return X86EMUL_CONTINUE;
3398 static int em_aam(struct x86_emulate_ctxt *ctxt)
3402 if (ctxt->src.val == 0)
3403 return emulate_de(ctxt);
3405 al = ctxt->dst.val & 0xff;
3406 ah = al / ctxt->src.val;
3407 al %= ctxt->src.val;
3409 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3411 /* Set PF, ZF, SF */
3412 ctxt->src.type = OP_IMM;
3414 ctxt->src.bytes = 1;
3415 fastop(ctxt, em_or);
3417 return X86EMUL_CONTINUE;
3420 static int em_aad(struct x86_emulate_ctxt *ctxt)
3422 u8 al = ctxt->dst.val & 0xff;
3423 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3425 al = (al + (ah * ctxt->src.val)) & 0xff;
3427 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3429 /* Set PF, ZF, SF */
3430 ctxt->src.type = OP_IMM;
3432 ctxt->src.bytes = 1;
3433 fastop(ctxt, em_or);
3435 return X86EMUL_CONTINUE;
3438 static int em_call(struct x86_emulate_ctxt *ctxt)
3441 long rel = ctxt->src.val;
3443 ctxt->src.val = (unsigned long)ctxt->_eip;
3444 rc = jmp_rel(ctxt, rel);
3445 if (rc != X86EMUL_CONTINUE)
3447 return em_push(ctxt);
3450 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3455 struct desc_struct old_desc, new_desc;
3456 const struct x86_emulate_ops *ops = ctxt->ops;
3457 int cpl = ctxt->ops->cpl(ctxt);
3458 enum x86emul_mode prev_mode = ctxt->mode;
3460 old_eip = ctxt->_eip;
3461 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3463 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3464 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3465 X86_TRANSFER_CALL_JMP, &new_desc);
3466 if (rc != X86EMUL_CONTINUE)
3469 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3470 if (rc != X86EMUL_CONTINUE)
3473 ctxt->src.val = old_cs;
3475 if (rc != X86EMUL_CONTINUE)
3478 ctxt->src.val = old_eip;
3480 /* If we failed, we tainted the memory, but the very least we should
3482 if (rc != X86EMUL_CONTINUE) {
3483 pr_warn_once("faulting far call emulation tainted memory\n");
3488 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3489 ctxt->mode = prev_mode;
3494 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3499 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3500 if (rc != X86EMUL_CONTINUE)
3502 rc = assign_eip_near(ctxt, eip);
3503 if (rc != X86EMUL_CONTINUE)
3505 rsp_increment(ctxt, ctxt->src.val);
3506 return X86EMUL_CONTINUE;
3509 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3511 /* Write back the register source. */
3512 ctxt->src.val = ctxt->dst.val;
3513 write_register_operand(&ctxt->src);
3515 /* Write back the memory destination with implicit LOCK prefix. */
3516 ctxt->dst.val = ctxt->src.orig_val;
3517 ctxt->lock_prefix = 1;
3518 return X86EMUL_CONTINUE;
3521 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3523 ctxt->dst.val = ctxt->src2.val;
3524 return fastop(ctxt, em_imul);
3527 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3529 ctxt->dst.type = OP_REG;
3530 ctxt->dst.bytes = ctxt->src.bytes;
3531 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3532 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3534 return X86EMUL_CONTINUE;
3537 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3541 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3542 return emulate_ud(ctxt);
3543 ctxt->dst.val = tsc_aux;
3544 return X86EMUL_CONTINUE;
3547 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3551 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3552 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3553 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3554 return X86EMUL_CONTINUE;
3557 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3561 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3562 return emulate_gp(ctxt, 0);
3563 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3564 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3565 return X86EMUL_CONTINUE;
3568 static int em_mov(struct x86_emulate_ctxt *ctxt)
3570 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3571 return X86EMUL_CONTINUE;
3574 #define FFL(x) bit(X86_FEATURE_##x)
3576 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3578 u32 ebx, ecx, edx, eax = 1;
3582 * Check MOVBE is set in the guest-visible CPUID leaf.
3584 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3585 if (!(ecx & FFL(MOVBE)))
3586 return emulate_ud(ctxt);
3588 switch (ctxt->op_bytes) {
3591 * From MOVBE definition: "...When the operand size is 16 bits,
3592 * the upper word of the destination register remains unchanged
3595 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3596 * rules so we have to do the operation almost per hand.
3598 tmp = (u16)ctxt->src.val;
3599 ctxt->dst.val &= ~0xffffUL;
3600 ctxt->dst.val |= (unsigned long)swab16(tmp);
3603 ctxt->dst.val = swab32((u32)ctxt->src.val);
3606 ctxt->dst.val = swab64(ctxt->src.val);
3611 return X86EMUL_CONTINUE;
3614 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3616 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3617 return emulate_gp(ctxt, 0);
3619 /* Disable writeback. */
3620 ctxt->dst.type = OP_NONE;
3621 return X86EMUL_CONTINUE;
3624 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3628 if (ctxt->mode == X86EMUL_MODE_PROT64)
3629 val = ctxt->src.val & ~0ULL;
3631 val = ctxt->src.val & ~0U;
3633 /* #UD condition is already handled. */
3634 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3635 return emulate_gp(ctxt, 0);
3637 /* Disable writeback. */
3638 ctxt->dst.type = OP_NONE;
3639 return X86EMUL_CONTINUE;
3642 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3646 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3647 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3648 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3649 return emulate_gp(ctxt, 0);
3651 return X86EMUL_CONTINUE;
3654 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3658 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3659 return emulate_gp(ctxt, 0);
3661 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3662 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3663 return X86EMUL_CONTINUE;
3666 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3668 if (ctxt->modrm_reg > VCPU_SREG_GS)
3669 return emulate_ud(ctxt);
3671 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3672 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3673 ctxt->dst.bytes = 2;
3674 return X86EMUL_CONTINUE;
3677 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3679 u16 sel = ctxt->src.val;
3681 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3682 return emulate_ud(ctxt);
3684 if (ctxt->modrm_reg == VCPU_SREG_SS)
3685 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3687 /* Disable writeback. */
3688 ctxt->dst.type = OP_NONE;
3689 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3692 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3694 u16 sel = ctxt->src.val;
3696 /* Disable writeback. */
3697 ctxt->dst.type = OP_NONE;
3698 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3701 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3703 u16 sel = ctxt->src.val;
3705 /* Disable writeback. */
3706 ctxt->dst.type = OP_NONE;
3707 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3710 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3715 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3716 if (rc == X86EMUL_CONTINUE)
3717 ctxt->ops->invlpg(ctxt, linear);
3718 /* Disable writeback. */
3719 ctxt->dst.type = OP_NONE;
3720 return X86EMUL_CONTINUE;
3723 static int em_clts(struct x86_emulate_ctxt *ctxt)
3727 cr0 = ctxt->ops->get_cr(ctxt, 0);
3729 ctxt->ops->set_cr(ctxt, 0, cr0);
3730 return X86EMUL_CONTINUE;
3733 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3735 int rc = ctxt->ops->fix_hypercall(ctxt);
3737 if (rc != X86EMUL_CONTINUE)
3740 /* Let the processor re-execute the fixed hypercall */
3741 ctxt->_eip = ctxt->eip;
3742 /* Disable writeback. */
3743 ctxt->dst.type = OP_NONE;
3744 return X86EMUL_CONTINUE;
3747 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3748 void (*get)(struct x86_emulate_ctxt *ctxt,
3749 struct desc_ptr *ptr))
3751 struct desc_ptr desc_ptr;
3753 if (ctxt->mode == X86EMUL_MODE_PROT64)
3755 get(ctxt, &desc_ptr);
3756 if (ctxt->op_bytes == 2) {
3758 desc_ptr.address &= 0x00ffffff;
3760 /* Disable writeback. */
3761 ctxt->dst.type = OP_NONE;
3762 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3763 &desc_ptr, 2 + ctxt->op_bytes);
3766 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3768 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3771 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3773 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3776 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3778 struct desc_ptr desc_ptr;
3781 if (ctxt->mode == X86EMUL_MODE_PROT64)
3783 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3784 &desc_ptr.size, &desc_ptr.address,
3786 if (rc != X86EMUL_CONTINUE)
3788 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3789 is_noncanonical_address(desc_ptr.address))
3790 return emulate_gp(ctxt, 0);
3792 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3794 ctxt->ops->set_idt(ctxt, &desc_ptr);
3795 /* Disable writeback. */
3796 ctxt->dst.type = OP_NONE;
3797 return X86EMUL_CONTINUE;
3800 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3802 return em_lgdt_lidt(ctxt, true);
3805 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3807 return em_lgdt_lidt(ctxt, false);
3810 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3812 if (ctxt->dst.type == OP_MEM)
3813 ctxt->dst.bytes = 2;
3814 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3815 return X86EMUL_CONTINUE;
3818 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3820 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3821 | (ctxt->src.val & 0x0f));
3822 ctxt->dst.type = OP_NONE;
3823 return X86EMUL_CONTINUE;
3826 static int em_loop(struct x86_emulate_ctxt *ctxt)
3828 int rc = X86EMUL_CONTINUE;
3830 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3831 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3832 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3833 rc = jmp_rel(ctxt, ctxt->src.val);
3838 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3840 int rc = X86EMUL_CONTINUE;
3842 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3843 rc = jmp_rel(ctxt, ctxt->src.val);
3848 static int em_in(struct x86_emulate_ctxt *ctxt)
3850 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3852 return X86EMUL_IO_NEEDED;
3854 return X86EMUL_CONTINUE;
3857 static int em_out(struct x86_emulate_ctxt *ctxt)
3859 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3861 /* Disable writeback. */
3862 ctxt->dst.type = OP_NONE;
3863 return X86EMUL_CONTINUE;
3866 static int em_cli(struct x86_emulate_ctxt *ctxt)
3868 if (emulator_bad_iopl(ctxt))
3869 return emulate_gp(ctxt, 0);
3871 ctxt->eflags &= ~X86_EFLAGS_IF;
3872 return X86EMUL_CONTINUE;
3875 static int em_sti(struct x86_emulate_ctxt *ctxt)
3877 if (emulator_bad_iopl(ctxt))
3878 return emulate_gp(ctxt, 0);
3880 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3881 ctxt->eflags |= X86_EFLAGS_IF;
3882 return X86EMUL_CONTINUE;
3885 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3887 u32 eax, ebx, ecx, edx;
3889 eax = reg_read(ctxt, VCPU_REGS_RAX);
3890 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3891 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3892 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3893 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3894 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3895 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3896 return X86EMUL_CONTINUE;
3899 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3903 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3905 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3907 ctxt->eflags &= ~0xffUL;
3908 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3909 return X86EMUL_CONTINUE;
3912 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3914 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3915 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3916 return X86EMUL_CONTINUE;
3919 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3921 switch (ctxt->op_bytes) {
3922 #ifdef CONFIG_X86_64
3924 asm("bswap %0" : "+r"(ctxt->dst.val));
3928 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3931 return X86EMUL_CONTINUE;
3934 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3936 /* emulating clflush regardless of cpuid */
3937 return X86EMUL_CONTINUE;
3940 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3942 /* emulating clflushopt regardless of cpuid */
3943 return X86EMUL_CONTINUE;
3946 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3948 ctxt->dst.val = (s32) ctxt->src.val;
3949 return X86EMUL_CONTINUE;
3952 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3954 u32 eax = 1, ebx, ecx = 0, edx;
3956 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3957 if (!(edx & FFL(FXSR)))
3958 return emulate_ud(ctxt);
3960 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3961 return emulate_nm(ctxt);
3964 * Don't emulate a case that should never be hit, instead of working
3965 * around a lack of fxsave64/fxrstor64 on old compilers.
3967 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3968 return X86EMUL_UNHANDLEABLE;
3970 return X86EMUL_CONTINUE;
3974 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3977 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3978 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3980 * 3) 64-bit mode with REX.W prefix
3981 * - like (2), but XMM 8-15 are being saved and restored
3982 * 4) 64-bit mode without REX.W prefix
3983 * - like (3), but FIP and FDP are 64 bit
3985 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3986 * desired result. (4) is not emulated.
3988 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3989 * and FPU DS) should match.
3991 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
3993 struct fxregs_state fx_state;
3997 rc = check_fxsr(ctxt);
3998 if (rc != X86EMUL_CONTINUE)
4001 ctxt->ops->get_fpu(ctxt);
4003 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4005 ctxt->ops->put_fpu(ctxt);
4007 if (rc != X86EMUL_CONTINUE)
4010 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
4011 size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
4013 size = offsetof(struct fxregs_state, xmm_space[0]);
4015 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4018 static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
4019 struct fxregs_state *new)
4021 int rc = X86EMUL_CONTINUE;
4022 struct fxregs_state old;
4024 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
4025 if (rc != X86EMUL_CONTINUE)
4029 * 64 bit host will restore XMM 8-15, which is not correct on non-64
4030 * bit guests. Load the current values in order to preserve 64 bit
4031 * XMMs after fxrstor.
4033 #ifdef CONFIG_X86_64
4034 /* XXX: accessing XMM 8-15 very awkwardly */
4035 memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
4039 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
4040 * does save and restore MXCSR.
4042 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
4043 memcpy(new->xmm_space, old.xmm_space, 8 * 16);
4048 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4050 struct fxregs_state fx_state;
4053 rc = check_fxsr(ctxt);
4054 if (rc != X86EMUL_CONTINUE)
4057 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
4058 if (rc != X86EMUL_CONTINUE)
4061 if (fx_state.mxcsr >> 16)
4062 return emulate_gp(ctxt, 0);
4064 ctxt->ops->get_fpu(ctxt);
4066 if (ctxt->mode < X86EMUL_MODE_PROT64)
4067 rc = fxrstor_fixup(ctxt, &fx_state);
4069 if (rc == X86EMUL_CONTINUE)
4070 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4072 ctxt->ops->put_fpu(ctxt);
4077 static bool valid_cr(int nr)
4089 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4091 if (!valid_cr(ctxt->modrm_reg))
4092 return emulate_ud(ctxt);
4094 return X86EMUL_CONTINUE;
4097 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4099 u64 new_val = ctxt->src.val64;
4100 int cr = ctxt->modrm_reg;
4103 static u64 cr_reserved_bits[] = {
4104 0xffffffff00000000ULL,
4105 0, 0, 0, /* CR3 checked later */
4112 return emulate_ud(ctxt);
4114 if (new_val & cr_reserved_bits[cr])
4115 return emulate_gp(ctxt, 0);
4120 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4121 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4122 return emulate_gp(ctxt, 0);
4124 cr4 = ctxt->ops->get_cr(ctxt, 4);
4125 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4127 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4128 !(cr4 & X86_CR4_PAE))
4129 return emulate_gp(ctxt, 0);
4136 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4137 if (efer & EFER_LMA)
4138 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
4141 return emulate_gp(ctxt, 0);
4146 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4148 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4149 return emulate_gp(ctxt, 0);
4155 return X86EMUL_CONTINUE;
4158 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4162 ctxt->ops->get_dr(ctxt, 7, &dr7);
4164 /* Check if DR7.Global_Enable is set */
4165 return dr7 & (1 << 13);
4168 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4170 int dr = ctxt->modrm_reg;
4174 return emulate_ud(ctxt);
4176 cr4 = ctxt->ops->get_cr(ctxt, 4);
4177 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4178 return emulate_ud(ctxt);
4180 if (check_dr7_gd(ctxt)) {
4183 ctxt->ops->get_dr(ctxt, 6, &dr6);
4185 dr6 |= DR6_BD | DR6_RTM;
4186 ctxt->ops->set_dr(ctxt, 6, dr6);
4187 return emulate_db(ctxt);
4190 return X86EMUL_CONTINUE;
4193 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4195 u64 new_val = ctxt->src.val64;
4196 int dr = ctxt->modrm_reg;
4198 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4199 return emulate_gp(ctxt, 0);
4201 return check_dr_read(ctxt);
4204 static int check_svme(struct x86_emulate_ctxt *ctxt)
4208 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4210 if (!(efer & EFER_SVME))
4211 return emulate_ud(ctxt);
4213 return X86EMUL_CONTINUE;
4216 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4218 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4220 /* Valid physical address? */
4221 if (rax & 0xffff000000000000ULL)
4222 return emulate_gp(ctxt, 0);
4224 return check_svme(ctxt);
4227 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4229 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4231 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4232 return emulate_ud(ctxt);
4234 return X86EMUL_CONTINUE;
4237 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4239 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4240 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4242 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4243 ctxt->ops->check_pmc(ctxt, rcx))
4244 return emulate_gp(ctxt, 0);
4246 return X86EMUL_CONTINUE;
4249 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4251 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4252 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4253 return emulate_gp(ctxt, 0);
4255 return X86EMUL_CONTINUE;
4258 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4260 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4261 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4262 return emulate_gp(ctxt, 0);
4264 return X86EMUL_CONTINUE;
4267 #define D(_y) { .flags = (_y) }
4268 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4269 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4270 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4271 #define N D(NotImpl)
4272 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4273 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4274 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4275 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4276 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4277 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4278 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4279 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4280 #define II(_f, _e, _i) \
4281 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4282 #define IIP(_f, _e, _i, _p) \
4283 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4284 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4285 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4287 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4288 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4289 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4290 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4291 #define I2bvIP(_f, _e, _i, _p) \
4292 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4294 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4295 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4296 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4298 static const struct opcode group7_rm0[] = {
4300 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4304 static const struct opcode group7_rm1[] = {
4305 DI(SrcNone | Priv, monitor),
4306 DI(SrcNone | Priv, mwait),
4310 static const struct opcode group7_rm3[] = {
4311 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4312 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4313 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4314 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4315 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4316 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4317 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4318 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4321 static const struct opcode group7_rm7[] = {
4323 DIP(SrcNone, rdtscp, check_rdtsc),
4327 static const struct opcode group1[] = {
4329 F(Lock | PageTable, em_or),
4332 F(Lock | PageTable, em_and),
4338 static const struct opcode group1A[] = {
4339 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4342 static const struct opcode group2[] = {
4343 F(DstMem | ModRM, em_rol),
4344 F(DstMem | ModRM, em_ror),
4345 F(DstMem | ModRM, em_rcl),
4346 F(DstMem | ModRM, em_rcr),
4347 F(DstMem | ModRM, em_shl),
4348 F(DstMem | ModRM, em_shr),
4349 F(DstMem | ModRM, em_shl),
4350 F(DstMem | ModRM, em_sar),
4353 static const struct opcode group3[] = {
4354 F(DstMem | SrcImm | NoWrite, em_test),
4355 F(DstMem | SrcImm | NoWrite, em_test),
4356 F(DstMem | SrcNone | Lock, em_not),
4357 F(DstMem | SrcNone | Lock, em_neg),
4358 F(DstXacc | Src2Mem, em_mul_ex),
4359 F(DstXacc | Src2Mem, em_imul_ex),
4360 F(DstXacc | Src2Mem, em_div_ex),
4361 F(DstXacc | Src2Mem, em_idiv_ex),
4364 static const struct opcode group4[] = {
4365 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4366 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4370 static const struct opcode group5[] = {
4371 F(DstMem | SrcNone | Lock, em_inc),
4372 F(DstMem | SrcNone | Lock, em_dec),
4373 I(SrcMem | NearBranch, em_call_near_abs),
4374 I(SrcMemFAddr | ImplicitOps, em_call_far),
4375 I(SrcMem | NearBranch, em_jmp_abs),
4376 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4377 I(SrcMem | Stack, em_push), D(Undefined),
4380 static const struct opcode group6[] = {
4381 DI(Prot | DstMem, sldt),
4382 DI(Prot | DstMem, str),
4383 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4384 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4388 static const struct group_dual group7 = { {
4389 II(Mov | DstMem, em_sgdt, sgdt),
4390 II(Mov | DstMem, em_sidt, sidt),
4391 II(SrcMem | Priv, em_lgdt, lgdt),
4392 II(SrcMem | Priv, em_lidt, lidt),
4393 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4394 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4395 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4399 N, EXT(0, group7_rm3),
4400 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4401 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4405 static const struct opcode group8[] = {
4407 F(DstMem | SrcImmByte | NoWrite, em_bt),
4408 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4409 F(DstMem | SrcImmByte | Lock, em_btr),
4410 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4414 * The "memory" destination is actually always a register, since we come
4415 * from the register case of group9.
4417 static const struct gprefix pfx_0f_c7_7 = {
4418 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4422 static const struct group_dual group9 = { {
4423 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4425 N, N, N, N, N, N, N,
4426 GP(0, &pfx_0f_c7_7),
4429 static const struct opcode group11[] = {
4430 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4434 static const struct gprefix pfx_0f_ae_7 = {
4435 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4438 static const struct group_dual group15 = { {
4439 I(ModRM | Aligned16, em_fxsave),
4440 I(ModRM | Aligned16, em_fxrstor),
4441 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4443 N, N, N, N, N, N, N, N,
4446 static const struct gprefix pfx_0f_6f_0f_7f = {
4447 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4450 static const struct instr_dual instr_dual_0f_2b = {
4454 static const struct gprefix pfx_0f_2b = {
4455 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4458 static const struct gprefix pfx_0f_28_0f_29 = {
4459 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4462 static const struct gprefix pfx_0f_e7 = {
4463 N, I(Sse, em_mov), N, N,
4466 static const struct escape escape_d9 = { {
4467 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4470 N, N, N, N, N, N, N, N,
4472 N, N, N, N, N, N, N, N,
4474 N, N, N, N, N, N, N, N,
4476 N, N, N, N, N, N, N, N,
4478 N, N, N, N, N, N, N, N,
4480 N, N, N, N, N, N, N, N,
4482 N, N, N, N, N, N, N, N,
4484 N, N, N, N, N, N, N, N,
4487 static const struct escape escape_db = { {
4488 N, N, N, N, N, N, N, N,
4491 N, N, N, N, N, N, N, N,
4493 N, N, N, N, N, N, N, N,
4495 N, N, N, N, N, N, N, N,
4497 N, N, N, N, N, N, N, N,
4499 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4501 N, N, N, N, N, N, N, N,
4503 N, N, N, N, N, N, N, N,
4505 N, N, N, N, N, N, N, N,
4508 static const struct escape escape_dd = { {
4509 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4512 N, N, N, N, N, N, N, N,
4514 N, N, N, N, N, N, N, N,
4516 N, N, N, N, N, N, N, N,
4518 N, N, N, N, N, N, N, N,
4520 N, N, N, N, N, N, N, N,
4522 N, N, N, N, N, N, N, N,
4524 N, N, N, N, N, N, N, N,
4526 N, N, N, N, N, N, N, N,
4529 static const struct instr_dual instr_dual_0f_c3 = {
4530 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4533 static const struct mode_dual mode_dual_63 = {
4534 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4537 static const struct opcode opcode_table[256] = {
4539 F6ALU(Lock, em_add),
4540 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4541 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4543 F6ALU(Lock | PageTable, em_or),
4544 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4547 F6ALU(Lock, em_adc),
4548 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4549 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4551 F6ALU(Lock, em_sbb),
4552 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4553 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4555 F6ALU(Lock | PageTable, em_and), N, N,
4557 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4559 F6ALU(Lock, em_xor), N, N,
4561 F6ALU(NoWrite, em_cmp), N, N,
4563 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4565 X8(I(SrcReg | Stack, em_push)),
4567 X8(I(DstReg | Stack, em_pop)),
4569 I(ImplicitOps | Stack | No64, em_pusha),
4570 I(ImplicitOps | Stack | No64, em_popa),
4571 N, MD(ModRM, &mode_dual_63),
4574 I(SrcImm | Mov | Stack, em_push),
4575 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4576 I(SrcImmByte | Mov | Stack, em_push),
4577 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4578 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4579 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4581 X16(D(SrcImmByte | NearBranch)),
4583 G(ByteOp | DstMem | SrcImm, group1),
4584 G(DstMem | SrcImm, group1),
4585 G(ByteOp | DstMem | SrcImm | No64, group1),
4586 G(DstMem | SrcImmByte, group1),
4587 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4588 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4590 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4591 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4592 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4593 D(ModRM | SrcMem | NoAccess | DstReg),
4594 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4597 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4599 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4600 I(SrcImmFAddr | No64, em_call_far), N,
4601 II(ImplicitOps | Stack, em_pushf, pushf),
4602 II(ImplicitOps | Stack, em_popf, popf),
4603 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4605 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4606 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4607 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4608 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4610 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4611 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4612 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4613 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4615 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4617 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4619 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4620 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4621 I(ImplicitOps | NearBranch, em_ret),
4622 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4623 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4624 G(ByteOp, group11), G(0, group11),
4626 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4627 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4628 I(ImplicitOps, em_ret_far),
4629 D(ImplicitOps), DI(SrcImmByte, intn),
4630 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4632 G(Src2One | ByteOp, group2), G(Src2One, group2),
4633 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4634 I(DstAcc | SrcImmUByte | No64, em_aam),
4635 I(DstAcc | SrcImmUByte | No64, em_aad),
4636 F(DstAcc | ByteOp | No64, em_salc),
4637 I(DstAcc | SrcXLat | ByteOp, em_mov),
4639 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4641 X3(I(SrcImmByte | NearBranch, em_loop)),
4642 I(SrcImmByte | NearBranch, em_jcxz),
4643 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4644 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4646 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4647 I(SrcImmFAddr | No64, em_jmp_far),
4648 D(SrcImmByte | ImplicitOps | NearBranch),
4649 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4650 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4652 N, DI(ImplicitOps, icebp), N, N,
4653 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4654 G(ByteOp, group3), G(0, group3),
4656 D(ImplicitOps), D(ImplicitOps),
4657 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4658 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4661 static const struct opcode twobyte_table[256] = {
4663 G(0, group6), GD(0, &group7), N, N,
4664 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4665 II(ImplicitOps | Priv, em_clts, clts), N,
4666 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4667 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4669 N, N, N, N, N, N, N, N,
4670 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4671 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4673 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4674 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4675 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4677 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4680 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4681 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4682 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4685 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4686 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4687 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4688 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4689 I(ImplicitOps | EmulateOnUD, em_sysenter),
4690 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4692 N, N, N, N, N, N, N, N,
4694 X16(D(DstReg | SrcMem | ModRM)),
4696 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4701 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4706 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4708 X16(D(SrcImm | NearBranch)),
4710 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4712 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4713 II(ImplicitOps, em_cpuid, cpuid),
4714 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4715 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4716 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4718 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4719 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4720 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4721 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4722 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4723 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4725 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4726 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4727 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4728 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4729 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4730 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4734 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4735 I(DstReg | SrcMem | ModRM, em_bsf_c),
4736 I(DstReg | SrcMem | ModRM, em_bsr_c),
4737 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4739 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4740 N, ID(0, &instr_dual_0f_c3),
4741 N, N, N, GD(0, &group9),
4743 X8(I(DstReg, em_bswap)),
4745 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4747 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4748 N, N, N, N, N, N, N, N,
4750 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4753 static const struct instr_dual instr_dual_0f_38_f0 = {
4754 I(DstReg | SrcMem | Mov, em_movbe), N
4757 static const struct instr_dual instr_dual_0f_38_f1 = {
4758 I(DstMem | SrcReg | Mov, em_movbe), N
4761 static const struct gprefix three_byte_0f_38_f0 = {
4762 ID(0, &instr_dual_0f_38_f0), N, N, N
4765 static const struct gprefix three_byte_0f_38_f1 = {
4766 ID(0, &instr_dual_0f_38_f1), N, N, N
4770 * Insns below are selected by the prefix which indexed by the third opcode
4773 static const struct opcode opcode_map_0f_38[256] = {
4775 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4777 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4779 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4780 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4801 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4805 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4811 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4812 unsigned size, bool sign_extension)
4814 int rc = X86EMUL_CONTINUE;
4818 op->addr.mem.ea = ctxt->_eip;
4819 /* NB. Immediates are sign-extended as necessary. */
4820 switch (op->bytes) {
4822 op->val = insn_fetch(s8, ctxt);
4825 op->val = insn_fetch(s16, ctxt);
4828 op->val = insn_fetch(s32, ctxt);
4831 op->val = insn_fetch(s64, ctxt);
4834 if (!sign_extension) {
4835 switch (op->bytes) {
4843 op->val &= 0xffffffff;
4851 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4854 int rc = X86EMUL_CONTINUE;
4858 decode_register_operand(ctxt, op);
4861 rc = decode_imm(ctxt, op, 1, false);
4864 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4868 if (ctxt->d & BitOp)
4869 fetch_bit_operand(ctxt);
4870 op->orig_val = op->val;
4873 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4877 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4878 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4879 fetch_register_operand(op);
4880 op->orig_val = op->val;
4884 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4885 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4886 fetch_register_operand(op);
4887 op->orig_val = op->val;
4890 if (ctxt->d & ByteOp) {
4895 op->bytes = ctxt->op_bytes;
4896 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4897 fetch_register_operand(op);
4898 op->orig_val = op->val;
4902 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4904 register_address(ctxt, VCPU_REGS_RDI);
4905 op->addr.mem.seg = VCPU_SREG_ES;
4912 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4913 fetch_register_operand(op);
4918 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4921 rc = decode_imm(ctxt, op, 1, true);
4929 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4932 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4935 ctxt->memop.bytes = 1;
4936 if (ctxt->memop.type == OP_REG) {
4937 ctxt->memop.addr.reg = decode_register(ctxt,
4938 ctxt->modrm_rm, true);
4939 fetch_register_operand(&ctxt->memop);
4943 ctxt->memop.bytes = 2;
4946 ctxt->memop.bytes = 4;
4949 rc = decode_imm(ctxt, op, 2, false);
4952 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4956 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4958 register_address(ctxt, VCPU_REGS_RSI);
4959 op->addr.mem.seg = ctxt->seg_override;
4965 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4968 reg_read(ctxt, VCPU_REGS_RBX) +
4969 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4970 op->addr.mem.seg = ctxt->seg_override;
4975 op->addr.mem.ea = ctxt->_eip;
4976 op->bytes = ctxt->op_bytes + 2;
4977 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4980 ctxt->memop.bytes = ctxt->op_bytes + 2;
4984 op->val = VCPU_SREG_ES;
4988 op->val = VCPU_SREG_CS;
4992 op->val = VCPU_SREG_SS;
4996 op->val = VCPU_SREG_DS;
5000 op->val = VCPU_SREG_FS;
5004 op->val = VCPU_SREG_GS;
5007 /* Special instructions do their own operand decoding. */
5009 op->type = OP_NONE; /* Disable writeback. */
5017 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5019 int rc = X86EMUL_CONTINUE;
5020 int mode = ctxt->mode;
5021 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5022 bool op_prefix = false;
5023 bool has_seg_override = false;
5024 struct opcode opcode;
5026 struct desc_struct desc;
5028 ctxt->memop.type = OP_NONE;
5029 ctxt->memopp = NULL;
5030 ctxt->_eip = ctxt->eip;
5031 ctxt->fetch.ptr = ctxt->fetch.data;
5032 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5033 ctxt->opcode_len = 1;
5034 ctxt->intercept = x86_intercept_none;
5036 memcpy(ctxt->fetch.data, insn, insn_len);
5038 rc = __do_insn_fetch_bytes(ctxt, 1);
5039 if (rc != X86EMUL_CONTINUE)
5044 case X86EMUL_MODE_REAL:
5045 case X86EMUL_MODE_VM86:
5046 def_op_bytes = def_ad_bytes = 2;
5047 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5049 def_op_bytes = def_ad_bytes = 4;
5051 case X86EMUL_MODE_PROT16:
5052 def_op_bytes = def_ad_bytes = 2;
5054 case X86EMUL_MODE_PROT32:
5055 def_op_bytes = def_ad_bytes = 4;
5057 #ifdef CONFIG_X86_64
5058 case X86EMUL_MODE_PROT64:
5064 return EMULATION_FAILED;
5067 ctxt->op_bytes = def_op_bytes;
5068 ctxt->ad_bytes = def_ad_bytes;
5070 /* Legacy prefixes. */
5072 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5073 case 0x66: /* operand-size override */
5075 /* switch between 2/4 bytes */
5076 ctxt->op_bytes = def_op_bytes ^ 6;
5078 case 0x67: /* address-size override */
5079 if (mode == X86EMUL_MODE_PROT64)
5080 /* switch between 4/8 bytes */
5081 ctxt->ad_bytes = def_ad_bytes ^ 12;
5083 /* switch between 2/4 bytes */
5084 ctxt->ad_bytes = def_ad_bytes ^ 6;
5086 case 0x26: /* ES override */
5087 has_seg_override = true;
5088 ctxt->seg_override = VCPU_SREG_ES;
5090 case 0x2e: /* CS override */
5091 has_seg_override = true;
5092 ctxt->seg_override = VCPU_SREG_CS;
5094 case 0x36: /* SS override */
5095 has_seg_override = true;
5096 ctxt->seg_override = VCPU_SREG_SS;
5098 case 0x3e: /* DS override */
5099 has_seg_override = true;
5100 ctxt->seg_override = VCPU_SREG_DS;
5102 case 0x64: /* FS override */
5103 has_seg_override = true;
5104 ctxt->seg_override = VCPU_SREG_FS;
5106 case 0x65: /* GS override */
5107 has_seg_override = true;
5108 ctxt->seg_override = VCPU_SREG_GS;
5110 case 0x40 ... 0x4f: /* REX */
5111 if (mode != X86EMUL_MODE_PROT64)
5113 ctxt->rex_prefix = ctxt->b;
5115 case 0xf0: /* LOCK */
5116 ctxt->lock_prefix = 1;
5118 case 0xf2: /* REPNE/REPNZ */
5119 case 0xf3: /* REP/REPE/REPZ */
5120 ctxt->rep_prefix = ctxt->b;
5126 /* Any legacy prefix after a REX prefix nullifies its effect. */
5128 ctxt->rex_prefix = 0;
5134 if (ctxt->rex_prefix & 8)
5135 ctxt->op_bytes = 8; /* REX.W */
5137 /* Opcode byte(s). */
5138 opcode = opcode_table[ctxt->b];
5139 /* Two-byte opcode? */
5140 if (ctxt->b == 0x0f) {
5141 ctxt->opcode_len = 2;
5142 ctxt->b = insn_fetch(u8, ctxt);
5143 opcode = twobyte_table[ctxt->b];
5145 /* 0F_38 opcode map */
5146 if (ctxt->b == 0x38) {
5147 ctxt->opcode_len = 3;
5148 ctxt->b = insn_fetch(u8, ctxt);
5149 opcode = opcode_map_0f_38[ctxt->b];
5152 ctxt->d = opcode.flags;
5154 if (ctxt->d & ModRM)
5155 ctxt->modrm = insn_fetch(u8, ctxt);
5157 /* vex-prefix instructions are not implemented */
5158 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5159 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5163 while (ctxt->d & GroupMask) {
5164 switch (ctxt->d & GroupMask) {
5166 goffset = (ctxt->modrm >> 3) & 7;
5167 opcode = opcode.u.group[goffset];
5170 goffset = (ctxt->modrm >> 3) & 7;
5171 if ((ctxt->modrm >> 6) == 3)
5172 opcode = opcode.u.gdual->mod3[goffset];
5174 opcode = opcode.u.gdual->mod012[goffset];
5177 goffset = ctxt->modrm & 7;
5178 opcode = opcode.u.group[goffset];
5181 if (ctxt->rep_prefix && op_prefix)
5182 return EMULATION_FAILED;
5183 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5184 switch (simd_prefix) {
5185 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5186 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5187 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5188 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5192 if (ctxt->modrm > 0xbf) {
5193 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5194 u32 index = array_index_nospec(
5195 ctxt->modrm - 0xc0, size);
5197 opcode = opcode.u.esc->high[index];
5199 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5203 if ((ctxt->modrm >> 6) == 3)
5204 opcode = opcode.u.idual->mod3;
5206 opcode = opcode.u.idual->mod012;
5209 if (ctxt->mode == X86EMUL_MODE_PROT64)
5210 opcode = opcode.u.mdual->mode64;
5212 opcode = opcode.u.mdual->mode32;
5215 return EMULATION_FAILED;
5218 ctxt->d &= ~(u64)GroupMask;
5219 ctxt->d |= opcode.flags;
5224 return EMULATION_FAILED;
5226 ctxt->execute = opcode.u.execute;
5228 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5229 return EMULATION_FAILED;
5231 if (unlikely(ctxt->d &
5232 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5235 * These are copied unconditionally here, and checked unconditionally
5236 * in x86_emulate_insn.
5238 ctxt->check_perm = opcode.check_perm;
5239 ctxt->intercept = opcode.intercept;
5241 if (ctxt->d & NotImpl)
5242 return EMULATION_FAILED;
5244 if (mode == X86EMUL_MODE_PROT64) {
5245 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5247 else if (ctxt->d & NearBranch)
5251 if (ctxt->d & Op3264) {
5252 if (mode == X86EMUL_MODE_PROT64)
5258 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5262 ctxt->op_bytes = 16;
5263 else if (ctxt->d & Mmx)
5267 /* ModRM and SIB bytes. */
5268 if (ctxt->d & ModRM) {
5269 rc = decode_modrm(ctxt, &ctxt->memop);
5270 if (!has_seg_override) {
5271 has_seg_override = true;
5272 ctxt->seg_override = ctxt->modrm_seg;
5274 } else if (ctxt->d & MemAbs)
5275 rc = decode_abs(ctxt, &ctxt->memop);
5276 if (rc != X86EMUL_CONTINUE)
5279 if (!has_seg_override)
5280 ctxt->seg_override = VCPU_SREG_DS;
5282 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5285 * Decode and fetch the source operand: register, memory
5288 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5289 if (rc != X86EMUL_CONTINUE)
5293 * Decode and fetch the second source operand: register, memory
5296 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5297 if (rc != X86EMUL_CONTINUE)
5300 /* Decode and fetch the destination operand: register or memory. */
5301 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5303 if (ctxt->rip_relative && likely(ctxt->memopp))
5304 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5305 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5308 if (rc == X86EMUL_PROPAGATE_FAULT)
5309 ctxt->have_exception = true;
5310 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5313 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5315 return ctxt->d & PageTable;
5318 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5320 /* The second termination condition only applies for REPE
5321 * and REPNE. Test if the repeat string operation prefix is
5322 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5323 * corresponding termination condition according to:
5324 * - if REPE/REPZ and ZF = 0 then done
5325 * - if REPNE/REPNZ and ZF = 1 then done
5327 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5328 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5329 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5330 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5331 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5332 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5338 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5342 ctxt->ops->get_fpu(ctxt);
5343 rc = asm_safe("fwait");
5344 ctxt->ops->put_fpu(ctxt);
5346 if (unlikely(rc != X86EMUL_CONTINUE))
5347 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5349 return X86EMUL_CONTINUE;
5352 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5355 if (op->type == OP_MM)
5356 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5359 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5361 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5363 if (!(ctxt->d & ByteOp))
5364 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5366 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5367 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5368 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5369 : "c"(ctxt->src2.val));
5371 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5372 if (!fop) /* exception is returned in fop variable */
5373 return emulate_de(ctxt);
5374 return X86EMUL_CONTINUE;
5377 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5379 memset(&ctxt->rip_relative, 0,
5380 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5382 ctxt->io_read.pos = 0;
5383 ctxt->io_read.end = 0;
5384 ctxt->mem_read.end = 0;
5387 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5389 const struct x86_emulate_ops *ops = ctxt->ops;
5390 int rc = X86EMUL_CONTINUE;
5391 int saved_dst_type = ctxt->dst.type;
5392 unsigned emul_flags;
5394 ctxt->mem_read.pos = 0;
5396 /* LOCK prefix is allowed only with some instructions */
5397 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5398 rc = emulate_ud(ctxt);
5402 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5403 rc = emulate_ud(ctxt);
5407 emul_flags = ctxt->ops->get_hflags(ctxt);
5408 if (unlikely(ctxt->d &
5409 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5410 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5411 (ctxt->d & Undefined)) {
5412 rc = emulate_ud(ctxt);
5416 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5417 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5418 rc = emulate_ud(ctxt);
5422 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5423 rc = emulate_nm(ctxt);
5427 if (ctxt->d & Mmx) {
5428 rc = flush_pending_x87_faults(ctxt);
5429 if (rc != X86EMUL_CONTINUE)
5432 * Now that we know the fpu is exception safe, we can fetch
5435 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5436 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5437 if (!(ctxt->d & Mov))
5438 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5441 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5442 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5443 X86_ICPT_PRE_EXCEPT);
5444 if (rc != X86EMUL_CONTINUE)
5448 /* Instruction can only be executed in protected mode */
5449 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5450 rc = emulate_ud(ctxt);
5454 /* Privileged instruction can be executed only in CPL=0 */
5455 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5456 if (ctxt->d & PrivUD)
5457 rc = emulate_ud(ctxt);
5459 rc = emulate_gp(ctxt, 0);
5463 /* Do instruction specific permission checks */
5464 if (ctxt->d & CheckPerm) {
5465 rc = ctxt->check_perm(ctxt);
5466 if (rc != X86EMUL_CONTINUE)
5470 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5471 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5472 X86_ICPT_POST_EXCEPT);
5473 if (rc != X86EMUL_CONTINUE)
5477 if (ctxt->rep_prefix && (ctxt->d & String)) {
5478 /* All REP prefixes have the same first termination condition */
5479 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5480 string_registers_quirk(ctxt);
5481 ctxt->eip = ctxt->_eip;
5482 ctxt->eflags &= ~X86_EFLAGS_RF;
5488 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5489 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5490 ctxt->src.valptr, ctxt->src.bytes);
5491 if (rc != X86EMUL_CONTINUE)
5493 ctxt->src.orig_val64 = ctxt->src.val64;
5496 if (ctxt->src2.type == OP_MEM) {
5497 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5498 &ctxt->src2.val, ctxt->src2.bytes);
5499 if (rc != X86EMUL_CONTINUE)
5503 if ((ctxt->d & DstMask) == ImplicitOps)
5507 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5508 /* optimisation - avoid slow emulated read if Mov */
5509 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5510 &ctxt->dst.val, ctxt->dst.bytes);
5511 if (rc != X86EMUL_CONTINUE) {
5512 if (!(ctxt->d & NoWrite) &&
5513 rc == X86EMUL_PROPAGATE_FAULT &&
5514 ctxt->exception.vector == PF_VECTOR)
5515 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5519 /* Copy full 64-bit value for CMPXCHG8B. */
5520 ctxt->dst.orig_val64 = ctxt->dst.val64;
5524 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5525 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5526 X86_ICPT_POST_MEMACCESS);
5527 if (rc != X86EMUL_CONTINUE)
5531 if (ctxt->rep_prefix && (ctxt->d & String))
5532 ctxt->eflags |= X86_EFLAGS_RF;
5534 ctxt->eflags &= ~X86_EFLAGS_RF;
5536 if (ctxt->execute) {
5537 if (ctxt->d & Fastop) {
5538 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5539 rc = fastop(ctxt, fop);
5540 if (rc != X86EMUL_CONTINUE)
5544 rc = ctxt->execute(ctxt);
5545 if (rc != X86EMUL_CONTINUE)
5550 if (ctxt->opcode_len == 2)
5552 else if (ctxt->opcode_len == 3)
5553 goto threebyte_insn;
5556 case 0x70 ... 0x7f: /* jcc (short) */
5557 if (test_cc(ctxt->b, ctxt->eflags))
5558 rc = jmp_rel(ctxt, ctxt->src.val);
5560 case 0x8d: /* lea r16/r32, m */
5561 ctxt->dst.val = ctxt->src.addr.mem.ea;
5563 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5564 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5565 ctxt->dst.type = OP_NONE;
5569 case 0x98: /* cbw/cwde/cdqe */
5570 switch (ctxt->op_bytes) {
5571 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5572 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5573 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5576 case 0xcc: /* int3 */
5577 rc = emulate_int(ctxt, 3);
5579 case 0xcd: /* int n */
5580 rc = emulate_int(ctxt, ctxt->src.val);
5582 case 0xce: /* into */
5583 if (ctxt->eflags & X86_EFLAGS_OF)
5584 rc = emulate_int(ctxt, 4);
5586 case 0xe9: /* jmp rel */
5587 case 0xeb: /* jmp rel short */
5588 rc = jmp_rel(ctxt, ctxt->src.val);
5589 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5591 case 0xf4: /* hlt */
5592 ctxt->ops->halt(ctxt);
5594 case 0xf5: /* cmc */
5595 /* complement carry flag from eflags reg */
5596 ctxt->eflags ^= X86_EFLAGS_CF;
5598 case 0xf8: /* clc */
5599 ctxt->eflags &= ~X86_EFLAGS_CF;
5601 case 0xf9: /* stc */
5602 ctxt->eflags |= X86_EFLAGS_CF;
5604 case 0xfc: /* cld */
5605 ctxt->eflags &= ~X86_EFLAGS_DF;
5607 case 0xfd: /* std */
5608 ctxt->eflags |= X86_EFLAGS_DF;
5611 goto cannot_emulate;
5614 if (rc != X86EMUL_CONTINUE)
5618 if (ctxt->d & SrcWrite) {
5619 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5620 rc = writeback(ctxt, &ctxt->src);
5621 if (rc != X86EMUL_CONTINUE)
5624 if (!(ctxt->d & NoWrite)) {
5625 rc = writeback(ctxt, &ctxt->dst);
5626 if (rc != X86EMUL_CONTINUE)
5631 * restore dst type in case the decoding will be reused
5632 * (happens for string instruction )
5634 ctxt->dst.type = saved_dst_type;
5636 if ((ctxt->d & SrcMask) == SrcSI)
5637 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5639 if ((ctxt->d & DstMask) == DstDI)
5640 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5642 if (ctxt->rep_prefix && (ctxt->d & String)) {
5644 struct read_cache *r = &ctxt->io_read;
5645 if ((ctxt->d & SrcMask) == SrcSI)
5646 count = ctxt->src.count;
5648 count = ctxt->dst.count;
5649 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5651 if (!string_insn_completed(ctxt)) {
5653 * Re-enter guest when pio read ahead buffer is empty
5654 * or, if it is not used, after each 1024 iteration.
5656 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5657 (r->end == 0 || r->end != r->pos)) {
5659 * Reset read cache. Usually happens before
5660 * decode, but since instruction is restarted
5661 * we have to do it here.
5663 ctxt->mem_read.end = 0;
5664 writeback_registers(ctxt);
5665 return EMULATION_RESTART;
5667 goto done; /* skip rip writeback */
5669 ctxt->eflags &= ~X86_EFLAGS_RF;
5672 ctxt->eip = ctxt->_eip;
5675 if (rc == X86EMUL_PROPAGATE_FAULT) {
5676 WARN_ON(ctxt->exception.vector > 0x1f);
5677 ctxt->have_exception = true;
5679 if (rc == X86EMUL_INTERCEPTED)
5680 return EMULATION_INTERCEPTED;
5682 if (rc == X86EMUL_CONTINUE)
5683 writeback_registers(ctxt);
5685 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5689 case 0x09: /* wbinvd */
5690 (ctxt->ops->wbinvd)(ctxt);
5692 case 0x08: /* invd */
5693 case 0x0d: /* GrpP (prefetch) */
5694 case 0x18: /* Grp16 (prefetch/nop) */
5695 case 0x1f: /* nop */
5697 case 0x20: /* mov cr, reg */
5698 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5700 case 0x21: /* mov from dr to reg */
5701 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5703 case 0x40 ... 0x4f: /* cmov */
5704 if (test_cc(ctxt->b, ctxt->eflags))
5705 ctxt->dst.val = ctxt->src.val;
5706 else if (ctxt->op_bytes != 4)
5707 ctxt->dst.type = OP_NONE; /* no writeback */
5709 case 0x80 ... 0x8f: /* jnz rel, etc*/
5710 if (test_cc(ctxt->b, ctxt->eflags))
5711 rc = jmp_rel(ctxt, ctxt->src.val);
5713 case 0x90 ... 0x9f: /* setcc r/m8 */
5714 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5716 case 0xb6 ... 0xb7: /* movzx */
5717 ctxt->dst.bytes = ctxt->op_bytes;
5718 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5719 : (u16) ctxt->src.val;
5721 case 0xbe ... 0xbf: /* movsx */
5722 ctxt->dst.bytes = ctxt->op_bytes;
5723 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5724 (s16) ctxt->src.val;
5727 goto cannot_emulate;
5732 if (rc != X86EMUL_CONTINUE)
5738 return EMULATION_FAILED;
5741 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5743 invalidate_registers(ctxt);
5746 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5748 writeback_registers(ctxt);