1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include <linux/nospec.h>
25 #include "kvm_cache_regs.h"
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
28 #include <asm/debugreg.h>
29 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
164 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
165 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
166 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
167 #define NoWrite ((u64)1 << 45) /* No writeback */
168 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
169 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
170 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
171 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
172 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
173 #define NearBranch ((u64)1 << 52) /* Near branches */
174 #define No16 ((u64)1 << 53) /* No 16 bit operand */
175 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
176 #define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
178 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
180 #define X2(x...) x, x
181 #define X3(x...) X2(x), x
182 #define X4(x...) X2(x), X2(x)
183 #define X5(x...) X4(x), x
184 #define X6(x...) X4(x), X2(x)
185 #define X7(x...) X4(x), X3(x)
186 #define X8(x...) X4(x), X4(x)
187 #define X16(x...) X8(x), X8(x)
189 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
190 #define FASTOP_SIZE 8
193 * fastop functions have a special calling convention:
198 * flags: rflags (in/out)
199 * ex: rsi (in:fastop pointer, out:zero if exception)
201 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
202 * different operand sizes can be reached by calculation, rather than a jump
203 * table (which would be bigger than the code).
205 * fastop functions are declared as taking a never-defined fastop parameter,
206 * so they can't be called from C directly.
215 int (*execute)(struct x86_emulate_ctxt *ctxt);
216 const struct opcode *group;
217 const struct group_dual *gdual;
218 const struct gprefix *gprefix;
219 const struct escape *esc;
220 const struct instr_dual *idual;
221 const struct mode_dual *mdual;
222 void (*fastop)(struct fastop *fake);
224 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
228 struct opcode mod012[8];
229 struct opcode mod3[8];
233 struct opcode pfx_no;
234 struct opcode pfx_66;
235 struct opcode pfx_f2;
236 struct opcode pfx_f3;
241 struct opcode high[64];
245 struct opcode mod012;
250 struct opcode mode32;
251 struct opcode mode64;
254 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
256 enum x86_transfer_type {
258 X86_TRANSFER_CALL_JMP,
260 X86_TRANSFER_TASK_SWITCH,
263 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 if (!(ctxt->regs_valid & (1 << nr))) {
266 ctxt->regs_valid |= 1 << nr;
267 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
269 return ctxt->_regs[nr];
272 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
274 ctxt->regs_valid |= 1 << nr;
275 ctxt->regs_dirty |= 1 << nr;
276 return &ctxt->_regs[nr];
279 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 return reg_write(ctxt, nr);
285 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
289 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
290 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
293 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
295 ctxt->regs_dirty = 0;
296 ctxt->regs_valid = 0;
300 * These EFLAGS bits are restored from saved value during emulation, and
301 * any changes are written back to the saved value after emulation.
303 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
304 X86_EFLAGS_PF|X86_EFLAGS_CF)
312 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
314 #define FOP_FUNC(name) \
315 ".align " __stringify(FASTOP_SIZE) " \n\t" \
316 ".type " name ", @function \n\t" \
319 #define FOP_RET "ret \n\t"
321 #define FOP_START(op) \
322 extern void em_##op(struct fastop *fake); \
323 asm(".pushsection .text, \"ax\" \n\t" \
324 ".global em_" #op " \n\t" \
331 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
334 #define FOP1E(op, dst) \
335 FOP_FUNC(#op "_" #dst) \
336 "10: " #op " %" #dst " \n\t" FOP_RET
338 #define FOP1EEX(op, dst) \
339 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
341 #define FASTOP1(op) \
346 ON64(FOP1E(op##q, rax)) \
349 /* 1-operand, using src2 (for MUL/DIV r/m) */
350 #define FASTOP1SRC2(op, name) \
355 ON64(FOP1E(op, rcx)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
359 #define FASTOP1SRC2EX(op, name) \
364 ON64(FOP1EEX(op, rcx)) \
367 #define FOP2E(op, dst, src) \
368 FOP_FUNC(#op "_" #dst "_" #src) \
369 #op " %" #src ", %" #dst " \n\t" FOP_RET
371 #define FASTOP2(op) \
373 FOP2E(op##b, al, dl) \
374 FOP2E(op##w, ax, dx) \
375 FOP2E(op##l, eax, edx) \
376 ON64(FOP2E(op##q, rax, rdx)) \
379 /* 2 operand, word only */
380 #define FASTOP2W(op) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, src is CL */
389 #define FASTOP2CL(op) \
391 FOP2E(op##b, al, cl) \
392 FOP2E(op##w, ax, cl) \
393 FOP2E(op##l, eax, cl) \
394 ON64(FOP2E(op##q, rax, cl)) \
397 /* 2 operand, src and dest are reversed */
398 #define FASTOP2R(op, name) \
400 FOP2E(op##b, dl, al) \
401 FOP2E(op##w, dx, ax) \
402 FOP2E(op##l, edx, eax) \
403 ON64(FOP2E(op##q, rdx, rax)) \
406 #define FOP3E(op, dst, src, src2) \
407 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
408 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
410 /* 3-operand, word-only, src2=cl */
411 #define FASTOP3WCL(op) \
414 FOP3E(op##w, ax, dx, cl) \
415 FOP3E(op##l, eax, edx, cl) \
416 ON64(FOP3E(op##q, rax, rdx, cl)) \
419 /* Special case for SETcc - 1 instruction per cc */
420 #define FOP_SETCC(op) \
422 ".type " #op ", @function \n\t" \
427 asm(".global kvm_fastop_exception \n"
428 "kvm_fastop_exception: xor %esi, %esi; ret");
449 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
453 * XXX: inoutclob user must know where the argument is being expanded.
454 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
456 #define asm_safe(insn, inoutclob...) \
460 asm volatile("1:" insn "\n" \
462 ".pushsection .fixup, \"ax\"\n" \
463 "3: movl $1, %[_fault]\n" \
466 _ASM_EXTABLE(1b, 3b) \
467 : [_fault] "+qm"(_fault) inoutclob ); \
469 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
472 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
473 enum x86_intercept intercept,
474 enum x86_intercept_stage stage)
476 struct x86_instruction_info info = {
477 .intercept = intercept,
478 .rep_prefix = ctxt->rep_prefix,
479 .modrm_mod = ctxt->modrm_mod,
480 .modrm_reg = ctxt->modrm_reg,
481 .modrm_rm = ctxt->modrm_rm,
482 .src_val = ctxt->src.val64,
483 .dst_val = ctxt->dst.val64,
484 .src_bytes = ctxt->src.bytes,
485 .dst_bytes = ctxt->dst.bytes,
486 .ad_bytes = ctxt->ad_bytes,
487 .next_rip = ctxt->eip,
490 return ctxt->ops->intercept(ctxt, &info, stage);
493 static void assign_masked(ulong *dest, ulong src, ulong mask)
495 *dest = (*dest & ~mask) | (src & mask);
498 static void assign_register(unsigned long *reg, u64 val, int bytes)
500 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
503 *(u8 *)reg = (u8)val;
506 *(u16 *)reg = (u16)val;
510 break; /* 64b: zero-extend */
517 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
519 return (1UL << (ctxt->ad_bytes << 3)) - 1;
522 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
525 struct desc_struct ss;
527 if (ctxt->mode == X86EMUL_MODE_PROT64)
529 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
530 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
533 static int stack_size(struct x86_emulate_ctxt *ctxt)
535 return (__fls(stack_mask(ctxt)) + 1) >> 3;
538 /* Access/update address held in a register, based on addressing mode. */
539 static inline unsigned long
540 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
542 if (ctxt->ad_bytes == sizeof(unsigned long))
545 return reg & ad_mask(ctxt);
548 static inline unsigned long
549 register_address(struct x86_emulate_ctxt *ctxt, int reg)
551 return address_mask(ctxt, reg_read(ctxt, reg));
554 static void masked_increment(ulong *reg, ulong mask, int inc)
556 assign_masked(reg, *reg + inc, mask);
560 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
562 ulong *preg = reg_rmw(ctxt, reg);
564 assign_register(preg, *preg + inc, ctxt->ad_bytes);
567 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
569 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
572 static u32 desc_limit_scaled(struct desc_struct *desc)
574 u32 limit = get_desc_limit(desc);
576 return desc->g ? (limit << 12) | 0xfff : limit;
579 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
581 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
584 return ctxt->ops->get_cached_segment_base(ctxt, seg);
587 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
588 u32 error, bool valid)
591 ctxt->exception.vector = vec;
592 ctxt->exception.error_code = error;
593 ctxt->exception.error_code_valid = valid;
594 return X86EMUL_PROPAGATE_FAULT;
597 static int emulate_db(struct x86_emulate_ctxt *ctxt)
599 return emulate_exception(ctxt, DB_VECTOR, 0, false);
602 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
604 return emulate_exception(ctxt, GP_VECTOR, err, true);
607 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
609 return emulate_exception(ctxt, SS_VECTOR, err, true);
612 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
614 return emulate_exception(ctxt, UD_VECTOR, 0, false);
617 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
619 return emulate_exception(ctxt, TS_VECTOR, err, true);
622 static int emulate_de(struct x86_emulate_ctxt *ctxt)
624 return emulate_exception(ctxt, DE_VECTOR, 0, false);
627 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
629 return emulate_exception(ctxt, NM_VECTOR, 0, false);
632 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
635 struct desc_struct desc;
637 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
641 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
646 struct desc_struct desc;
648 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
649 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
653 * x86 defines three classes of vector instructions: explicitly
654 * aligned, explicitly unaligned, and the rest, which change behaviour
655 * depending on whether they're AVX encoded or not.
657 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
658 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
659 * 512 bytes of data must be aligned to a 16 byte boundary.
661 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
663 if (likely(size < 16))
666 if (ctxt->d & Aligned)
668 else if (ctxt->d & Unaligned)
670 else if (ctxt->d & Avx)
672 else if (ctxt->d & Aligned16)
678 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
679 struct segmented_address addr,
680 unsigned *max_size, unsigned size,
681 bool write, bool fetch,
682 enum x86emul_mode mode, ulong *linear)
684 struct desc_struct desc;
690 la = seg_base(ctxt, addr.seg) + addr.ea;
693 case X86EMUL_MODE_PROT64:
695 if (is_noncanonical_address(la))
698 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
699 if (size > *max_size)
703 *linear = la = (u32)la;
704 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
708 /* code segment in protected mode or read-only data segment */
709 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
710 || !(desc.type & 2)) && write)
712 /* unreadable code segment */
713 if (!fetch && (desc.type & 8) && !(desc.type & 2))
715 lim = desc_limit_scaled(&desc);
716 if (!(desc.type & 8) && (desc.type & 4)) {
717 /* expand-down segment */
720 lim = desc.d ? 0xffffffff : 0xffff;
724 if (lim == 0xffffffff)
727 *max_size = (u64)lim + 1 - addr.ea;
728 if (size > *max_size)
733 if (la & (insn_alignment(ctxt, size) - 1))
734 return emulate_gp(ctxt, 0);
735 return X86EMUL_CONTINUE;
737 if (addr.seg == VCPU_SREG_SS)
738 return emulate_ss(ctxt, 0);
740 return emulate_gp(ctxt, 0);
743 static int linearize(struct x86_emulate_ctxt *ctxt,
744 struct segmented_address addr,
745 unsigned size, bool write,
749 return __linearize(ctxt, addr, &max_size, size, write, false,
753 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
754 enum x86emul_mode mode)
759 struct segmented_address addr = { .seg = VCPU_SREG_CS,
762 if (ctxt->op_bytes != sizeof(unsigned long))
763 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
764 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
765 if (rc == X86EMUL_CONTINUE)
766 ctxt->_eip = addr.ea;
770 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
772 return assign_eip(ctxt, dst, ctxt->mode);
775 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
776 const struct desc_struct *cs_desc)
778 enum x86emul_mode mode = ctxt->mode;
782 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
786 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
788 mode = X86EMUL_MODE_PROT64;
790 mode = X86EMUL_MODE_PROT32; /* temporary value */
793 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
794 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
795 rc = assign_eip(ctxt, dst, mode);
796 if (rc == X86EMUL_CONTINUE)
801 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
803 return assign_eip_near(ctxt, ctxt->_eip + rel);
806 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
807 void *data, unsigned size)
809 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
812 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
813 ulong linear, void *data,
816 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
819 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
820 struct segmented_address addr,
827 rc = linearize(ctxt, addr, size, false, &linear);
828 if (rc != X86EMUL_CONTINUE)
830 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
833 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
834 struct segmented_address addr,
841 rc = linearize(ctxt, addr, size, true, &linear);
842 if (rc != X86EMUL_CONTINUE)
844 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
848 * Prefetch the remaining bytes of the instruction without crossing page
849 * boundary if they are not in fetch_cache yet.
851 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
854 unsigned size, max_size;
855 unsigned long linear;
856 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
857 struct segmented_address addr = { .seg = VCPU_SREG_CS,
858 .ea = ctxt->eip + cur_size };
861 * We do not know exactly how many bytes will be needed, and
862 * __linearize is expensive, so fetch as much as possible. We
863 * just have to avoid going beyond the 15 byte limit, the end
864 * of the segment, or the end of the page.
866 * __linearize is called with size 0 so that it does not do any
867 * boundary check itself. Instead, we use max_size to check
870 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
872 if (unlikely(rc != X86EMUL_CONTINUE))
875 size = min_t(unsigned, 15UL ^ cur_size, max_size);
876 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
879 * One instruction can only straddle two pages,
880 * and one has been loaded at the beginning of
881 * x86_decode_insn. So, if not enough bytes
882 * still, we must have hit the 15-byte boundary.
884 if (unlikely(size < op_size))
885 return emulate_gp(ctxt, 0);
887 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
888 size, &ctxt->exception);
889 if (unlikely(rc != X86EMUL_CONTINUE))
891 ctxt->fetch.end += size;
892 return X86EMUL_CONTINUE;
895 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
898 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
900 if (unlikely(done_size < size))
901 return __do_insn_fetch_bytes(ctxt, size - done_size);
903 return X86EMUL_CONTINUE;
906 /* Fetch next part of the instruction being emulated. */
907 #define insn_fetch(_type, _ctxt) \
910 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
911 if (rc != X86EMUL_CONTINUE) \
913 ctxt->_eip += sizeof(_type); \
914 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
915 ctxt->fetch.ptr += sizeof(_type); \
919 #define insn_fetch_arr(_arr, _size, _ctxt) \
921 rc = do_insn_fetch_bytes(_ctxt, _size); \
922 if (rc != X86EMUL_CONTINUE) \
924 ctxt->_eip += (_size); \
925 memcpy(_arr, ctxt->fetch.ptr, _size); \
926 ctxt->fetch.ptr += (_size); \
930 * Given the 'reg' portion of a ModRM byte, and a register block, return a
931 * pointer into the block that addresses the relevant register.
932 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
934 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
938 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
940 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
941 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
943 p = reg_rmw(ctxt, modrm_reg);
947 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
948 struct segmented_address addr,
949 u16 *size, unsigned long *address, int op_bytes)
956 rc = segmented_read_std(ctxt, addr, size, 2);
957 if (rc != X86EMUL_CONTINUE)
960 rc = segmented_read_std(ctxt, addr, address, op_bytes);
974 FASTOP1SRC2(mul, mul_ex);
975 FASTOP1SRC2(imul, imul_ex);
976 FASTOP1SRC2EX(div, div_ex);
977 FASTOP1SRC2EX(idiv, idiv_ex);
1006 FASTOP2R(cmp, cmp_r);
1008 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1010 /* If src is zero, do not writeback, but update flags */
1011 if (ctxt->src.val == 0)
1012 ctxt->dst.type = OP_NONE;
1013 return fastop(ctxt, em_bsf);
1016 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1018 /* If src is zero, do not writeback, but update flags */
1019 if (ctxt->src.val == 0)
1020 ctxt->dst.type = OP_NONE;
1021 return fastop(ctxt, em_bsr);
1024 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1027 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1029 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1030 asm("push %[flags]; popf; " CALL_NOSPEC
1031 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1035 static void fetch_register_operand(struct operand *op)
1037 switch (op->bytes) {
1039 op->val = *(u8 *)op->addr.reg;
1042 op->val = *(u16 *)op->addr.reg;
1045 op->val = *(u32 *)op->addr.reg;
1048 op->val = *(u64 *)op->addr.reg;
1053 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1055 ctxt->ops->get_fpu(ctxt);
1057 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1058 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1059 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1060 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1061 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1062 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1063 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1064 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1065 #ifdef CONFIG_X86_64
1066 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1067 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1068 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1069 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1070 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1071 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1072 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1073 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1077 ctxt->ops->put_fpu(ctxt);
1080 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1083 ctxt->ops->get_fpu(ctxt);
1085 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1086 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1087 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1088 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1089 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1090 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1091 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1092 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1093 #ifdef CONFIG_X86_64
1094 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1095 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1096 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1097 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1098 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1099 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1100 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1101 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1105 ctxt->ops->put_fpu(ctxt);
1108 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1110 ctxt->ops->get_fpu(ctxt);
1112 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1113 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1114 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1115 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1116 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1117 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1118 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1119 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1122 ctxt->ops->put_fpu(ctxt);
1125 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1127 ctxt->ops->get_fpu(ctxt);
1129 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1130 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1131 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1132 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1133 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1134 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1135 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1136 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1139 ctxt->ops->put_fpu(ctxt);
1142 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1144 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1145 return emulate_nm(ctxt);
1147 ctxt->ops->get_fpu(ctxt);
1148 asm volatile("fninit");
1149 ctxt->ops->put_fpu(ctxt);
1150 return X86EMUL_CONTINUE;
1153 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1157 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1158 return emulate_nm(ctxt);
1160 ctxt->ops->get_fpu(ctxt);
1161 asm volatile("fnstcw %0": "+m"(fcw));
1162 ctxt->ops->put_fpu(ctxt);
1164 ctxt->dst.val = fcw;
1166 return X86EMUL_CONTINUE;
1169 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1173 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1174 return emulate_nm(ctxt);
1176 ctxt->ops->get_fpu(ctxt);
1177 asm volatile("fnstsw %0": "+m"(fsw));
1178 ctxt->ops->put_fpu(ctxt);
1180 ctxt->dst.val = fsw;
1182 return X86EMUL_CONTINUE;
1185 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1188 unsigned reg = ctxt->modrm_reg;
1190 if (!(ctxt->d & ModRM))
1191 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1193 if (ctxt->d & Sse) {
1197 read_sse_reg(ctxt, &op->vec_val, reg);
1200 if (ctxt->d & Mmx) {
1209 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1210 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1212 fetch_register_operand(op);
1213 op->orig_val = op->val;
1216 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1218 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1219 ctxt->modrm_seg = VCPU_SREG_SS;
1222 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1226 int index_reg, base_reg, scale;
1227 int rc = X86EMUL_CONTINUE;
1230 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1231 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1232 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1234 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1235 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1236 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1237 ctxt->modrm_seg = VCPU_SREG_DS;
1239 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1241 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1242 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1244 if (ctxt->d & Sse) {
1247 op->addr.xmm = ctxt->modrm_rm;
1248 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1251 if (ctxt->d & Mmx) {
1254 op->addr.mm = ctxt->modrm_rm & 7;
1257 fetch_register_operand(op);
1263 if (ctxt->ad_bytes == 2) {
1264 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1265 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1266 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1267 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1269 /* 16-bit ModR/M decode. */
1270 switch (ctxt->modrm_mod) {
1272 if (ctxt->modrm_rm == 6)
1273 modrm_ea += insn_fetch(u16, ctxt);
1276 modrm_ea += insn_fetch(s8, ctxt);
1279 modrm_ea += insn_fetch(u16, ctxt);
1282 switch (ctxt->modrm_rm) {
1284 modrm_ea += bx + si;
1287 modrm_ea += bx + di;
1290 modrm_ea += bp + si;
1293 modrm_ea += bp + di;
1302 if (ctxt->modrm_mod != 0)
1309 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1310 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1311 ctxt->modrm_seg = VCPU_SREG_SS;
1312 modrm_ea = (u16)modrm_ea;
1314 /* 32/64-bit ModR/M decode. */
1315 if ((ctxt->modrm_rm & 7) == 4) {
1316 sib = insn_fetch(u8, ctxt);
1317 index_reg |= (sib >> 3) & 7;
1318 base_reg |= sib & 7;
1321 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1322 modrm_ea += insn_fetch(s32, ctxt);
1324 modrm_ea += reg_read(ctxt, base_reg);
1325 adjust_modrm_seg(ctxt, base_reg);
1326 /* Increment ESP on POP [ESP] */
1327 if ((ctxt->d & IncSP) &&
1328 base_reg == VCPU_REGS_RSP)
1329 modrm_ea += ctxt->op_bytes;
1332 modrm_ea += reg_read(ctxt, index_reg) << scale;
1333 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1334 modrm_ea += insn_fetch(s32, ctxt);
1335 if (ctxt->mode == X86EMUL_MODE_PROT64)
1336 ctxt->rip_relative = 1;
1338 base_reg = ctxt->modrm_rm;
1339 modrm_ea += reg_read(ctxt, base_reg);
1340 adjust_modrm_seg(ctxt, base_reg);
1342 switch (ctxt->modrm_mod) {
1344 modrm_ea += insn_fetch(s8, ctxt);
1347 modrm_ea += insn_fetch(s32, ctxt);
1351 op->addr.mem.ea = modrm_ea;
1352 if (ctxt->ad_bytes != 8)
1353 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1359 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1362 int rc = X86EMUL_CONTINUE;
1365 switch (ctxt->ad_bytes) {
1367 op->addr.mem.ea = insn_fetch(u16, ctxt);
1370 op->addr.mem.ea = insn_fetch(u32, ctxt);
1373 op->addr.mem.ea = insn_fetch(u64, ctxt);
1380 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1384 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1385 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1387 if (ctxt->src.bytes == 2)
1388 sv = (s16)ctxt->src.val & (s16)mask;
1389 else if (ctxt->src.bytes == 4)
1390 sv = (s32)ctxt->src.val & (s32)mask;
1392 sv = (s64)ctxt->src.val & (s64)mask;
1394 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1395 ctxt->dst.addr.mem.ea + (sv >> 3));
1398 /* only subword offset */
1399 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1402 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1403 unsigned long addr, void *dest, unsigned size)
1406 struct read_cache *mc = &ctxt->mem_read;
1408 if (mc->pos < mc->end)
1411 WARN_ON((mc->end + size) >= sizeof(mc->data));
1413 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1415 if (rc != X86EMUL_CONTINUE)
1421 memcpy(dest, mc->data + mc->pos, size);
1423 return X86EMUL_CONTINUE;
1426 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1427 struct segmented_address addr,
1434 rc = linearize(ctxt, addr, size, false, &linear);
1435 if (rc != X86EMUL_CONTINUE)
1437 return read_emulated(ctxt, linear, data, size);
1440 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1441 struct segmented_address addr,
1448 rc = linearize(ctxt, addr, size, true, &linear);
1449 if (rc != X86EMUL_CONTINUE)
1451 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1455 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1456 struct segmented_address addr,
1457 const void *orig_data, const void *data,
1463 rc = linearize(ctxt, addr, size, true, &linear);
1464 if (rc != X86EMUL_CONTINUE)
1466 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1467 size, &ctxt->exception);
1470 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1471 unsigned int size, unsigned short port,
1474 struct read_cache *rc = &ctxt->io_read;
1476 if (rc->pos == rc->end) { /* refill pio read ahead */
1477 unsigned int in_page, n;
1478 unsigned int count = ctxt->rep_prefix ?
1479 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1480 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1481 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1482 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1483 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1486 rc->pos = rc->end = 0;
1487 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1492 if (ctxt->rep_prefix && (ctxt->d & String) &&
1493 !(ctxt->eflags & X86_EFLAGS_DF)) {
1494 ctxt->dst.data = rc->data + rc->pos;
1495 ctxt->dst.type = OP_MEM_STR;
1496 ctxt->dst.count = (rc->end - rc->pos) / size;
1499 memcpy(dest, rc->data + rc->pos, size);
1505 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1506 u16 index, struct desc_struct *desc)
1511 ctxt->ops->get_idt(ctxt, &dt);
1513 if (dt.size < index * 8 + 7)
1514 return emulate_gp(ctxt, index << 3 | 0x2);
1516 addr = dt.address + index * 8;
1517 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1520 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1521 u16 selector, struct desc_ptr *dt)
1523 const struct x86_emulate_ops *ops = ctxt->ops;
1526 if (selector & 1 << 2) {
1527 struct desc_struct desc;
1530 memset (dt, 0, sizeof *dt);
1531 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1535 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1536 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1538 ops->get_gdt(ctxt, dt);
1541 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1542 u16 selector, ulong *desc_addr_p)
1545 u16 index = selector >> 3;
1548 get_descriptor_table_ptr(ctxt, selector, &dt);
1550 if (dt.size < index * 8 + 7)
1551 return emulate_gp(ctxt, selector & 0xfffc);
1553 addr = dt.address + index * 8;
1555 #ifdef CONFIG_X86_64
1556 if (addr >> 32 != 0) {
1559 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1560 if (!(efer & EFER_LMA))
1565 *desc_addr_p = addr;
1566 return X86EMUL_CONTINUE;
1569 /* allowed just for 8 bytes segments */
1570 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1571 u16 selector, struct desc_struct *desc,
1576 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1577 if (rc != X86EMUL_CONTINUE)
1580 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1583 /* allowed just for 8 bytes segments */
1584 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1585 u16 selector, struct desc_struct *desc)
1590 rc = get_descriptor_ptr(ctxt, selector, &addr);
1591 if (rc != X86EMUL_CONTINUE)
1594 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1597 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1598 u16 selector, int seg, u8 cpl,
1599 enum x86_transfer_type transfer,
1600 struct desc_struct *desc)
1602 struct desc_struct seg_desc, old_desc;
1604 unsigned err_vec = GP_VECTOR;
1606 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1612 memset(&seg_desc, 0, sizeof seg_desc);
1614 if (ctxt->mode == X86EMUL_MODE_REAL) {
1615 /* set real mode segment descriptor (keep limit etc. for
1617 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1618 set_desc_base(&seg_desc, selector << 4);
1620 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1621 /* VM86 needs a clean new segment descriptor */
1622 set_desc_base(&seg_desc, selector << 4);
1623 set_desc_limit(&seg_desc, 0xffff);
1633 /* TR should be in GDT only */
1634 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1637 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1638 if (null_selector) {
1639 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1642 if (seg == VCPU_SREG_SS) {
1643 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1647 * ctxt->ops->set_segment expects the CPL to be in
1648 * SS.DPL, so fake an expand-up 32-bit data segment.
1658 /* Skip all following checks */
1662 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1663 if (ret != X86EMUL_CONTINUE)
1666 err_code = selector & 0xfffc;
1667 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1670 /* can't load system descriptor into segment selector */
1671 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1672 if (transfer == X86_TRANSFER_CALL_JMP)
1673 return X86EMUL_UNHANDLEABLE;
1678 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1687 * segment is not a writable data segment or segment
1688 * selector's RPL != CPL or segment selector's RPL != CPL
1690 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1694 if (!(seg_desc.type & 8))
1697 if (seg_desc.type & 4) {
1703 if (rpl > cpl || dpl != cpl)
1706 /* in long-mode d/b must be clear if l is set */
1707 if (seg_desc.d && seg_desc.l) {
1710 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1711 if (efer & EFER_LMA)
1715 /* CS(RPL) <- CPL */
1716 selector = (selector & 0xfffc) | cpl;
1719 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1721 old_desc = seg_desc;
1722 seg_desc.type |= 2; /* busy */
1723 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1724 sizeof(seg_desc), &ctxt->exception);
1725 if (ret != X86EMUL_CONTINUE)
1728 case VCPU_SREG_LDTR:
1729 if (seg_desc.s || seg_desc.type != 2)
1732 default: /* DS, ES, FS, or GS */
1734 * segment is not a data or readable code segment or
1735 * ((segment is a data or nonconforming code segment)
1736 * and (both RPL and CPL > DPL))
1738 if ((seg_desc.type & 0xa) == 0x8 ||
1739 (((seg_desc.type & 0xc) != 0xc) &&
1740 (rpl > dpl && cpl > dpl)))
1746 /* mark segment as accessed */
1747 if (!(seg_desc.type & 1)) {
1749 ret = write_segment_descriptor(ctxt, selector,
1751 if (ret != X86EMUL_CONTINUE)
1754 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1755 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1756 if (ret != X86EMUL_CONTINUE)
1758 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1759 ((u64)base3 << 32)))
1760 return emulate_gp(ctxt, 0);
1763 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1766 return X86EMUL_CONTINUE;
1768 return emulate_exception(ctxt, err_vec, err_code, true);
1771 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1772 u16 selector, int seg)
1774 u8 cpl = ctxt->ops->cpl(ctxt);
1777 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1778 * they can load it at CPL<3 (Intel's manual says only LSS can,
1781 * However, the Intel manual says that putting IST=1/DPL=3 in
1782 * an interrupt gate will result in SS=3 (the AMD manual instead
1783 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1784 * and only forbid it here.
1786 if (seg == VCPU_SREG_SS && selector == 3 &&
1787 ctxt->mode == X86EMUL_MODE_PROT64)
1788 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1790 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1791 X86_TRANSFER_NONE, NULL);
1794 static void write_register_operand(struct operand *op)
1796 return assign_register(op->addr.reg, op->val, op->bytes);
1799 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1803 write_register_operand(op);
1806 if (ctxt->lock_prefix)
1807 return segmented_cmpxchg(ctxt,
1813 return segmented_write(ctxt,
1819 return segmented_write(ctxt,
1822 op->bytes * op->count);
1825 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1828 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1836 return X86EMUL_CONTINUE;
1839 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1841 struct segmented_address addr;
1843 rsp_increment(ctxt, -bytes);
1844 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1845 addr.seg = VCPU_SREG_SS;
1847 return segmented_write(ctxt, addr, data, bytes);
1850 static int em_push(struct x86_emulate_ctxt *ctxt)
1852 /* Disable writeback. */
1853 ctxt->dst.type = OP_NONE;
1854 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1857 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1858 void *dest, int len)
1861 struct segmented_address addr;
1863 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1864 addr.seg = VCPU_SREG_SS;
1865 rc = segmented_read(ctxt, addr, dest, len);
1866 if (rc != X86EMUL_CONTINUE)
1869 rsp_increment(ctxt, len);
1873 static int em_pop(struct x86_emulate_ctxt *ctxt)
1875 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1878 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1879 void *dest, int len)
1882 unsigned long val, change_mask;
1883 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1884 int cpl = ctxt->ops->cpl(ctxt);
1886 rc = emulate_pop(ctxt, &val, len);
1887 if (rc != X86EMUL_CONTINUE)
1890 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1891 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1892 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1893 X86_EFLAGS_AC | X86_EFLAGS_ID;
1895 switch(ctxt->mode) {
1896 case X86EMUL_MODE_PROT64:
1897 case X86EMUL_MODE_PROT32:
1898 case X86EMUL_MODE_PROT16:
1900 change_mask |= X86_EFLAGS_IOPL;
1902 change_mask |= X86_EFLAGS_IF;
1904 case X86EMUL_MODE_VM86:
1906 return emulate_gp(ctxt, 0);
1907 change_mask |= X86_EFLAGS_IF;
1909 default: /* real mode */
1910 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1914 *(unsigned long *)dest =
1915 (ctxt->eflags & ~change_mask) | (val & change_mask);
1920 static int em_popf(struct x86_emulate_ctxt *ctxt)
1922 ctxt->dst.type = OP_REG;
1923 ctxt->dst.addr.reg = &ctxt->eflags;
1924 ctxt->dst.bytes = ctxt->op_bytes;
1925 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1928 static int em_enter(struct x86_emulate_ctxt *ctxt)
1931 unsigned frame_size = ctxt->src.val;
1932 unsigned nesting_level = ctxt->src2.val & 31;
1936 return X86EMUL_UNHANDLEABLE;
1938 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1939 rc = push(ctxt, &rbp, stack_size(ctxt));
1940 if (rc != X86EMUL_CONTINUE)
1942 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1944 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1945 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1947 return X86EMUL_CONTINUE;
1950 static int em_leave(struct x86_emulate_ctxt *ctxt)
1952 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1954 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1957 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1959 int seg = ctxt->src2.val;
1961 ctxt->src.val = get_segment_selector(ctxt, seg);
1962 if (ctxt->op_bytes == 4) {
1963 rsp_increment(ctxt, -2);
1967 return em_push(ctxt);
1970 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1972 int seg = ctxt->src2.val;
1973 unsigned long selector;
1976 rc = emulate_pop(ctxt, &selector, 2);
1977 if (rc != X86EMUL_CONTINUE)
1980 if (ctxt->modrm_reg == VCPU_SREG_SS)
1981 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1982 if (ctxt->op_bytes > 2)
1983 rsp_increment(ctxt, ctxt->op_bytes - 2);
1985 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1989 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1991 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1992 int rc = X86EMUL_CONTINUE;
1993 int reg = VCPU_REGS_RAX;
1995 while (reg <= VCPU_REGS_RDI) {
1996 (reg == VCPU_REGS_RSP) ?
1997 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2000 if (rc != X86EMUL_CONTINUE)
2009 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2011 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2012 return em_push(ctxt);
2015 static int em_popa(struct x86_emulate_ctxt *ctxt)
2017 int rc = X86EMUL_CONTINUE;
2018 int reg = VCPU_REGS_RDI;
2021 while (reg >= VCPU_REGS_RAX) {
2022 if (reg == VCPU_REGS_RSP) {
2023 rsp_increment(ctxt, ctxt->op_bytes);
2027 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2028 if (rc != X86EMUL_CONTINUE)
2030 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2036 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2038 const struct x86_emulate_ops *ops = ctxt->ops;
2045 /* TODO: Add limit checks */
2046 ctxt->src.val = ctxt->eflags;
2048 if (rc != X86EMUL_CONTINUE)
2051 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2053 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2055 if (rc != X86EMUL_CONTINUE)
2058 ctxt->src.val = ctxt->_eip;
2060 if (rc != X86EMUL_CONTINUE)
2063 ops->get_idt(ctxt, &dt);
2065 eip_addr = dt.address + (irq << 2);
2066 cs_addr = dt.address + (irq << 2) + 2;
2068 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2069 if (rc != X86EMUL_CONTINUE)
2072 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2073 if (rc != X86EMUL_CONTINUE)
2076 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2077 if (rc != X86EMUL_CONTINUE)
2085 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2089 invalidate_registers(ctxt);
2090 rc = __emulate_int_real(ctxt, irq);
2091 if (rc == X86EMUL_CONTINUE)
2092 writeback_registers(ctxt);
2096 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2098 switch(ctxt->mode) {
2099 case X86EMUL_MODE_REAL:
2100 return __emulate_int_real(ctxt, irq);
2101 case X86EMUL_MODE_VM86:
2102 case X86EMUL_MODE_PROT16:
2103 case X86EMUL_MODE_PROT32:
2104 case X86EMUL_MODE_PROT64:
2106 /* Protected mode interrupts unimplemented yet */
2107 return X86EMUL_UNHANDLEABLE;
2111 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2113 int rc = X86EMUL_CONTINUE;
2114 unsigned long temp_eip = 0;
2115 unsigned long temp_eflags = 0;
2116 unsigned long cs = 0;
2117 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2118 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2119 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2120 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2121 X86_EFLAGS_AC | X86_EFLAGS_ID |
2123 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2126 /* TODO: Add stack limit check */
2128 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2130 if (rc != X86EMUL_CONTINUE)
2133 if (temp_eip & ~0xffff)
2134 return emulate_gp(ctxt, 0);
2136 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2138 if (rc != X86EMUL_CONTINUE)
2141 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2143 if (rc != X86EMUL_CONTINUE)
2146 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2148 if (rc != X86EMUL_CONTINUE)
2151 ctxt->_eip = temp_eip;
2153 if (ctxt->op_bytes == 4)
2154 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2155 else if (ctxt->op_bytes == 2) {
2156 ctxt->eflags &= ~0xffff;
2157 ctxt->eflags |= temp_eflags;
2160 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2161 ctxt->eflags |= X86_EFLAGS_FIXED;
2162 ctxt->ops->set_nmi_mask(ctxt, false);
2167 static int em_iret(struct x86_emulate_ctxt *ctxt)
2169 switch(ctxt->mode) {
2170 case X86EMUL_MODE_REAL:
2171 return emulate_iret_real(ctxt);
2172 case X86EMUL_MODE_VM86:
2173 case X86EMUL_MODE_PROT16:
2174 case X86EMUL_MODE_PROT32:
2175 case X86EMUL_MODE_PROT64:
2177 /* iret from protected mode unimplemented yet */
2178 return X86EMUL_UNHANDLEABLE;
2182 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2186 struct desc_struct new_desc;
2187 u8 cpl = ctxt->ops->cpl(ctxt);
2189 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2191 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2192 X86_TRANSFER_CALL_JMP,
2194 if (rc != X86EMUL_CONTINUE)
2197 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2198 /* Error handling is not implemented. */
2199 if (rc != X86EMUL_CONTINUE)
2200 return X86EMUL_UNHANDLEABLE;
2205 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2207 return assign_eip_near(ctxt, ctxt->src.val);
2210 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2215 old_eip = ctxt->_eip;
2216 rc = assign_eip_near(ctxt, ctxt->src.val);
2217 if (rc != X86EMUL_CONTINUE)
2219 ctxt->src.val = old_eip;
2224 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2226 u64 old = ctxt->dst.orig_val64;
2228 if (ctxt->dst.bytes == 16)
2229 return X86EMUL_UNHANDLEABLE;
2231 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2232 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2233 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2234 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2235 ctxt->eflags &= ~X86_EFLAGS_ZF;
2237 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2238 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2240 ctxt->eflags |= X86_EFLAGS_ZF;
2242 return X86EMUL_CONTINUE;
2245 static int em_ret(struct x86_emulate_ctxt *ctxt)
2250 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2251 if (rc != X86EMUL_CONTINUE)
2254 return assign_eip_near(ctxt, eip);
2257 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2260 unsigned long eip, cs;
2261 int cpl = ctxt->ops->cpl(ctxt);
2262 struct desc_struct new_desc;
2264 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2265 if (rc != X86EMUL_CONTINUE)
2267 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2268 if (rc != X86EMUL_CONTINUE)
2270 /* Outer-privilege level return is not implemented */
2271 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2272 return X86EMUL_UNHANDLEABLE;
2273 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2276 if (rc != X86EMUL_CONTINUE)
2278 rc = assign_eip_far(ctxt, eip, &new_desc);
2279 /* Error handling is not implemented. */
2280 if (rc != X86EMUL_CONTINUE)
2281 return X86EMUL_UNHANDLEABLE;
2286 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2290 rc = em_ret_far(ctxt);
2291 if (rc != X86EMUL_CONTINUE)
2293 rsp_increment(ctxt, ctxt->src.val);
2294 return X86EMUL_CONTINUE;
2297 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2299 /* Save real source value, then compare EAX against destination. */
2300 ctxt->dst.orig_val = ctxt->dst.val;
2301 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2302 ctxt->src.orig_val = ctxt->src.val;
2303 ctxt->src.val = ctxt->dst.orig_val;
2304 fastop(ctxt, em_cmp);
2306 if (ctxt->eflags & X86_EFLAGS_ZF) {
2307 /* Success: write back to memory; no update of EAX */
2308 ctxt->src.type = OP_NONE;
2309 ctxt->dst.val = ctxt->src.orig_val;
2311 /* Failure: write the value we saw to EAX. */
2312 ctxt->src.type = OP_REG;
2313 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2314 ctxt->src.val = ctxt->dst.orig_val;
2315 /* Create write-cycle to dest by writing the same value */
2316 ctxt->dst.val = ctxt->dst.orig_val;
2318 return X86EMUL_CONTINUE;
2321 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2323 int seg = ctxt->src2.val;
2327 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2329 rc = load_segment_descriptor(ctxt, sel, seg);
2330 if (rc != X86EMUL_CONTINUE)
2333 ctxt->dst.val = ctxt->src.val;
2337 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2339 u32 eax, ebx, ecx, edx;
2343 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2344 return edx & bit(X86_FEATURE_LM);
2347 #define GET_SMSTATE(type, smbase, offset) \
2350 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2352 if (r != X86EMUL_CONTINUE) \
2353 return X86EMUL_UNHANDLEABLE; \
2357 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2359 desc->g = (flags >> 23) & 1;
2360 desc->d = (flags >> 22) & 1;
2361 desc->l = (flags >> 21) & 1;
2362 desc->avl = (flags >> 20) & 1;
2363 desc->p = (flags >> 15) & 1;
2364 desc->dpl = (flags >> 13) & 3;
2365 desc->s = (flags >> 12) & 1;
2366 desc->type = (flags >> 8) & 15;
2369 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2371 struct desc_struct desc;
2375 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2378 offset = 0x7f84 + n * 12;
2380 offset = 0x7f2c + (n - 3) * 12;
2382 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2383 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2384 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2385 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2386 return X86EMUL_CONTINUE;
2389 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2391 struct desc_struct desc;
2396 offset = 0x7e00 + n * 16;
2398 selector = GET_SMSTATE(u16, smbase, offset);
2399 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2400 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2401 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2402 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2404 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2405 return X86EMUL_CONTINUE;
2408 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2409 u64 cr0, u64 cr3, u64 cr4)
2414 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2416 if (cr4 & X86_CR4_PCIDE) {
2421 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2423 return X86EMUL_UNHANDLEABLE;
2426 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2427 * Then enable protected mode. However, PCID cannot be enabled
2428 * if EFER.LMA=0, so set it separately.
2430 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2432 return X86EMUL_UNHANDLEABLE;
2434 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2436 return X86EMUL_UNHANDLEABLE;
2438 if (cr4 & X86_CR4_PCIDE) {
2439 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2441 return X86EMUL_UNHANDLEABLE;
2443 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2445 return X86EMUL_UNHANDLEABLE;
2450 return X86EMUL_CONTINUE;
2453 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2455 struct desc_struct desc;
2458 u32 val, cr0, cr3, cr4;
2461 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2462 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2463 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2464 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2466 for (i = 0; i < 8; i++)
2467 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2469 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2470 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2471 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2472 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2474 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2475 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2476 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2477 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2478 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2480 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2481 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2482 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2483 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2484 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2486 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2487 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2488 ctxt->ops->set_gdt(ctxt, &dt);
2490 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2491 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2492 ctxt->ops->set_idt(ctxt, &dt);
2494 for (i = 0; i < 6; i++) {
2495 int r = rsm_load_seg_32(ctxt, smbase, i);
2496 if (r != X86EMUL_CONTINUE)
2500 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2502 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2504 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2507 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2509 struct desc_struct desc;
2511 u64 val, cr0, cr3, cr4;
2516 for (i = 0; i < 16; i++)
2517 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2519 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2520 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2522 val = GET_SMSTATE(u32, smbase, 0x7f68);
2523 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2524 val = GET_SMSTATE(u32, smbase, 0x7f60);
2525 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2527 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2528 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2529 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2530 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2531 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2532 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2534 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2535 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2536 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2537 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2538 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2539 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2541 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2542 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2543 ctxt->ops->set_idt(ctxt, &dt);
2545 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2546 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2547 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2548 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2549 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2550 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2552 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2553 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2554 ctxt->ops->set_gdt(ctxt, &dt);
2556 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2557 if (r != X86EMUL_CONTINUE)
2560 for (i = 0; i < 6; i++) {
2561 r = rsm_load_seg_64(ctxt, smbase, i);
2562 if (r != X86EMUL_CONTINUE)
2566 return X86EMUL_CONTINUE;
2569 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2571 unsigned long cr0, cr4, efer;
2575 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2576 return emulate_ud(ctxt);
2579 * Get back to real mode, to prepare a safe state in which to load
2580 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2581 * supports long mode.
2583 if (emulator_has_longmode(ctxt)) {
2584 struct desc_struct cs_desc;
2586 /* Zero CR4.PCIDE before CR0.PG. */
2587 cr4 = ctxt->ops->get_cr(ctxt, 4);
2588 if (cr4 & X86_CR4_PCIDE)
2589 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2591 /* A 32-bit code segment is required to clear EFER.LMA. */
2592 memset(&cs_desc, 0, sizeof(cs_desc));
2594 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2595 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2598 /* For the 64-bit case, this will clear EFER.LMA. */
2599 cr0 = ctxt->ops->get_cr(ctxt, 0);
2600 if (cr0 & X86_CR0_PE)
2601 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2603 if (emulator_has_longmode(ctxt)) {
2604 /* Clear CR4.PAE before clearing EFER.LME. */
2605 cr4 = ctxt->ops->get_cr(ctxt, 4);
2606 if (cr4 & X86_CR4_PAE)
2607 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2609 /* And finally go back to 32-bit mode. */
2611 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2614 smbase = ctxt->ops->get_smbase(ctxt);
2615 if (emulator_has_longmode(ctxt))
2616 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2618 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2620 if (ret != X86EMUL_CONTINUE) {
2621 /* FIXME: should triple fault */
2622 return X86EMUL_UNHANDLEABLE;
2625 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2626 ctxt->ops->set_nmi_mask(ctxt, false);
2628 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2629 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2630 return X86EMUL_CONTINUE;
2634 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2635 struct desc_struct *cs, struct desc_struct *ss)
2637 cs->l = 0; /* will be adjusted later */
2638 set_desc_base(cs, 0); /* flat segment */
2639 cs->g = 1; /* 4kb granularity */
2640 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2641 cs->type = 0x0b; /* Read, Execute, Accessed */
2643 cs->dpl = 0; /* will be adjusted later */
2648 set_desc_base(ss, 0); /* flat segment */
2649 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2650 ss->g = 1; /* 4kb granularity */
2652 ss->type = 0x03; /* Read/Write, Accessed */
2653 ss->d = 1; /* 32bit stack segment */
2660 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2662 u32 eax, ebx, ecx, edx;
2665 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2666 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2667 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2668 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2671 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2673 const struct x86_emulate_ops *ops = ctxt->ops;
2674 u32 eax, ebx, ecx, edx;
2677 * syscall should always be enabled in longmode - so only become
2678 * vendor specific (cpuid) if other modes are active...
2680 if (ctxt->mode == X86EMUL_MODE_PROT64)
2685 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2687 * Intel ("GenuineIntel")
2688 * remark: Intel CPUs only support "syscall" in 64bit
2689 * longmode. Also an 64bit guest with a
2690 * 32bit compat-app running will #UD !! While this
2691 * behaviour can be fixed (by emulating) into AMD
2692 * response - CPUs of AMD can't behave like Intel.
2694 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2695 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2696 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2699 /* AMD ("AuthenticAMD") */
2700 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2701 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2702 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2705 /* AMD ("AMDisbetter!") */
2706 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2707 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2708 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2711 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2715 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2717 const struct x86_emulate_ops *ops = ctxt->ops;
2718 struct desc_struct cs, ss;
2723 /* syscall is not available in real mode */
2724 if (ctxt->mode == X86EMUL_MODE_REAL ||
2725 ctxt->mode == X86EMUL_MODE_VM86)
2726 return emulate_ud(ctxt);
2728 if (!(em_syscall_is_enabled(ctxt)))
2729 return emulate_ud(ctxt);
2731 ops->get_msr(ctxt, MSR_EFER, &efer);
2732 setup_syscalls_segments(ctxt, &cs, &ss);
2734 if (!(efer & EFER_SCE))
2735 return emulate_ud(ctxt);
2737 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2739 cs_sel = (u16)(msr_data & 0xfffc);
2740 ss_sel = (u16)(msr_data + 8);
2742 if (efer & EFER_LMA) {
2746 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2747 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2749 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2750 if (efer & EFER_LMA) {
2751 #ifdef CONFIG_X86_64
2752 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2755 ctxt->mode == X86EMUL_MODE_PROT64 ?
2756 MSR_LSTAR : MSR_CSTAR, &msr_data);
2757 ctxt->_eip = msr_data;
2759 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2760 ctxt->eflags &= ~msr_data;
2761 ctxt->eflags |= X86_EFLAGS_FIXED;
2765 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2766 ctxt->_eip = (u32)msr_data;
2768 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2771 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2772 return X86EMUL_CONTINUE;
2775 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2777 const struct x86_emulate_ops *ops = ctxt->ops;
2778 struct desc_struct cs, ss;
2783 ops->get_msr(ctxt, MSR_EFER, &efer);
2784 /* inject #GP if in real mode */
2785 if (ctxt->mode == X86EMUL_MODE_REAL)
2786 return emulate_gp(ctxt, 0);
2789 * Not recognized on AMD in compat mode (but is recognized in legacy
2792 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2793 && !vendor_intel(ctxt))
2794 return emulate_ud(ctxt);
2796 /* sysenter/sysexit have not been tested in 64bit mode. */
2797 if (ctxt->mode == X86EMUL_MODE_PROT64)
2798 return X86EMUL_UNHANDLEABLE;
2800 setup_syscalls_segments(ctxt, &cs, &ss);
2802 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2803 if ((msr_data & 0xfffc) == 0x0)
2804 return emulate_gp(ctxt, 0);
2806 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2807 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2808 ss_sel = cs_sel + 8;
2809 if (efer & EFER_LMA) {
2814 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2815 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2817 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2818 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2820 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2821 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2824 return X86EMUL_CONTINUE;
2827 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2829 const struct x86_emulate_ops *ops = ctxt->ops;
2830 struct desc_struct cs, ss;
2831 u64 msr_data, rcx, rdx;
2833 u16 cs_sel = 0, ss_sel = 0;
2835 /* inject #GP if in real mode or Virtual 8086 mode */
2836 if (ctxt->mode == X86EMUL_MODE_REAL ||
2837 ctxt->mode == X86EMUL_MODE_VM86)
2838 return emulate_gp(ctxt, 0);
2840 setup_syscalls_segments(ctxt, &cs, &ss);
2842 if ((ctxt->rex_prefix & 0x8) != 0x0)
2843 usermode = X86EMUL_MODE_PROT64;
2845 usermode = X86EMUL_MODE_PROT32;
2847 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2848 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2852 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2854 case X86EMUL_MODE_PROT32:
2855 cs_sel = (u16)(msr_data + 16);
2856 if ((msr_data & 0xfffc) == 0x0)
2857 return emulate_gp(ctxt, 0);
2858 ss_sel = (u16)(msr_data + 24);
2862 case X86EMUL_MODE_PROT64:
2863 cs_sel = (u16)(msr_data + 32);
2864 if (msr_data == 0x0)
2865 return emulate_gp(ctxt, 0);
2866 ss_sel = cs_sel + 8;
2869 if (is_noncanonical_address(rcx) ||
2870 is_noncanonical_address(rdx))
2871 return emulate_gp(ctxt, 0);
2874 cs_sel |= SEGMENT_RPL_MASK;
2875 ss_sel |= SEGMENT_RPL_MASK;
2877 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2878 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2881 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2883 return X86EMUL_CONTINUE;
2886 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2889 if (ctxt->mode == X86EMUL_MODE_REAL)
2891 if (ctxt->mode == X86EMUL_MODE_VM86)
2893 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2894 return ctxt->ops->cpl(ctxt) > iopl;
2897 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2900 const struct x86_emulate_ops *ops = ctxt->ops;
2901 struct desc_struct tr_seg;
2904 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2905 unsigned mask = (1 << len) - 1;
2908 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2911 if (desc_limit_scaled(&tr_seg) < 103)
2913 base = get_desc_base(&tr_seg);
2914 #ifdef CONFIG_X86_64
2915 base |= ((u64)base3) << 32;
2917 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2918 if (r != X86EMUL_CONTINUE)
2920 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2922 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2923 if (r != X86EMUL_CONTINUE)
2925 if ((perm >> bit_idx) & mask)
2930 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2936 if (emulator_bad_iopl(ctxt))
2937 if (!emulator_io_port_access_allowed(ctxt, port, len))
2940 ctxt->perm_ok = true;
2945 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2948 * Intel CPUs mask the counter and pointers in quite strange
2949 * manner when ECX is zero due to REP-string optimizations.
2951 #ifdef CONFIG_X86_64
2952 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2955 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2958 case 0xa4: /* movsb */
2959 case 0xa5: /* movsd/w */
2960 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2962 case 0xaa: /* stosb */
2963 case 0xab: /* stosd/w */
2964 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2969 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2970 struct tss_segment_16 *tss)
2972 tss->ip = ctxt->_eip;
2973 tss->flag = ctxt->eflags;
2974 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2975 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2976 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2977 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2978 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2979 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2980 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2981 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2983 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2984 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2985 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2986 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2987 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2990 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2991 struct tss_segment_16 *tss)
2996 ctxt->_eip = tss->ip;
2997 ctxt->eflags = tss->flag | 2;
2998 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2999 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3000 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3001 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3002 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3003 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3004 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3005 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3008 * SDM says that segment selectors are loaded before segment
3011 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3012 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3013 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3014 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3015 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3020 * Now load segment descriptors. If fault happens at this stage
3021 * it is handled in a context of new task
3023 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3024 X86_TRANSFER_TASK_SWITCH, NULL);
3025 if (ret != X86EMUL_CONTINUE)
3027 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3028 X86_TRANSFER_TASK_SWITCH, NULL);
3029 if (ret != X86EMUL_CONTINUE)
3031 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3032 X86_TRANSFER_TASK_SWITCH, NULL);
3033 if (ret != X86EMUL_CONTINUE)
3035 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3036 X86_TRANSFER_TASK_SWITCH, NULL);
3037 if (ret != X86EMUL_CONTINUE)
3039 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3040 X86_TRANSFER_TASK_SWITCH, NULL);
3041 if (ret != X86EMUL_CONTINUE)
3044 return X86EMUL_CONTINUE;
3047 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3048 u16 tss_selector, u16 old_tss_sel,
3049 ulong old_tss_base, struct desc_struct *new_desc)
3051 struct tss_segment_16 tss_seg;
3053 u32 new_tss_base = get_desc_base(new_desc);
3055 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3056 if (ret != X86EMUL_CONTINUE)
3059 save_state_to_tss16(ctxt, &tss_seg);
3061 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3062 if (ret != X86EMUL_CONTINUE)
3065 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3066 if (ret != X86EMUL_CONTINUE)
3069 if (old_tss_sel != 0xffff) {
3070 tss_seg.prev_task_link = old_tss_sel;
3072 ret = linear_write_system(ctxt, new_tss_base,
3073 &tss_seg.prev_task_link,
3074 sizeof tss_seg.prev_task_link);
3075 if (ret != X86EMUL_CONTINUE)
3079 return load_state_from_tss16(ctxt, &tss_seg);
3082 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3083 struct tss_segment_32 *tss)
3085 /* CR3 and ldt selector are not saved intentionally */
3086 tss->eip = ctxt->_eip;
3087 tss->eflags = ctxt->eflags;
3088 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3089 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3090 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3091 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3092 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3093 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3094 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3095 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3097 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3098 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3099 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3100 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3101 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3102 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3105 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3106 struct tss_segment_32 *tss)
3111 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3112 return emulate_gp(ctxt, 0);
3113 ctxt->_eip = tss->eip;
3114 ctxt->eflags = tss->eflags | 2;
3116 /* General purpose registers */
3117 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3118 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3119 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3120 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3121 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3122 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3123 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3124 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3127 * SDM says that segment selectors are loaded before segment
3128 * descriptors. This is important because CPL checks will
3131 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3132 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3133 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3134 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3135 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3136 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3137 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3140 * If we're switching between Protected Mode and VM86, we need to make
3141 * sure to update the mode before loading the segment descriptors so
3142 * that the selectors are interpreted correctly.
3144 if (ctxt->eflags & X86_EFLAGS_VM) {
3145 ctxt->mode = X86EMUL_MODE_VM86;
3148 ctxt->mode = X86EMUL_MODE_PROT32;
3153 * Now load segment descriptors. If fault happenes at this stage
3154 * it is handled in a context of new task
3156 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3157 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3158 if (ret != X86EMUL_CONTINUE)
3160 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3161 X86_TRANSFER_TASK_SWITCH, NULL);
3162 if (ret != X86EMUL_CONTINUE)
3164 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3165 X86_TRANSFER_TASK_SWITCH, NULL);
3166 if (ret != X86EMUL_CONTINUE)
3168 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3169 X86_TRANSFER_TASK_SWITCH, NULL);
3170 if (ret != X86EMUL_CONTINUE)
3172 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3173 X86_TRANSFER_TASK_SWITCH, NULL);
3174 if (ret != X86EMUL_CONTINUE)
3176 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3177 X86_TRANSFER_TASK_SWITCH, NULL);
3178 if (ret != X86EMUL_CONTINUE)
3180 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3181 X86_TRANSFER_TASK_SWITCH, NULL);
3186 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3187 u16 tss_selector, u16 old_tss_sel,
3188 ulong old_tss_base, struct desc_struct *new_desc)
3190 struct tss_segment_32 tss_seg;
3192 u32 new_tss_base = get_desc_base(new_desc);
3193 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3194 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3196 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3197 if (ret != X86EMUL_CONTINUE)
3200 save_state_to_tss32(ctxt, &tss_seg);
3202 /* Only GP registers and segment selectors are saved */
3203 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3204 ldt_sel_offset - eip_offset);
3205 if (ret != X86EMUL_CONTINUE)
3208 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3209 if (ret != X86EMUL_CONTINUE)
3212 if (old_tss_sel != 0xffff) {
3213 tss_seg.prev_task_link = old_tss_sel;
3215 ret = linear_write_system(ctxt, new_tss_base,
3216 &tss_seg.prev_task_link,
3217 sizeof tss_seg.prev_task_link);
3218 if (ret != X86EMUL_CONTINUE)
3222 return load_state_from_tss32(ctxt, &tss_seg);
3225 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3226 u16 tss_selector, int idt_index, int reason,
3227 bool has_error_code, u32 error_code)
3229 const struct x86_emulate_ops *ops = ctxt->ops;
3230 struct desc_struct curr_tss_desc, next_tss_desc;
3232 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3233 ulong old_tss_base =
3234 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3236 ulong desc_addr, dr7;
3238 /* FIXME: old_tss_base == ~0 ? */
3240 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3241 if (ret != X86EMUL_CONTINUE)
3243 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3244 if (ret != X86EMUL_CONTINUE)
3247 /* FIXME: check that next_tss_desc is tss */
3250 * Check privileges. The three cases are task switch caused by...
3252 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3253 * 2. Exception/IRQ/iret: No check is performed
3254 * 3. jmp/call to TSS/task-gate: No check is performed since the
3255 * hardware checks it before exiting.
3257 if (reason == TASK_SWITCH_GATE) {
3258 if (idt_index != -1) {
3259 /* Software interrupts */
3260 struct desc_struct task_gate_desc;
3263 ret = read_interrupt_descriptor(ctxt, idt_index,
3265 if (ret != X86EMUL_CONTINUE)
3268 dpl = task_gate_desc.dpl;
3269 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3270 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3274 desc_limit = desc_limit_scaled(&next_tss_desc);
3275 if (!next_tss_desc.p ||
3276 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3277 desc_limit < 0x2b)) {
3278 return emulate_ts(ctxt, tss_selector & 0xfffc);
3281 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3282 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3283 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3286 if (reason == TASK_SWITCH_IRET)
3287 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3289 /* set back link to prev task only if NT bit is set in eflags
3290 note that old_tss_sel is not used after this point */
3291 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3292 old_tss_sel = 0xffff;
3294 if (next_tss_desc.type & 8)
3295 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3296 old_tss_base, &next_tss_desc);
3298 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3299 old_tss_base, &next_tss_desc);
3300 if (ret != X86EMUL_CONTINUE)
3303 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3304 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3306 if (reason != TASK_SWITCH_IRET) {
3307 next_tss_desc.type |= (1 << 1); /* set busy flag */
3308 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3311 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3312 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3314 if (has_error_code) {
3315 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3316 ctxt->lock_prefix = 0;
3317 ctxt->src.val = (unsigned long) error_code;
3318 ret = em_push(ctxt);
3321 ops->get_dr(ctxt, 7, &dr7);
3322 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3327 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3328 u16 tss_selector, int idt_index, int reason,
3329 bool has_error_code, u32 error_code)
3333 invalidate_registers(ctxt);
3334 ctxt->_eip = ctxt->eip;
3335 ctxt->dst.type = OP_NONE;
3337 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3338 has_error_code, error_code);
3340 if (rc == X86EMUL_CONTINUE) {
3341 ctxt->eip = ctxt->_eip;
3342 writeback_registers(ctxt);
3345 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3348 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3351 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3353 register_address_increment(ctxt, reg, df * op->bytes);
3354 op->addr.mem.ea = register_address(ctxt, reg);
3357 static int em_das(struct x86_emulate_ctxt *ctxt)
3360 bool af, cf, old_cf;
3362 cf = ctxt->eflags & X86_EFLAGS_CF;
3368 af = ctxt->eflags & X86_EFLAGS_AF;
3369 if ((al & 0x0f) > 9 || af) {
3371 cf = old_cf | (al >= 250);
3376 if (old_al > 0x99 || old_cf) {
3382 /* Set PF, ZF, SF */
3383 ctxt->src.type = OP_IMM;
3385 ctxt->src.bytes = 1;
3386 fastop(ctxt, em_or);
3387 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3389 ctxt->eflags |= X86_EFLAGS_CF;
3391 ctxt->eflags |= X86_EFLAGS_AF;
3392 return X86EMUL_CONTINUE;
3395 static int em_aam(struct x86_emulate_ctxt *ctxt)
3399 if (ctxt->src.val == 0)
3400 return emulate_de(ctxt);
3402 al = ctxt->dst.val & 0xff;
3403 ah = al / ctxt->src.val;
3404 al %= ctxt->src.val;
3406 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3408 /* Set PF, ZF, SF */
3409 ctxt->src.type = OP_IMM;
3411 ctxt->src.bytes = 1;
3412 fastop(ctxt, em_or);
3414 return X86EMUL_CONTINUE;
3417 static int em_aad(struct x86_emulate_ctxt *ctxt)
3419 u8 al = ctxt->dst.val & 0xff;
3420 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3422 al = (al + (ah * ctxt->src.val)) & 0xff;
3424 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3426 /* Set PF, ZF, SF */
3427 ctxt->src.type = OP_IMM;
3429 ctxt->src.bytes = 1;
3430 fastop(ctxt, em_or);
3432 return X86EMUL_CONTINUE;
3435 static int em_call(struct x86_emulate_ctxt *ctxt)
3438 long rel = ctxt->src.val;
3440 ctxt->src.val = (unsigned long)ctxt->_eip;
3441 rc = jmp_rel(ctxt, rel);
3442 if (rc != X86EMUL_CONTINUE)
3444 return em_push(ctxt);
3447 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3452 struct desc_struct old_desc, new_desc;
3453 const struct x86_emulate_ops *ops = ctxt->ops;
3454 int cpl = ctxt->ops->cpl(ctxt);
3455 enum x86emul_mode prev_mode = ctxt->mode;
3457 old_eip = ctxt->_eip;
3458 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3460 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3461 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3462 X86_TRANSFER_CALL_JMP, &new_desc);
3463 if (rc != X86EMUL_CONTINUE)
3466 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3467 if (rc != X86EMUL_CONTINUE)
3470 ctxt->src.val = old_cs;
3472 if (rc != X86EMUL_CONTINUE)
3475 ctxt->src.val = old_eip;
3477 /* If we failed, we tainted the memory, but the very least we should
3479 if (rc != X86EMUL_CONTINUE) {
3480 pr_warn_once("faulting far call emulation tainted memory\n");
3485 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3486 ctxt->mode = prev_mode;
3491 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3496 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3497 if (rc != X86EMUL_CONTINUE)
3499 rc = assign_eip_near(ctxt, eip);
3500 if (rc != X86EMUL_CONTINUE)
3502 rsp_increment(ctxt, ctxt->src.val);
3503 return X86EMUL_CONTINUE;
3506 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3508 /* Write back the register source. */
3509 ctxt->src.val = ctxt->dst.val;
3510 write_register_operand(&ctxt->src);
3512 /* Write back the memory destination with implicit LOCK prefix. */
3513 ctxt->dst.val = ctxt->src.orig_val;
3514 ctxt->lock_prefix = 1;
3515 return X86EMUL_CONTINUE;
3518 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3520 ctxt->dst.val = ctxt->src2.val;
3521 return fastop(ctxt, em_imul);
3524 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3526 ctxt->dst.type = OP_REG;
3527 ctxt->dst.bytes = ctxt->src.bytes;
3528 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3529 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3531 return X86EMUL_CONTINUE;
3534 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3538 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3539 return emulate_ud(ctxt);
3540 ctxt->dst.val = tsc_aux;
3541 return X86EMUL_CONTINUE;
3544 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3548 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3549 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3550 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3551 return X86EMUL_CONTINUE;
3554 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3558 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3559 return emulate_gp(ctxt, 0);
3560 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3561 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3562 return X86EMUL_CONTINUE;
3565 static int em_mov(struct x86_emulate_ctxt *ctxt)
3567 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3568 return X86EMUL_CONTINUE;
3571 #define FFL(x) bit(X86_FEATURE_##x)
3573 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3575 u32 ebx, ecx, edx, eax = 1;
3579 * Check MOVBE is set in the guest-visible CPUID leaf.
3581 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3582 if (!(ecx & FFL(MOVBE)))
3583 return emulate_ud(ctxt);
3585 switch (ctxt->op_bytes) {
3588 * From MOVBE definition: "...When the operand size is 16 bits,
3589 * the upper word of the destination register remains unchanged
3592 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3593 * rules so we have to do the operation almost per hand.
3595 tmp = (u16)ctxt->src.val;
3596 ctxt->dst.val &= ~0xffffUL;
3597 ctxt->dst.val |= (unsigned long)swab16(tmp);
3600 ctxt->dst.val = swab32((u32)ctxt->src.val);
3603 ctxt->dst.val = swab64(ctxt->src.val);
3608 return X86EMUL_CONTINUE;
3611 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3613 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3614 return emulate_gp(ctxt, 0);
3616 /* Disable writeback. */
3617 ctxt->dst.type = OP_NONE;
3618 return X86EMUL_CONTINUE;
3621 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3625 if (ctxt->mode == X86EMUL_MODE_PROT64)
3626 val = ctxt->src.val & ~0ULL;
3628 val = ctxt->src.val & ~0U;
3630 /* #UD condition is already handled. */
3631 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3632 return emulate_gp(ctxt, 0);
3634 /* Disable writeback. */
3635 ctxt->dst.type = OP_NONE;
3636 return X86EMUL_CONTINUE;
3639 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3643 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3644 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3645 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3646 return emulate_gp(ctxt, 0);
3648 return X86EMUL_CONTINUE;
3651 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3655 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3656 return emulate_gp(ctxt, 0);
3658 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3659 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3660 return X86EMUL_CONTINUE;
3663 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3665 if (ctxt->modrm_reg > VCPU_SREG_GS)
3666 return emulate_ud(ctxt);
3668 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3669 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3670 ctxt->dst.bytes = 2;
3671 return X86EMUL_CONTINUE;
3674 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3676 u16 sel = ctxt->src.val;
3678 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3679 return emulate_ud(ctxt);
3681 if (ctxt->modrm_reg == VCPU_SREG_SS)
3682 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3684 /* Disable writeback. */
3685 ctxt->dst.type = OP_NONE;
3686 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3689 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3691 u16 sel = ctxt->src.val;
3693 /* Disable writeback. */
3694 ctxt->dst.type = OP_NONE;
3695 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3698 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3700 u16 sel = ctxt->src.val;
3702 /* Disable writeback. */
3703 ctxt->dst.type = OP_NONE;
3704 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3707 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3712 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3713 if (rc == X86EMUL_CONTINUE)
3714 ctxt->ops->invlpg(ctxt, linear);
3715 /* Disable writeback. */
3716 ctxt->dst.type = OP_NONE;
3717 return X86EMUL_CONTINUE;
3720 static int em_clts(struct x86_emulate_ctxt *ctxt)
3724 cr0 = ctxt->ops->get_cr(ctxt, 0);
3726 ctxt->ops->set_cr(ctxt, 0, cr0);
3727 return X86EMUL_CONTINUE;
3730 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3732 int rc = ctxt->ops->fix_hypercall(ctxt);
3734 if (rc != X86EMUL_CONTINUE)
3737 /* Let the processor re-execute the fixed hypercall */
3738 ctxt->_eip = ctxt->eip;
3739 /* Disable writeback. */
3740 ctxt->dst.type = OP_NONE;
3741 return X86EMUL_CONTINUE;
3744 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3745 void (*get)(struct x86_emulate_ctxt *ctxt,
3746 struct desc_ptr *ptr))
3748 struct desc_ptr desc_ptr;
3750 if (ctxt->mode == X86EMUL_MODE_PROT64)
3752 get(ctxt, &desc_ptr);
3753 if (ctxt->op_bytes == 2) {
3755 desc_ptr.address &= 0x00ffffff;
3757 /* Disable writeback. */
3758 ctxt->dst.type = OP_NONE;
3759 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3760 &desc_ptr, 2 + ctxt->op_bytes);
3763 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3765 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3768 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3770 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3773 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3775 struct desc_ptr desc_ptr;
3778 if (ctxt->mode == X86EMUL_MODE_PROT64)
3780 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3781 &desc_ptr.size, &desc_ptr.address,
3783 if (rc != X86EMUL_CONTINUE)
3785 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3786 is_noncanonical_address(desc_ptr.address))
3787 return emulate_gp(ctxt, 0);
3789 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3791 ctxt->ops->set_idt(ctxt, &desc_ptr);
3792 /* Disable writeback. */
3793 ctxt->dst.type = OP_NONE;
3794 return X86EMUL_CONTINUE;
3797 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3799 return em_lgdt_lidt(ctxt, true);
3802 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3804 return em_lgdt_lidt(ctxt, false);
3807 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3809 if (ctxt->dst.type == OP_MEM)
3810 ctxt->dst.bytes = 2;
3811 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3812 return X86EMUL_CONTINUE;
3815 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3817 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3818 | (ctxt->src.val & 0x0f));
3819 ctxt->dst.type = OP_NONE;
3820 return X86EMUL_CONTINUE;
3823 static int em_loop(struct x86_emulate_ctxt *ctxt)
3825 int rc = X86EMUL_CONTINUE;
3827 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3828 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3829 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3830 rc = jmp_rel(ctxt, ctxt->src.val);
3835 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3837 int rc = X86EMUL_CONTINUE;
3839 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3840 rc = jmp_rel(ctxt, ctxt->src.val);
3845 static int em_in(struct x86_emulate_ctxt *ctxt)
3847 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3849 return X86EMUL_IO_NEEDED;
3851 return X86EMUL_CONTINUE;
3854 static int em_out(struct x86_emulate_ctxt *ctxt)
3856 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3858 /* Disable writeback. */
3859 ctxt->dst.type = OP_NONE;
3860 return X86EMUL_CONTINUE;
3863 static int em_cli(struct x86_emulate_ctxt *ctxt)
3865 if (emulator_bad_iopl(ctxt))
3866 return emulate_gp(ctxt, 0);
3868 ctxt->eflags &= ~X86_EFLAGS_IF;
3869 return X86EMUL_CONTINUE;
3872 static int em_sti(struct x86_emulate_ctxt *ctxt)
3874 if (emulator_bad_iopl(ctxt))
3875 return emulate_gp(ctxt, 0);
3877 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3878 ctxt->eflags |= X86_EFLAGS_IF;
3879 return X86EMUL_CONTINUE;
3882 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3884 u32 eax, ebx, ecx, edx;
3886 eax = reg_read(ctxt, VCPU_REGS_RAX);
3887 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3888 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3889 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3890 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3891 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3892 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3893 return X86EMUL_CONTINUE;
3896 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3900 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3902 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3904 ctxt->eflags &= ~0xffUL;
3905 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3906 return X86EMUL_CONTINUE;
3909 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3911 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3912 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3913 return X86EMUL_CONTINUE;
3916 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3918 switch (ctxt->op_bytes) {
3919 #ifdef CONFIG_X86_64
3921 asm("bswap %0" : "+r"(ctxt->dst.val));
3925 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3928 return X86EMUL_CONTINUE;
3931 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3933 /* emulating clflush regardless of cpuid */
3934 return X86EMUL_CONTINUE;
3937 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3939 /* emulating clflushopt regardless of cpuid */
3940 return X86EMUL_CONTINUE;
3943 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3945 ctxt->dst.val = (s32) ctxt->src.val;
3946 return X86EMUL_CONTINUE;
3949 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3951 u32 eax = 1, ebx, ecx = 0, edx;
3953 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3954 if (!(edx & FFL(FXSR)))
3955 return emulate_ud(ctxt);
3957 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3958 return emulate_nm(ctxt);
3961 * Don't emulate a case that should never be hit, instead of working
3962 * around a lack of fxsave64/fxrstor64 on old compilers.
3964 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3965 return X86EMUL_UNHANDLEABLE;
3967 return X86EMUL_CONTINUE;
3971 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3974 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3975 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3977 * 3) 64-bit mode with REX.W prefix
3978 * - like (2), but XMM 8-15 are being saved and restored
3979 * 4) 64-bit mode without REX.W prefix
3980 * - like (3), but FIP and FDP are 64 bit
3982 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3983 * desired result. (4) is not emulated.
3985 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3986 * and FPU DS) should match.
3988 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
3990 struct fxregs_state fx_state;
3994 rc = check_fxsr(ctxt);
3995 if (rc != X86EMUL_CONTINUE)
3998 ctxt->ops->get_fpu(ctxt);
4000 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4002 ctxt->ops->put_fpu(ctxt);
4004 if (rc != X86EMUL_CONTINUE)
4007 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
4008 size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
4010 size = offsetof(struct fxregs_state, xmm_space[0]);
4012 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4015 static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
4016 struct fxregs_state *new)
4018 int rc = X86EMUL_CONTINUE;
4019 struct fxregs_state old;
4021 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
4022 if (rc != X86EMUL_CONTINUE)
4026 * 64 bit host will restore XMM 8-15, which is not correct on non-64
4027 * bit guests. Load the current values in order to preserve 64 bit
4028 * XMMs after fxrstor.
4030 #ifdef CONFIG_X86_64
4031 /* XXX: accessing XMM 8-15 very awkwardly */
4032 memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
4036 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
4037 * does save and restore MXCSR.
4039 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
4040 memcpy(new->xmm_space, old.xmm_space, 8 * 16);
4045 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4047 struct fxregs_state fx_state;
4050 rc = check_fxsr(ctxt);
4051 if (rc != X86EMUL_CONTINUE)
4054 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
4055 if (rc != X86EMUL_CONTINUE)
4058 if (fx_state.mxcsr >> 16)
4059 return emulate_gp(ctxt, 0);
4061 ctxt->ops->get_fpu(ctxt);
4063 if (ctxt->mode < X86EMUL_MODE_PROT64)
4064 rc = fxrstor_fixup(ctxt, &fx_state);
4066 if (rc == X86EMUL_CONTINUE)
4067 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4069 ctxt->ops->put_fpu(ctxt);
4074 static bool valid_cr(int nr)
4086 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4088 if (!valid_cr(ctxt->modrm_reg))
4089 return emulate_ud(ctxt);
4091 return X86EMUL_CONTINUE;
4094 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4096 u64 new_val = ctxt->src.val64;
4097 int cr = ctxt->modrm_reg;
4100 static u64 cr_reserved_bits[] = {
4101 0xffffffff00000000ULL,
4102 0, 0, 0, /* CR3 checked later */
4109 return emulate_ud(ctxt);
4111 if (new_val & cr_reserved_bits[cr])
4112 return emulate_gp(ctxt, 0);
4117 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4118 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4119 return emulate_gp(ctxt, 0);
4121 cr4 = ctxt->ops->get_cr(ctxt, 4);
4122 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4124 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4125 !(cr4 & X86_CR4_PAE))
4126 return emulate_gp(ctxt, 0);
4133 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4134 if (efer & EFER_LMA)
4135 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
4138 return emulate_gp(ctxt, 0);
4143 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4145 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4146 return emulate_gp(ctxt, 0);
4152 return X86EMUL_CONTINUE;
4155 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4159 ctxt->ops->get_dr(ctxt, 7, &dr7);
4161 /* Check if DR7.Global_Enable is set */
4162 return dr7 & (1 << 13);
4165 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4167 int dr = ctxt->modrm_reg;
4171 return emulate_ud(ctxt);
4173 cr4 = ctxt->ops->get_cr(ctxt, 4);
4174 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4175 return emulate_ud(ctxt);
4177 if (check_dr7_gd(ctxt)) {
4180 ctxt->ops->get_dr(ctxt, 6, &dr6);
4182 dr6 |= DR6_BD | DR6_RTM;
4183 ctxt->ops->set_dr(ctxt, 6, dr6);
4184 return emulate_db(ctxt);
4187 return X86EMUL_CONTINUE;
4190 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4192 u64 new_val = ctxt->src.val64;
4193 int dr = ctxt->modrm_reg;
4195 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4196 return emulate_gp(ctxt, 0);
4198 return check_dr_read(ctxt);
4201 static int check_svme(struct x86_emulate_ctxt *ctxt)
4205 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4207 if (!(efer & EFER_SVME))
4208 return emulate_ud(ctxt);
4210 return X86EMUL_CONTINUE;
4213 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4215 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4217 /* Valid physical address? */
4218 if (rax & 0xffff000000000000ULL)
4219 return emulate_gp(ctxt, 0);
4221 return check_svme(ctxt);
4224 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4226 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4228 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4229 return emulate_ud(ctxt);
4231 return X86EMUL_CONTINUE;
4234 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4236 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4237 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4239 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4240 ctxt->ops->check_pmc(ctxt, rcx))
4241 return emulate_gp(ctxt, 0);
4243 return X86EMUL_CONTINUE;
4246 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4248 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4249 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4250 return emulate_gp(ctxt, 0);
4252 return X86EMUL_CONTINUE;
4255 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4257 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4258 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4259 return emulate_gp(ctxt, 0);
4261 return X86EMUL_CONTINUE;
4264 #define D(_y) { .flags = (_y) }
4265 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4266 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4267 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4268 #define N D(NotImpl)
4269 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4270 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4271 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4272 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4273 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4274 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4275 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4276 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4277 #define II(_f, _e, _i) \
4278 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4279 #define IIP(_f, _e, _i, _p) \
4280 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4281 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4282 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4284 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4285 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4286 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4287 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4288 #define I2bvIP(_f, _e, _i, _p) \
4289 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4291 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4292 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4293 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4295 static const struct opcode group7_rm0[] = {
4297 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4301 static const struct opcode group7_rm1[] = {
4302 DI(SrcNone | Priv, monitor),
4303 DI(SrcNone | Priv, mwait),
4307 static const struct opcode group7_rm3[] = {
4308 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4309 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4310 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4311 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4312 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4313 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4314 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4315 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4318 static const struct opcode group7_rm7[] = {
4320 DIP(SrcNone, rdtscp, check_rdtsc),
4324 static const struct opcode group1[] = {
4326 F(Lock | PageTable, em_or),
4329 F(Lock | PageTable, em_and),
4335 static const struct opcode group1A[] = {
4336 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4339 static const struct opcode group2[] = {
4340 F(DstMem | ModRM, em_rol),
4341 F(DstMem | ModRM, em_ror),
4342 F(DstMem | ModRM, em_rcl),
4343 F(DstMem | ModRM, em_rcr),
4344 F(DstMem | ModRM, em_shl),
4345 F(DstMem | ModRM, em_shr),
4346 F(DstMem | ModRM, em_shl),
4347 F(DstMem | ModRM, em_sar),
4350 static const struct opcode group3[] = {
4351 F(DstMem | SrcImm | NoWrite, em_test),
4352 F(DstMem | SrcImm | NoWrite, em_test),
4353 F(DstMem | SrcNone | Lock, em_not),
4354 F(DstMem | SrcNone | Lock, em_neg),
4355 F(DstXacc | Src2Mem, em_mul_ex),
4356 F(DstXacc | Src2Mem, em_imul_ex),
4357 F(DstXacc | Src2Mem, em_div_ex),
4358 F(DstXacc | Src2Mem, em_idiv_ex),
4361 static const struct opcode group4[] = {
4362 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4363 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4367 static const struct opcode group5[] = {
4368 F(DstMem | SrcNone | Lock, em_inc),
4369 F(DstMem | SrcNone | Lock, em_dec),
4370 I(SrcMem | NearBranch, em_call_near_abs),
4371 I(SrcMemFAddr | ImplicitOps, em_call_far),
4372 I(SrcMem | NearBranch, em_jmp_abs),
4373 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4374 I(SrcMem | Stack, em_push), D(Undefined),
4377 static const struct opcode group6[] = {
4378 DI(Prot | DstMem, sldt),
4379 DI(Prot | DstMem, str),
4380 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4381 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4385 static const struct group_dual group7 = { {
4386 II(Mov | DstMem, em_sgdt, sgdt),
4387 II(Mov | DstMem, em_sidt, sidt),
4388 II(SrcMem | Priv, em_lgdt, lgdt),
4389 II(SrcMem | Priv, em_lidt, lidt),
4390 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4391 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4392 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4396 N, EXT(0, group7_rm3),
4397 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4398 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4402 static const struct opcode group8[] = {
4404 F(DstMem | SrcImmByte | NoWrite, em_bt),
4405 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4406 F(DstMem | SrcImmByte | Lock, em_btr),
4407 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4411 * The "memory" destination is actually always a register, since we come
4412 * from the register case of group9.
4414 static const struct gprefix pfx_0f_c7_7 = {
4415 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4419 static const struct group_dual group9 = { {
4420 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4422 N, N, N, N, N, N, N,
4423 GP(0, &pfx_0f_c7_7),
4426 static const struct opcode group11[] = {
4427 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4431 static const struct gprefix pfx_0f_ae_7 = {
4432 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4435 static const struct group_dual group15 = { {
4436 I(ModRM | Aligned16, em_fxsave),
4437 I(ModRM | Aligned16, em_fxrstor),
4438 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4440 N, N, N, N, N, N, N, N,
4443 static const struct gprefix pfx_0f_6f_0f_7f = {
4444 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4447 static const struct instr_dual instr_dual_0f_2b = {
4451 static const struct gprefix pfx_0f_2b = {
4452 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4455 static const struct gprefix pfx_0f_28_0f_29 = {
4456 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4459 static const struct gprefix pfx_0f_e7 = {
4460 N, I(Sse, em_mov), N, N,
4463 static const struct escape escape_d9 = { {
4464 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4467 N, N, N, N, N, N, N, N,
4469 N, N, N, N, N, N, N, N,
4471 N, N, N, N, N, N, N, N,
4473 N, N, N, N, N, N, N, N,
4475 N, N, N, N, N, N, N, N,
4477 N, N, N, N, N, N, N, N,
4479 N, N, N, N, N, N, N, N,
4481 N, N, N, N, N, N, N, N,
4484 static const struct escape escape_db = { {
4485 N, N, N, N, N, N, N, N,
4488 N, N, N, N, N, N, N, N,
4490 N, N, N, N, N, N, N, N,
4492 N, N, N, N, N, N, N, N,
4494 N, N, N, N, N, N, N, N,
4496 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4498 N, N, N, N, N, N, N, N,
4500 N, N, N, N, N, N, N, N,
4502 N, N, N, N, N, N, N, N,
4505 static const struct escape escape_dd = { {
4506 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4509 N, N, N, N, N, N, N, N,
4511 N, N, N, N, N, N, N, N,
4513 N, N, N, N, N, N, N, N,
4515 N, N, N, N, N, N, N, N,
4517 N, N, N, N, N, N, N, N,
4519 N, N, N, N, N, N, N, N,
4521 N, N, N, N, N, N, N, N,
4523 N, N, N, N, N, N, N, N,
4526 static const struct instr_dual instr_dual_0f_c3 = {
4527 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4530 static const struct mode_dual mode_dual_63 = {
4531 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4534 static const struct opcode opcode_table[256] = {
4536 F6ALU(Lock, em_add),
4537 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4538 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4540 F6ALU(Lock | PageTable, em_or),
4541 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4544 F6ALU(Lock, em_adc),
4545 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4546 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4548 F6ALU(Lock, em_sbb),
4549 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4550 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4552 F6ALU(Lock | PageTable, em_and), N, N,
4554 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4556 F6ALU(Lock, em_xor), N, N,
4558 F6ALU(NoWrite, em_cmp), N, N,
4560 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4562 X8(I(SrcReg | Stack, em_push)),
4564 X8(I(DstReg | Stack, em_pop)),
4566 I(ImplicitOps | Stack | No64, em_pusha),
4567 I(ImplicitOps | Stack | No64, em_popa),
4568 N, MD(ModRM, &mode_dual_63),
4571 I(SrcImm | Mov | Stack, em_push),
4572 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4573 I(SrcImmByte | Mov | Stack, em_push),
4574 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4575 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4576 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4578 X16(D(SrcImmByte | NearBranch)),
4580 G(ByteOp | DstMem | SrcImm, group1),
4581 G(DstMem | SrcImm, group1),
4582 G(ByteOp | DstMem | SrcImm | No64, group1),
4583 G(DstMem | SrcImmByte, group1),
4584 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4585 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4587 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4588 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4589 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4590 D(ModRM | SrcMem | NoAccess | DstReg),
4591 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4594 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4596 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4597 I(SrcImmFAddr | No64, em_call_far), N,
4598 II(ImplicitOps | Stack, em_pushf, pushf),
4599 II(ImplicitOps | Stack, em_popf, popf),
4600 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4602 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4603 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4604 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4605 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4607 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4608 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4609 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4610 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4612 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4614 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4616 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4617 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4618 I(ImplicitOps | NearBranch, em_ret),
4619 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4620 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4621 G(ByteOp, group11), G(0, group11),
4623 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4624 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4625 I(ImplicitOps, em_ret_far),
4626 D(ImplicitOps), DI(SrcImmByte, intn),
4627 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4629 G(Src2One | ByteOp, group2), G(Src2One, group2),
4630 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4631 I(DstAcc | SrcImmUByte | No64, em_aam),
4632 I(DstAcc | SrcImmUByte | No64, em_aad),
4633 F(DstAcc | ByteOp | No64, em_salc),
4634 I(DstAcc | SrcXLat | ByteOp, em_mov),
4636 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4638 X3(I(SrcImmByte | NearBranch, em_loop)),
4639 I(SrcImmByte | NearBranch, em_jcxz),
4640 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4641 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4643 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4644 I(SrcImmFAddr | No64, em_jmp_far),
4645 D(SrcImmByte | ImplicitOps | NearBranch),
4646 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4647 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4649 N, DI(ImplicitOps, icebp), N, N,
4650 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4651 G(ByteOp, group3), G(0, group3),
4653 D(ImplicitOps), D(ImplicitOps),
4654 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4655 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4658 static const struct opcode twobyte_table[256] = {
4660 G(0, group6), GD(0, &group7), N, N,
4661 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4662 II(ImplicitOps | Priv, em_clts, clts), N,
4663 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4664 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4666 N, N, N, N, N, N, N, N,
4667 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4668 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4670 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4671 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4672 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4674 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4677 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4678 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4679 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4682 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4683 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4684 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4685 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4686 I(ImplicitOps | EmulateOnUD, em_sysenter),
4687 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4689 N, N, N, N, N, N, N, N,
4691 X16(D(DstReg | SrcMem | ModRM)),
4693 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4698 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4703 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4705 X16(D(SrcImm | NearBranch)),
4707 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4709 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4710 II(ImplicitOps, em_cpuid, cpuid),
4711 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4712 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4713 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4715 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4716 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4717 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4718 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4719 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4720 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4722 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4723 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4724 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4725 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4726 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4727 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4731 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4732 I(DstReg | SrcMem | ModRM, em_bsf_c),
4733 I(DstReg | SrcMem | ModRM, em_bsr_c),
4734 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4736 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4737 N, ID(0, &instr_dual_0f_c3),
4738 N, N, N, GD(0, &group9),
4740 X8(I(DstReg, em_bswap)),
4742 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4744 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4745 N, N, N, N, N, N, N, N,
4747 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4750 static const struct instr_dual instr_dual_0f_38_f0 = {
4751 I(DstReg | SrcMem | Mov, em_movbe), N
4754 static const struct instr_dual instr_dual_0f_38_f1 = {
4755 I(DstMem | SrcReg | Mov, em_movbe), N
4758 static const struct gprefix three_byte_0f_38_f0 = {
4759 ID(0, &instr_dual_0f_38_f0), N, N, N
4762 static const struct gprefix three_byte_0f_38_f1 = {
4763 ID(0, &instr_dual_0f_38_f1), N, N, N
4767 * Insns below are selected by the prefix which indexed by the third opcode
4770 static const struct opcode opcode_map_0f_38[256] = {
4772 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4774 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4776 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4777 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4798 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4802 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4808 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4809 unsigned size, bool sign_extension)
4811 int rc = X86EMUL_CONTINUE;
4815 op->addr.mem.ea = ctxt->_eip;
4816 /* NB. Immediates are sign-extended as necessary. */
4817 switch (op->bytes) {
4819 op->val = insn_fetch(s8, ctxt);
4822 op->val = insn_fetch(s16, ctxt);
4825 op->val = insn_fetch(s32, ctxt);
4828 op->val = insn_fetch(s64, ctxt);
4831 if (!sign_extension) {
4832 switch (op->bytes) {
4840 op->val &= 0xffffffff;
4848 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4851 int rc = X86EMUL_CONTINUE;
4855 decode_register_operand(ctxt, op);
4858 rc = decode_imm(ctxt, op, 1, false);
4861 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4865 if (ctxt->d & BitOp)
4866 fetch_bit_operand(ctxt);
4867 op->orig_val = op->val;
4870 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4874 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4875 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4876 fetch_register_operand(op);
4877 op->orig_val = op->val;
4881 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4882 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4883 fetch_register_operand(op);
4884 op->orig_val = op->val;
4887 if (ctxt->d & ByteOp) {
4892 op->bytes = ctxt->op_bytes;
4893 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4894 fetch_register_operand(op);
4895 op->orig_val = op->val;
4899 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4901 register_address(ctxt, VCPU_REGS_RDI);
4902 op->addr.mem.seg = VCPU_SREG_ES;
4909 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4910 fetch_register_operand(op);
4915 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4918 rc = decode_imm(ctxt, op, 1, true);
4926 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4929 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4932 ctxt->memop.bytes = 1;
4933 if (ctxt->memop.type == OP_REG) {
4934 ctxt->memop.addr.reg = decode_register(ctxt,
4935 ctxt->modrm_rm, true);
4936 fetch_register_operand(&ctxt->memop);
4940 ctxt->memop.bytes = 2;
4943 ctxt->memop.bytes = 4;
4946 rc = decode_imm(ctxt, op, 2, false);
4949 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4953 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4955 register_address(ctxt, VCPU_REGS_RSI);
4956 op->addr.mem.seg = ctxt->seg_override;
4962 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4965 reg_read(ctxt, VCPU_REGS_RBX) +
4966 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4967 op->addr.mem.seg = ctxt->seg_override;
4972 op->addr.mem.ea = ctxt->_eip;
4973 op->bytes = ctxt->op_bytes + 2;
4974 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4977 ctxt->memop.bytes = ctxt->op_bytes + 2;
4981 op->val = VCPU_SREG_ES;
4985 op->val = VCPU_SREG_CS;
4989 op->val = VCPU_SREG_SS;
4993 op->val = VCPU_SREG_DS;
4997 op->val = VCPU_SREG_FS;
5001 op->val = VCPU_SREG_GS;
5004 /* Special instructions do their own operand decoding. */
5006 op->type = OP_NONE; /* Disable writeback. */
5014 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5016 int rc = X86EMUL_CONTINUE;
5017 int mode = ctxt->mode;
5018 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5019 bool op_prefix = false;
5020 bool has_seg_override = false;
5021 struct opcode opcode;
5023 struct desc_struct desc;
5025 ctxt->memop.type = OP_NONE;
5026 ctxt->memopp = NULL;
5027 ctxt->_eip = ctxt->eip;
5028 ctxt->fetch.ptr = ctxt->fetch.data;
5029 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5030 ctxt->opcode_len = 1;
5031 ctxt->intercept = x86_intercept_none;
5033 memcpy(ctxt->fetch.data, insn, insn_len);
5035 rc = __do_insn_fetch_bytes(ctxt, 1);
5036 if (rc != X86EMUL_CONTINUE)
5041 case X86EMUL_MODE_REAL:
5042 case X86EMUL_MODE_VM86:
5043 def_op_bytes = def_ad_bytes = 2;
5044 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5046 def_op_bytes = def_ad_bytes = 4;
5048 case X86EMUL_MODE_PROT16:
5049 def_op_bytes = def_ad_bytes = 2;
5051 case X86EMUL_MODE_PROT32:
5052 def_op_bytes = def_ad_bytes = 4;
5054 #ifdef CONFIG_X86_64
5055 case X86EMUL_MODE_PROT64:
5061 return EMULATION_FAILED;
5064 ctxt->op_bytes = def_op_bytes;
5065 ctxt->ad_bytes = def_ad_bytes;
5067 /* Legacy prefixes. */
5069 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5070 case 0x66: /* operand-size override */
5072 /* switch between 2/4 bytes */
5073 ctxt->op_bytes = def_op_bytes ^ 6;
5075 case 0x67: /* address-size override */
5076 if (mode == X86EMUL_MODE_PROT64)
5077 /* switch between 4/8 bytes */
5078 ctxt->ad_bytes = def_ad_bytes ^ 12;
5080 /* switch between 2/4 bytes */
5081 ctxt->ad_bytes = def_ad_bytes ^ 6;
5083 case 0x26: /* ES override */
5084 has_seg_override = true;
5085 ctxt->seg_override = VCPU_SREG_ES;
5087 case 0x2e: /* CS override */
5088 has_seg_override = true;
5089 ctxt->seg_override = VCPU_SREG_CS;
5091 case 0x36: /* SS override */
5092 has_seg_override = true;
5093 ctxt->seg_override = VCPU_SREG_SS;
5095 case 0x3e: /* DS override */
5096 has_seg_override = true;
5097 ctxt->seg_override = VCPU_SREG_DS;
5099 case 0x64: /* FS override */
5100 has_seg_override = true;
5101 ctxt->seg_override = VCPU_SREG_FS;
5103 case 0x65: /* GS override */
5104 has_seg_override = true;
5105 ctxt->seg_override = VCPU_SREG_GS;
5107 case 0x40 ... 0x4f: /* REX */
5108 if (mode != X86EMUL_MODE_PROT64)
5110 ctxt->rex_prefix = ctxt->b;
5112 case 0xf0: /* LOCK */
5113 ctxt->lock_prefix = 1;
5115 case 0xf2: /* REPNE/REPNZ */
5116 case 0xf3: /* REP/REPE/REPZ */
5117 ctxt->rep_prefix = ctxt->b;
5123 /* Any legacy prefix after a REX prefix nullifies its effect. */
5125 ctxt->rex_prefix = 0;
5131 if (ctxt->rex_prefix & 8)
5132 ctxt->op_bytes = 8; /* REX.W */
5134 /* Opcode byte(s). */
5135 opcode = opcode_table[ctxt->b];
5136 /* Two-byte opcode? */
5137 if (ctxt->b == 0x0f) {
5138 ctxt->opcode_len = 2;
5139 ctxt->b = insn_fetch(u8, ctxt);
5140 opcode = twobyte_table[ctxt->b];
5142 /* 0F_38 opcode map */
5143 if (ctxt->b == 0x38) {
5144 ctxt->opcode_len = 3;
5145 ctxt->b = insn_fetch(u8, ctxt);
5146 opcode = opcode_map_0f_38[ctxt->b];
5149 ctxt->d = opcode.flags;
5151 if (ctxt->d & ModRM)
5152 ctxt->modrm = insn_fetch(u8, ctxt);
5154 /* vex-prefix instructions are not implemented */
5155 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5156 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5160 while (ctxt->d & GroupMask) {
5161 switch (ctxt->d & GroupMask) {
5163 goffset = (ctxt->modrm >> 3) & 7;
5164 opcode = opcode.u.group[goffset];
5167 goffset = (ctxt->modrm >> 3) & 7;
5168 if ((ctxt->modrm >> 6) == 3)
5169 opcode = opcode.u.gdual->mod3[goffset];
5171 opcode = opcode.u.gdual->mod012[goffset];
5174 goffset = ctxt->modrm & 7;
5175 opcode = opcode.u.group[goffset];
5178 if (ctxt->rep_prefix && op_prefix)
5179 return EMULATION_FAILED;
5180 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5181 switch (simd_prefix) {
5182 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5183 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5184 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5185 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5189 if (ctxt->modrm > 0xbf) {
5190 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5191 u32 index = array_index_nospec(
5192 ctxt->modrm - 0xc0, size);
5194 opcode = opcode.u.esc->high[index];
5196 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5200 if ((ctxt->modrm >> 6) == 3)
5201 opcode = opcode.u.idual->mod3;
5203 opcode = opcode.u.idual->mod012;
5206 if (ctxt->mode == X86EMUL_MODE_PROT64)
5207 opcode = opcode.u.mdual->mode64;
5209 opcode = opcode.u.mdual->mode32;
5212 return EMULATION_FAILED;
5215 ctxt->d &= ~(u64)GroupMask;
5216 ctxt->d |= opcode.flags;
5221 return EMULATION_FAILED;
5223 ctxt->execute = opcode.u.execute;
5225 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5226 return EMULATION_FAILED;
5228 if (unlikely(ctxt->d &
5229 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5232 * These are copied unconditionally here, and checked unconditionally
5233 * in x86_emulate_insn.
5235 ctxt->check_perm = opcode.check_perm;
5236 ctxt->intercept = opcode.intercept;
5238 if (ctxt->d & NotImpl)
5239 return EMULATION_FAILED;
5241 if (mode == X86EMUL_MODE_PROT64) {
5242 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5244 else if (ctxt->d & NearBranch)
5248 if (ctxt->d & Op3264) {
5249 if (mode == X86EMUL_MODE_PROT64)
5255 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5259 ctxt->op_bytes = 16;
5260 else if (ctxt->d & Mmx)
5264 /* ModRM and SIB bytes. */
5265 if (ctxt->d & ModRM) {
5266 rc = decode_modrm(ctxt, &ctxt->memop);
5267 if (!has_seg_override) {
5268 has_seg_override = true;
5269 ctxt->seg_override = ctxt->modrm_seg;
5271 } else if (ctxt->d & MemAbs)
5272 rc = decode_abs(ctxt, &ctxt->memop);
5273 if (rc != X86EMUL_CONTINUE)
5276 if (!has_seg_override)
5277 ctxt->seg_override = VCPU_SREG_DS;
5279 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5282 * Decode and fetch the source operand: register, memory
5285 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5286 if (rc != X86EMUL_CONTINUE)
5290 * Decode and fetch the second source operand: register, memory
5293 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5294 if (rc != X86EMUL_CONTINUE)
5297 /* Decode and fetch the destination operand: register or memory. */
5298 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5300 if (ctxt->rip_relative && likely(ctxt->memopp))
5301 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5302 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5305 if (rc == X86EMUL_PROPAGATE_FAULT)
5306 ctxt->have_exception = true;
5307 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5310 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5312 return ctxt->d & PageTable;
5315 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5317 /* The second termination condition only applies for REPE
5318 * and REPNE. Test if the repeat string operation prefix is
5319 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5320 * corresponding termination condition according to:
5321 * - if REPE/REPZ and ZF = 0 then done
5322 * - if REPNE/REPNZ and ZF = 1 then done
5324 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5325 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5326 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5327 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5328 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5329 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5335 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5339 ctxt->ops->get_fpu(ctxt);
5340 rc = asm_safe("fwait");
5341 ctxt->ops->put_fpu(ctxt);
5343 if (unlikely(rc != X86EMUL_CONTINUE))
5344 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5346 return X86EMUL_CONTINUE;
5349 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5352 if (op->type == OP_MM)
5353 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5356 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5358 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5360 if (!(ctxt->d & ByteOp))
5361 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5363 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5364 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5365 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5366 : "c"(ctxt->src2.val));
5368 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5369 if (!fop) /* exception is returned in fop variable */
5370 return emulate_de(ctxt);
5371 return X86EMUL_CONTINUE;
5374 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5376 memset(&ctxt->rip_relative, 0,
5377 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5379 ctxt->io_read.pos = 0;
5380 ctxt->io_read.end = 0;
5381 ctxt->mem_read.end = 0;
5384 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5386 const struct x86_emulate_ops *ops = ctxt->ops;
5387 int rc = X86EMUL_CONTINUE;
5388 int saved_dst_type = ctxt->dst.type;
5389 unsigned emul_flags;
5391 ctxt->mem_read.pos = 0;
5393 /* LOCK prefix is allowed only with some instructions */
5394 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5395 rc = emulate_ud(ctxt);
5399 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5400 rc = emulate_ud(ctxt);
5404 emul_flags = ctxt->ops->get_hflags(ctxt);
5405 if (unlikely(ctxt->d &
5406 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5407 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5408 (ctxt->d & Undefined)) {
5409 rc = emulate_ud(ctxt);
5413 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5414 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5415 rc = emulate_ud(ctxt);
5419 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5420 rc = emulate_nm(ctxt);
5424 if (ctxt->d & Mmx) {
5425 rc = flush_pending_x87_faults(ctxt);
5426 if (rc != X86EMUL_CONTINUE)
5429 * Now that we know the fpu is exception safe, we can fetch
5432 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5433 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5434 if (!(ctxt->d & Mov))
5435 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5438 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5439 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5440 X86_ICPT_PRE_EXCEPT);
5441 if (rc != X86EMUL_CONTINUE)
5445 /* Instruction can only be executed in protected mode */
5446 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5447 rc = emulate_ud(ctxt);
5451 /* Privileged instruction can be executed only in CPL=0 */
5452 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5453 if (ctxt->d & PrivUD)
5454 rc = emulate_ud(ctxt);
5456 rc = emulate_gp(ctxt, 0);
5460 /* Do instruction specific permission checks */
5461 if (ctxt->d & CheckPerm) {
5462 rc = ctxt->check_perm(ctxt);
5463 if (rc != X86EMUL_CONTINUE)
5467 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5468 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5469 X86_ICPT_POST_EXCEPT);
5470 if (rc != X86EMUL_CONTINUE)
5474 if (ctxt->rep_prefix && (ctxt->d & String)) {
5475 /* All REP prefixes have the same first termination condition */
5476 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5477 string_registers_quirk(ctxt);
5478 ctxt->eip = ctxt->_eip;
5479 ctxt->eflags &= ~X86_EFLAGS_RF;
5485 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5486 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5487 ctxt->src.valptr, ctxt->src.bytes);
5488 if (rc != X86EMUL_CONTINUE)
5490 ctxt->src.orig_val64 = ctxt->src.val64;
5493 if (ctxt->src2.type == OP_MEM) {
5494 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5495 &ctxt->src2.val, ctxt->src2.bytes);
5496 if (rc != X86EMUL_CONTINUE)
5500 if ((ctxt->d & DstMask) == ImplicitOps)
5504 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5505 /* optimisation - avoid slow emulated read if Mov */
5506 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5507 &ctxt->dst.val, ctxt->dst.bytes);
5508 if (rc != X86EMUL_CONTINUE) {
5509 if (!(ctxt->d & NoWrite) &&
5510 rc == X86EMUL_PROPAGATE_FAULT &&
5511 ctxt->exception.vector == PF_VECTOR)
5512 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5516 /* Copy full 64-bit value for CMPXCHG8B. */
5517 ctxt->dst.orig_val64 = ctxt->dst.val64;
5521 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5522 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5523 X86_ICPT_POST_MEMACCESS);
5524 if (rc != X86EMUL_CONTINUE)
5528 if (ctxt->rep_prefix && (ctxt->d & String))
5529 ctxt->eflags |= X86_EFLAGS_RF;
5531 ctxt->eflags &= ~X86_EFLAGS_RF;
5533 if (ctxt->execute) {
5534 if (ctxt->d & Fastop) {
5535 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5536 rc = fastop(ctxt, fop);
5537 if (rc != X86EMUL_CONTINUE)
5541 rc = ctxt->execute(ctxt);
5542 if (rc != X86EMUL_CONTINUE)
5547 if (ctxt->opcode_len == 2)
5549 else if (ctxt->opcode_len == 3)
5550 goto threebyte_insn;
5553 case 0x70 ... 0x7f: /* jcc (short) */
5554 if (test_cc(ctxt->b, ctxt->eflags))
5555 rc = jmp_rel(ctxt, ctxt->src.val);
5557 case 0x8d: /* lea r16/r32, m */
5558 ctxt->dst.val = ctxt->src.addr.mem.ea;
5560 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5561 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5562 ctxt->dst.type = OP_NONE;
5566 case 0x98: /* cbw/cwde/cdqe */
5567 switch (ctxt->op_bytes) {
5568 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5569 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5570 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5573 case 0xcc: /* int3 */
5574 rc = emulate_int(ctxt, 3);
5576 case 0xcd: /* int n */
5577 rc = emulate_int(ctxt, ctxt->src.val);
5579 case 0xce: /* into */
5580 if (ctxt->eflags & X86_EFLAGS_OF)
5581 rc = emulate_int(ctxt, 4);
5583 case 0xe9: /* jmp rel */
5584 case 0xeb: /* jmp rel short */
5585 rc = jmp_rel(ctxt, ctxt->src.val);
5586 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5588 case 0xf4: /* hlt */
5589 ctxt->ops->halt(ctxt);
5591 case 0xf5: /* cmc */
5592 /* complement carry flag from eflags reg */
5593 ctxt->eflags ^= X86_EFLAGS_CF;
5595 case 0xf8: /* clc */
5596 ctxt->eflags &= ~X86_EFLAGS_CF;
5598 case 0xf9: /* stc */
5599 ctxt->eflags |= X86_EFLAGS_CF;
5601 case 0xfc: /* cld */
5602 ctxt->eflags &= ~X86_EFLAGS_DF;
5604 case 0xfd: /* std */
5605 ctxt->eflags |= X86_EFLAGS_DF;
5608 goto cannot_emulate;
5611 if (rc != X86EMUL_CONTINUE)
5615 if (ctxt->d & SrcWrite) {
5616 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5617 rc = writeback(ctxt, &ctxt->src);
5618 if (rc != X86EMUL_CONTINUE)
5621 if (!(ctxt->d & NoWrite)) {
5622 rc = writeback(ctxt, &ctxt->dst);
5623 if (rc != X86EMUL_CONTINUE)
5628 * restore dst type in case the decoding will be reused
5629 * (happens for string instruction )
5631 ctxt->dst.type = saved_dst_type;
5633 if ((ctxt->d & SrcMask) == SrcSI)
5634 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5636 if ((ctxt->d & DstMask) == DstDI)
5637 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5639 if (ctxt->rep_prefix && (ctxt->d & String)) {
5641 struct read_cache *r = &ctxt->io_read;
5642 if ((ctxt->d & SrcMask) == SrcSI)
5643 count = ctxt->src.count;
5645 count = ctxt->dst.count;
5646 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5648 if (!string_insn_completed(ctxt)) {
5650 * Re-enter guest when pio read ahead buffer is empty
5651 * or, if it is not used, after each 1024 iteration.
5653 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5654 (r->end == 0 || r->end != r->pos)) {
5656 * Reset read cache. Usually happens before
5657 * decode, but since instruction is restarted
5658 * we have to do it here.
5660 ctxt->mem_read.end = 0;
5661 writeback_registers(ctxt);
5662 return EMULATION_RESTART;
5664 goto done; /* skip rip writeback */
5666 ctxt->eflags &= ~X86_EFLAGS_RF;
5669 ctxt->eip = ctxt->_eip;
5672 if (rc == X86EMUL_PROPAGATE_FAULT) {
5673 WARN_ON(ctxt->exception.vector > 0x1f);
5674 ctxt->have_exception = true;
5676 if (rc == X86EMUL_INTERCEPTED)
5677 return EMULATION_INTERCEPTED;
5679 if (rc == X86EMUL_CONTINUE)
5680 writeback_registers(ctxt);
5682 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5686 case 0x09: /* wbinvd */
5687 (ctxt->ops->wbinvd)(ctxt);
5689 case 0x08: /* invd */
5690 case 0x0d: /* GrpP (prefetch) */
5691 case 0x18: /* Grp16 (prefetch/nop) */
5692 case 0x1f: /* nop */
5694 case 0x20: /* mov cr, reg */
5695 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5697 case 0x21: /* mov from dr to reg */
5698 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5700 case 0x40 ... 0x4f: /* cmov */
5701 if (test_cc(ctxt->b, ctxt->eflags))
5702 ctxt->dst.val = ctxt->src.val;
5703 else if (ctxt->op_bytes != 4)
5704 ctxt->dst.type = OP_NONE; /* no writeback */
5706 case 0x80 ... 0x8f: /* jnz rel, etc*/
5707 if (test_cc(ctxt->b, ctxt->eflags))
5708 rc = jmp_rel(ctxt, ctxt->src.val);
5710 case 0x90 ... 0x9f: /* setcc r/m8 */
5711 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5713 case 0xb6 ... 0xb7: /* movzx */
5714 ctxt->dst.bytes = ctxt->op_bytes;
5715 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5716 : (u16) ctxt->src.val;
5718 case 0xbe ... 0xbf: /* movsx */
5719 ctxt->dst.bytes = ctxt->op_bytes;
5720 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5721 (s16) ctxt->src.val;
5724 goto cannot_emulate;
5729 if (rc != X86EMUL_CONTINUE)
5735 return EMULATION_FAILED;
5738 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5740 invalidate_registers(ctxt);
5743 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5745 writeback_registers(ctxt);