1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <linux/nospec.h>
27 #include <asm/kvm_emulate.h>
28 #include <linux/stringify.h>
29 #include <asm/debugreg.h>
30 #include <asm/nospec-branch.h>
39 #define OpImplicit 1ull /* No generic decode */
40 #define OpReg 2ull /* Register */
41 #define OpMem 3ull /* Memory */
42 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
43 #define OpDI 5ull /* ES:DI/EDI/RDI */
44 #define OpMem64 6ull /* Memory, 64-bit */
45 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
46 #define OpDX 8ull /* DX register */
47 #define OpCL 9ull /* CL register (for shifts) */
48 #define OpImmByte 10ull /* 8-bit sign extended immediate */
49 #define OpOne 11ull /* Implied 1 */
50 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
51 #define OpMem16 13ull /* Memory operand (16-bit). */
52 #define OpMem32 14ull /* Memory operand (32-bit). */
53 #define OpImmU 15ull /* Immediate operand, zero extended */
54 #define OpSI 16ull /* SI/ESI/RSI */
55 #define OpImmFAddr 17ull /* Immediate far address */
56 #define OpMemFAddr 18ull /* Far address in memory */
57 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
58 #define OpES 20ull /* ES */
59 #define OpCS 21ull /* CS */
60 #define OpSS 22ull /* SS */
61 #define OpDS 23ull /* DS */
62 #define OpFS 24ull /* FS */
63 #define OpGS 25ull /* GS */
64 #define OpMem8 26ull /* 8-bit zero extended memory operand */
65 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
66 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
67 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
68 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
70 #define OpBits 5 /* Width of operand field */
71 #define OpMask ((1ull << OpBits) - 1)
74 * Opcode effective-address decode tables.
75 * Note that we only emulate instructions that have at least one memory
76 * operand (excluding implicit stack references). We assume that stack
77 * references and instruction fetches will never occur in special memory
78 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
82 /* Operand sizes: 8-bit operands or specified/overridden size. */
83 #define ByteOp (1<<0) /* 8-bit operands. */
84 /* Destination operand type. */
86 #define ImplicitOps (OpImplicit << DstShift)
87 #define DstReg (OpReg << DstShift)
88 #define DstMem (OpMem << DstShift)
89 #define DstAcc (OpAcc << DstShift)
90 #define DstDI (OpDI << DstShift)
91 #define DstMem64 (OpMem64 << DstShift)
92 #define DstMem16 (OpMem16 << DstShift)
93 #define DstImmUByte (OpImmUByte << DstShift)
94 #define DstDX (OpDX << DstShift)
95 #define DstAccLo (OpAccLo << DstShift)
96 #define DstMask (OpMask << DstShift)
97 /* Source operand type. */
99 #define SrcNone (OpNone << SrcShift)
100 #define SrcReg (OpReg << SrcShift)
101 #define SrcMem (OpMem << SrcShift)
102 #define SrcMem16 (OpMem16 << SrcShift)
103 #define SrcMem32 (OpMem32 << SrcShift)
104 #define SrcImm (OpImm << SrcShift)
105 #define SrcImmByte (OpImmByte << SrcShift)
106 #define SrcOne (OpOne << SrcShift)
107 #define SrcImmUByte (OpImmUByte << SrcShift)
108 #define SrcImmU (OpImmU << SrcShift)
109 #define SrcSI (OpSI << SrcShift)
110 #define SrcXLat (OpXLat << SrcShift)
111 #define SrcImmFAddr (OpImmFAddr << SrcShift)
112 #define SrcMemFAddr (OpMemFAddr << SrcShift)
113 #define SrcAcc (OpAcc << SrcShift)
114 #define SrcImmU16 (OpImmU16 << SrcShift)
115 #define SrcImm64 (OpImm64 << SrcShift)
116 #define SrcDX (OpDX << SrcShift)
117 #define SrcMem8 (OpMem8 << SrcShift)
118 #define SrcAccHi (OpAccHi << SrcShift)
119 #define SrcMask (OpMask << SrcShift)
120 #define BitOp (1<<11)
121 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
122 #define String (1<<13) /* String instruction (rep capable) */
123 #define Stack (1<<14) /* Stack instruction (push/pop) */
124 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
125 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
126 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
127 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
128 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
129 #define Escape (5<<15) /* Escape to coprocessor instruction */
130 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
131 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
132 #define Sse (1<<18) /* SSE Vector instruction */
133 /* Generic ModRM decode. */
134 #define ModRM (1<<19)
135 /* Destination is only written; never read. */
138 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
139 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
140 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
141 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
142 #define Undefined (1<<25) /* No Such Instruction */
143 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
144 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
146 #define PageTable (1 << 29) /* instruction used to write page table */
147 #define NotImpl (1 << 30) /* instruction is not implemented */
148 /* Source 2 operand type */
149 #define Src2Shift (31)
150 #define Src2None (OpNone << Src2Shift)
151 #define Src2Mem (OpMem << Src2Shift)
152 #define Src2CL (OpCL << Src2Shift)
153 #define Src2ImmByte (OpImmByte << Src2Shift)
154 #define Src2One (OpOne << Src2Shift)
155 #define Src2Imm (OpImm << Src2Shift)
156 #define Src2ES (OpES << Src2Shift)
157 #define Src2CS (OpCS << Src2Shift)
158 #define Src2SS (OpSS << Src2Shift)
159 #define Src2DS (OpDS << Src2Shift)
160 #define Src2FS (OpFS << Src2Shift)
161 #define Src2GS (OpGS << Src2Shift)
162 #define Src2Mask (OpMask << Src2Shift)
163 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
167 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
168 #define NoWrite ((u64)1 << 45) /* No writeback */
169 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
170 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
171 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
172 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
173 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
174 #define NearBranch ((u64)1 << 52) /* Near branches */
175 #define No16 ((u64)1 << 53) /* No 16 bit operand */
176 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
177 #define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
179 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
181 #define X2(x...) x, x
182 #define X3(x...) X2(x), x
183 #define X4(x...) X2(x), X2(x)
184 #define X5(x...) X4(x), x
185 #define X6(x...) X4(x), X2(x)
186 #define X7(x...) X4(x), X3(x)
187 #define X8(x...) X4(x), X4(x)
188 #define X16(x...) X8(x), X8(x)
190 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
191 #define FASTOP_SIZE 8
194 * fastop functions have a special calling convention:
199 * flags: rflags (in/out)
200 * ex: rsi (in:fastop pointer, out:zero if exception)
202 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
203 * different operand sizes can be reached by calculation, rather than a jump
204 * table (which would be bigger than the code).
206 * fastop functions are declared as taking a never-defined fastop parameter,
207 * so they can't be called from C directly.
216 int (*execute)(struct x86_emulate_ctxt *ctxt);
217 const struct opcode *group;
218 const struct group_dual *gdual;
219 const struct gprefix *gprefix;
220 const struct escape *esc;
221 const struct instr_dual *idual;
222 const struct mode_dual *mdual;
223 void (*fastop)(struct fastop *fake);
225 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
229 struct opcode mod012[8];
230 struct opcode mod3[8];
234 struct opcode pfx_no;
235 struct opcode pfx_66;
236 struct opcode pfx_f2;
237 struct opcode pfx_f3;
242 struct opcode high[64];
246 struct opcode mod012;
251 struct opcode mode32;
252 struct opcode mode64;
255 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
257 enum x86_transfer_type {
259 X86_TRANSFER_CALL_JMP,
261 X86_TRANSFER_TASK_SWITCH,
264 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
266 if (!(ctxt->regs_valid & (1 << nr))) {
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
270 return ctxt->_regs[nr];
273 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 ctxt->regs_valid |= 1 << nr;
276 ctxt->regs_dirty |= 1 << nr;
277 return &ctxt->_regs[nr];
280 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
283 return reg_write(ctxt, nr);
286 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
290 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
291 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
294 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
296 ctxt->regs_dirty = 0;
297 ctxt->regs_valid = 0;
301 * These EFLAGS bits are restored from saved value during emulation, and
302 * any changes are written back to the saved value after emulation.
304 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
305 X86_EFLAGS_PF|X86_EFLAGS_CF)
313 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
315 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
316 #define FOP_RET "ret \n\t"
318 #define FOP_START(op) \
319 extern void em_##op(struct fastop *fake); \
320 asm(".pushsection .text, \"ax\" \n\t" \
321 ".global em_" #op " \n\t" \
328 #define FOPNOP() FOP_ALIGN FOP_RET
330 #define FOP1E(op, dst) \
331 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
333 #define FOP1EEX(op, dst) \
334 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
336 #define FASTOP1(op) \
341 ON64(FOP1E(op##q, rax)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m) */
345 #define FASTOP1SRC2(op, name) \
350 ON64(FOP1E(op, rcx)) \
353 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
354 #define FASTOP1SRC2EX(op, name) \
359 ON64(FOP1EEX(op, rcx)) \
362 #define FOP2E(op, dst, src) \
363 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
365 #define FASTOP2(op) \
367 FOP2E(op##b, al, dl) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, word only */
374 #define FASTOP2W(op) \
377 FOP2E(op##w, ax, dx) \
378 FOP2E(op##l, eax, edx) \
379 ON64(FOP2E(op##q, rax, rdx)) \
382 /* 2 operand, src is CL */
383 #define FASTOP2CL(op) \
385 FOP2E(op##b, al, cl) \
386 FOP2E(op##w, ax, cl) \
387 FOP2E(op##l, eax, cl) \
388 ON64(FOP2E(op##q, rax, cl)) \
391 /* 2 operand, src and dest are reversed */
392 #define FASTOP2R(op, name) \
394 FOP2E(op##b, dl, al) \
395 FOP2E(op##w, dx, ax) \
396 FOP2E(op##l, edx, eax) \
397 ON64(FOP2E(op##q, rdx, rax)) \
400 #define FOP3E(op, dst, src, src2) \
401 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
403 /* 3-operand, word-only, src2=cl */
404 #define FASTOP3WCL(op) \
407 FOP3E(op##w, ax, dx, cl) \
408 FOP3E(op##l, eax, edx, cl) \
409 ON64(FOP3E(op##q, rax, rdx, cl)) \
412 /* Special case for SETcc - 1 instruction per cc */
413 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
415 asm(".global kvm_fastop_exception \n"
416 "kvm_fastop_exception: xor %esi, %esi; ret");
437 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
441 * XXX: inoutclob user must know where the argument is being expanded.
442 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
444 #define asm_safe(insn, inoutclob...) \
448 asm volatile("1:" insn "\n" \
450 ".pushsection .fixup, \"ax\"\n" \
451 "3: movl $1, %[_fault]\n" \
454 _ASM_EXTABLE(1b, 3b) \
455 : [_fault] "+qm"(_fault) inoutclob ); \
457 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
460 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
461 enum x86_intercept intercept,
462 enum x86_intercept_stage stage)
464 struct x86_instruction_info info = {
465 .intercept = intercept,
466 .rep_prefix = ctxt->rep_prefix,
467 .modrm_mod = ctxt->modrm_mod,
468 .modrm_reg = ctxt->modrm_reg,
469 .modrm_rm = ctxt->modrm_rm,
470 .src_val = ctxt->src.val64,
471 .dst_val = ctxt->dst.val64,
472 .src_bytes = ctxt->src.bytes,
473 .dst_bytes = ctxt->dst.bytes,
474 .ad_bytes = ctxt->ad_bytes,
475 .next_rip = ctxt->eip,
478 return ctxt->ops->intercept(ctxt, &info, stage);
481 static void assign_masked(ulong *dest, ulong src, ulong mask)
483 *dest = (*dest & ~mask) | (src & mask);
486 static void assign_register(unsigned long *reg, u64 val, int bytes)
488 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
491 *(u8 *)reg = (u8)val;
494 *(u16 *)reg = (u16)val;
498 break; /* 64b: zero-extend */
505 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
507 return (1UL << (ctxt->ad_bytes << 3)) - 1;
510 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
513 struct desc_struct ss;
515 if (ctxt->mode == X86EMUL_MODE_PROT64)
517 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
518 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
521 static int stack_size(struct x86_emulate_ctxt *ctxt)
523 return (__fls(stack_mask(ctxt)) + 1) >> 3;
526 /* Access/update address held in a register, based on addressing mode. */
527 static inline unsigned long
528 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
530 if (ctxt->ad_bytes == sizeof(unsigned long))
533 return reg & ad_mask(ctxt);
536 static inline unsigned long
537 register_address(struct x86_emulate_ctxt *ctxt, int reg)
539 return address_mask(ctxt, reg_read(ctxt, reg));
542 static void masked_increment(ulong *reg, ulong mask, int inc)
544 assign_masked(reg, *reg + inc, mask);
548 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
550 ulong *preg = reg_rmw(ctxt, reg);
552 assign_register(preg, *preg + inc, ctxt->ad_bytes);
555 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
557 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
560 static u32 desc_limit_scaled(struct desc_struct *desc)
562 u32 limit = get_desc_limit(desc);
564 return desc->g ? (limit << 12) | 0xfff : limit;
567 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
569 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
572 return ctxt->ops->get_cached_segment_base(ctxt, seg);
575 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
576 u32 error, bool valid)
579 ctxt->exception.vector = vec;
580 ctxt->exception.error_code = error;
581 ctxt->exception.error_code_valid = valid;
582 return X86EMUL_PROPAGATE_FAULT;
585 static int emulate_db(struct x86_emulate_ctxt *ctxt)
587 return emulate_exception(ctxt, DB_VECTOR, 0, false);
590 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
592 return emulate_exception(ctxt, GP_VECTOR, err, true);
595 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
597 return emulate_exception(ctxt, SS_VECTOR, err, true);
600 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
602 return emulate_exception(ctxt, UD_VECTOR, 0, false);
605 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
607 return emulate_exception(ctxt, TS_VECTOR, err, true);
610 static int emulate_de(struct x86_emulate_ctxt *ctxt)
612 return emulate_exception(ctxt, DE_VECTOR, 0, false);
615 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
617 return emulate_exception(ctxt, NM_VECTOR, 0, false);
620 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
623 struct desc_struct desc;
625 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
629 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
634 struct desc_struct desc;
636 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
637 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
641 * x86 defines three classes of vector instructions: explicitly
642 * aligned, explicitly unaligned, and the rest, which change behaviour
643 * depending on whether they're AVX encoded or not.
645 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
646 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
647 * 512 bytes of data must be aligned to a 16 byte boundary.
649 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
651 if (likely(size < 16))
654 if (ctxt->d & Aligned)
656 else if (ctxt->d & Unaligned)
658 else if (ctxt->d & Avx)
660 else if (ctxt->d & Aligned16)
666 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
667 struct segmented_address addr,
668 unsigned *max_size, unsigned size,
669 bool write, bool fetch,
670 enum x86emul_mode mode, ulong *linear)
672 struct desc_struct desc;
678 la = seg_base(ctxt, addr.seg) + addr.ea;
681 case X86EMUL_MODE_PROT64:
683 if (is_noncanonical_address(la))
686 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
687 if (size > *max_size)
691 *linear = la = (u32)la;
692 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
696 /* code segment in protected mode or read-only data segment */
697 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
698 || !(desc.type & 2)) && write)
700 /* unreadable code segment */
701 if (!fetch && (desc.type & 8) && !(desc.type & 2))
703 lim = desc_limit_scaled(&desc);
704 if (!(desc.type & 8) && (desc.type & 4)) {
705 /* expand-down segment */
708 lim = desc.d ? 0xffffffff : 0xffff;
712 if (lim == 0xffffffff)
715 *max_size = (u64)lim + 1 - addr.ea;
716 if (size > *max_size)
721 if (la & (insn_alignment(ctxt, size) - 1))
722 return emulate_gp(ctxt, 0);
723 return X86EMUL_CONTINUE;
725 if (addr.seg == VCPU_SREG_SS)
726 return emulate_ss(ctxt, 0);
728 return emulate_gp(ctxt, 0);
731 static int linearize(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 unsigned size, bool write,
737 return __linearize(ctxt, addr, &max_size, size, write, false,
741 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
742 enum x86emul_mode mode)
747 struct segmented_address addr = { .seg = VCPU_SREG_CS,
750 if (ctxt->op_bytes != sizeof(unsigned long))
751 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
752 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
753 if (rc == X86EMUL_CONTINUE)
754 ctxt->_eip = addr.ea;
758 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
760 return assign_eip(ctxt, dst, ctxt->mode);
763 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
764 const struct desc_struct *cs_desc)
766 enum x86emul_mode mode = ctxt->mode;
770 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
774 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
776 mode = X86EMUL_MODE_PROT64;
778 mode = X86EMUL_MODE_PROT32; /* temporary value */
781 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
782 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
783 rc = assign_eip(ctxt, dst, mode);
784 if (rc == X86EMUL_CONTINUE)
789 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
791 return assign_eip_near(ctxt, ctxt->_eip + rel);
794 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
795 void *data, unsigned size)
797 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
800 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
801 ulong linear, void *data,
804 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
807 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
808 struct segmented_address addr,
815 rc = linearize(ctxt, addr, size, false, &linear);
816 if (rc != X86EMUL_CONTINUE)
818 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
821 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
822 struct segmented_address addr,
829 rc = linearize(ctxt, addr, size, true, &linear);
830 if (rc != X86EMUL_CONTINUE)
832 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
836 * Prefetch the remaining bytes of the instruction without crossing page
837 * boundary if they are not in fetch_cache yet.
839 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
842 unsigned size, max_size;
843 unsigned long linear;
844 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
845 struct segmented_address addr = { .seg = VCPU_SREG_CS,
846 .ea = ctxt->eip + cur_size };
849 * We do not know exactly how many bytes will be needed, and
850 * __linearize is expensive, so fetch as much as possible. We
851 * just have to avoid going beyond the 15 byte limit, the end
852 * of the segment, or the end of the page.
854 * __linearize is called with size 0 so that it does not do any
855 * boundary check itself. Instead, we use max_size to check
858 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
860 if (unlikely(rc != X86EMUL_CONTINUE))
863 size = min_t(unsigned, 15UL ^ cur_size, max_size);
864 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
867 * One instruction can only straddle two pages,
868 * and one has been loaded at the beginning of
869 * x86_decode_insn. So, if not enough bytes
870 * still, we must have hit the 15-byte boundary.
872 if (unlikely(size < op_size))
873 return emulate_gp(ctxt, 0);
875 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
876 size, &ctxt->exception);
877 if (unlikely(rc != X86EMUL_CONTINUE))
879 ctxt->fetch.end += size;
880 return X86EMUL_CONTINUE;
883 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
886 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
888 if (unlikely(done_size < size))
889 return __do_insn_fetch_bytes(ctxt, size - done_size);
891 return X86EMUL_CONTINUE;
894 /* Fetch next part of the instruction being emulated. */
895 #define insn_fetch(_type, _ctxt) \
898 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
899 if (rc != X86EMUL_CONTINUE) \
901 ctxt->_eip += sizeof(_type); \
902 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
903 ctxt->fetch.ptr += sizeof(_type); \
907 #define insn_fetch_arr(_arr, _size, _ctxt) \
909 rc = do_insn_fetch_bytes(_ctxt, _size); \
910 if (rc != X86EMUL_CONTINUE) \
912 ctxt->_eip += (_size); \
913 memcpy(_arr, ctxt->fetch.ptr, _size); \
914 ctxt->fetch.ptr += (_size); \
918 * Given the 'reg' portion of a ModRM byte, and a register block, return a
919 * pointer into the block that addresses the relevant register.
920 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
922 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
926 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
928 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
929 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
931 p = reg_rmw(ctxt, modrm_reg);
935 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
936 struct segmented_address addr,
937 u16 *size, unsigned long *address, int op_bytes)
944 rc = segmented_read_std(ctxt, addr, size, 2);
945 if (rc != X86EMUL_CONTINUE)
948 rc = segmented_read_std(ctxt, addr, address, op_bytes);
962 FASTOP1SRC2(mul, mul_ex);
963 FASTOP1SRC2(imul, imul_ex);
964 FASTOP1SRC2EX(div, div_ex);
965 FASTOP1SRC2EX(idiv, idiv_ex);
994 FASTOP2R(cmp, cmp_r);
996 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
998 /* If src is zero, do not writeback, but update flags */
999 if (ctxt->src.val == 0)
1000 ctxt->dst.type = OP_NONE;
1001 return fastop(ctxt, em_bsf);
1004 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1006 /* If src is zero, do not writeback, but update flags */
1007 if (ctxt->src.val == 0)
1008 ctxt->dst.type = OP_NONE;
1009 return fastop(ctxt, em_bsr);
1012 static u8 test_cc(unsigned int condition, unsigned long flags)
1015 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1017 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1018 asm("push %[flags]; popf; " CALL_NOSPEC
1019 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1023 static void fetch_register_operand(struct operand *op)
1025 switch (op->bytes) {
1027 op->val = *(u8 *)op->addr.reg;
1030 op->val = *(u16 *)op->addr.reg;
1033 op->val = *(u32 *)op->addr.reg;
1036 op->val = *(u64 *)op->addr.reg;
1041 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1043 ctxt->ops->get_fpu(ctxt);
1045 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1046 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1047 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1048 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1049 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1050 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1051 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1052 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1053 #ifdef CONFIG_X86_64
1054 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1055 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1056 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1057 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1058 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1059 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1060 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1061 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1065 ctxt->ops->put_fpu(ctxt);
1068 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1071 ctxt->ops->get_fpu(ctxt);
1073 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1074 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1075 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1076 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1077 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1078 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1079 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1080 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1081 #ifdef CONFIG_X86_64
1082 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1083 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1084 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1085 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1086 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1087 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1088 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1089 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1093 ctxt->ops->put_fpu(ctxt);
1096 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1098 ctxt->ops->get_fpu(ctxt);
1100 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1101 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1102 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1103 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1104 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1105 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1106 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1107 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1110 ctxt->ops->put_fpu(ctxt);
1113 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1115 ctxt->ops->get_fpu(ctxt);
1117 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1118 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1119 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1120 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1121 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1122 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1123 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1124 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1127 ctxt->ops->put_fpu(ctxt);
1130 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1132 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1133 return emulate_nm(ctxt);
1135 ctxt->ops->get_fpu(ctxt);
1136 asm volatile("fninit");
1137 ctxt->ops->put_fpu(ctxt);
1138 return X86EMUL_CONTINUE;
1141 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1145 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1146 return emulate_nm(ctxt);
1148 ctxt->ops->get_fpu(ctxt);
1149 asm volatile("fnstcw %0": "+m"(fcw));
1150 ctxt->ops->put_fpu(ctxt);
1152 ctxt->dst.val = fcw;
1154 return X86EMUL_CONTINUE;
1157 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1161 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1162 return emulate_nm(ctxt);
1164 ctxt->ops->get_fpu(ctxt);
1165 asm volatile("fnstsw %0": "+m"(fsw));
1166 ctxt->ops->put_fpu(ctxt);
1168 ctxt->dst.val = fsw;
1170 return X86EMUL_CONTINUE;
1173 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1176 unsigned reg = ctxt->modrm_reg;
1178 if (!(ctxt->d & ModRM))
1179 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1181 if (ctxt->d & Sse) {
1185 read_sse_reg(ctxt, &op->vec_val, reg);
1188 if (ctxt->d & Mmx) {
1197 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1198 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1200 fetch_register_operand(op);
1201 op->orig_val = op->val;
1204 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1206 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1207 ctxt->modrm_seg = VCPU_SREG_SS;
1210 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1214 int index_reg, base_reg, scale;
1215 int rc = X86EMUL_CONTINUE;
1218 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1219 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1220 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1222 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1223 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1224 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1225 ctxt->modrm_seg = VCPU_SREG_DS;
1227 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1229 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1230 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1232 if (ctxt->d & Sse) {
1235 op->addr.xmm = ctxt->modrm_rm;
1236 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1239 if (ctxt->d & Mmx) {
1242 op->addr.mm = ctxt->modrm_rm & 7;
1245 fetch_register_operand(op);
1251 if (ctxt->ad_bytes == 2) {
1252 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1253 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1254 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1255 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1257 /* 16-bit ModR/M decode. */
1258 switch (ctxt->modrm_mod) {
1260 if (ctxt->modrm_rm == 6)
1261 modrm_ea += insn_fetch(u16, ctxt);
1264 modrm_ea += insn_fetch(s8, ctxt);
1267 modrm_ea += insn_fetch(u16, ctxt);
1270 switch (ctxt->modrm_rm) {
1272 modrm_ea += bx + si;
1275 modrm_ea += bx + di;
1278 modrm_ea += bp + si;
1281 modrm_ea += bp + di;
1290 if (ctxt->modrm_mod != 0)
1297 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1298 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1299 ctxt->modrm_seg = VCPU_SREG_SS;
1300 modrm_ea = (u16)modrm_ea;
1302 /* 32/64-bit ModR/M decode. */
1303 if ((ctxt->modrm_rm & 7) == 4) {
1304 sib = insn_fetch(u8, ctxt);
1305 index_reg |= (sib >> 3) & 7;
1306 base_reg |= sib & 7;
1309 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1310 modrm_ea += insn_fetch(s32, ctxt);
1312 modrm_ea += reg_read(ctxt, base_reg);
1313 adjust_modrm_seg(ctxt, base_reg);
1314 /* Increment ESP on POP [ESP] */
1315 if ((ctxt->d & IncSP) &&
1316 base_reg == VCPU_REGS_RSP)
1317 modrm_ea += ctxt->op_bytes;
1320 modrm_ea += reg_read(ctxt, index_reg) << scale;
1321 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1322 modrm_ea += insn_fetch(s32, ctxt);
1323 if (ctxt->mode == X86EMUL_MODE_PROT64)
1324 ctxt->rip_relative = 1;
1326 base_reg = ctxt->modrm_rm;
1327 modrm_ea += reg_read(ctxt, base_reg);
1328 adjust_modrm_seg(ctxt, base_reg);
1330 switch (ctxt->modrm_mod) {
1332 modrm_ea += insn_fetch(s8, ctxt);
1335 modrm_ea += insn_fetch(s32, ctxt);
1339 op->addr.mem.ea = modrm_ea;
1340 if (ctxt->ad_bytes != 8)
1341 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1347 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1350 int rc = X86EMUL_CONTINUE;
1353 switch (ctxt->ad_bytes) {
1355 op->addr.mem.ea = insn_fetch(u16, ctxt);
1358 op->addr.mem.ea = insn_fetch(u32, ctxt);
1361 op->addr.mem.ea = insn_fetch(u64, ctxt);
1368 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1372 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1373 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1375 if (ctxt->src.bytes == 2)
1376 sv = (s16)ctxt->src.val & (s16)mask;
1377 else if (ctxt->src.bytes == 4)
1378 sv = (s32)ctxt->src.val & (s32)mask;
1380 sv = (s64)ctxt->src.val & (s64)mask;
1382 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1383 ctxt->dst.addr.mem.ea + (sv >> 3));
1386 /* only subword offset */
1387 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1390 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1391 unsigned long addr, void *dest, unsigned size)
1394 struct read_cache *mc = &ctxt->mem_read;
1396 if (mc->pos < mc->end)
1399 WARN_ON((mc->end + size) >= sizeof(mc->data));
1401 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1403 if (rc != X86EMUL_CONTINUE)
1409 memcpy(dest, mc->data + mc->pos, size);
1411 return X86EMUL_CONTINUE;
1414 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1415 struct segmented_address addr,
1422 rc = linearize(ctxt, addr, size, false, &linear);
1423 if (rc != X86EMUL_CONTINUE)
1425 return read_emulated(ctxt, linear, data, size);
1428 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1429 struct segmented_address addr,
1436 rc = linearize(ctxt, addr, size, true, &linear);
1437 if (rc != X86EMUL_CONTINUE)
1439 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1443 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1444 struct segmented_address addr,
1445 const void *orig_data, const void *data,
1451 rc = linearize(ctxt, addr, size, true, &linear);
1452 if (rc != X86EMUL_CONTINUE)
1454 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1455 size, &ctxt->exception);
1458 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1459 unsigned int size, unsigned short port,
1462 struct read_cache *rc = &ctxt->io_read;
1464 if (rc->pos == rc->end) { /* refill pio read ahead */
1465 unsigned int in_page, n;
1466 unsigned int count = ctxt->rep_prefix ?
1467 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1468 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1469 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1470 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1471 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1474 rc->pos = rc->end = 0;
1475 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1480 if (ctxt->rep_prefix && (ctxt->d & String) &&
1481 !(ctxt->eflags & X86_EFLAGS_DF)) {
1482 ctxt->dst.data = rc->data + rc->pos;
1483 ctxt->dst.type = OP_MEM_STR;
1484 ctxt->dst.count = (rc->end - rc->pos) / size;
1487 memcpy(dest, rc->data + rc->pos, size);
1493 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1494 u16 index, struct desc_struct *desc)
1499 ctxt->ops->get_idt(ctxt, &dt);
1501 if (dt.size < index * 8 + 7)
1502 return emulate_gp(ctxt, index << 3 | 0x2);
1504 addr = dt.address + index * 8;
1505 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1508 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1509 u16 selector, struct desc_ptr *dt)
1511 const struct x86_emulate_ops *ops = ctxt->ops;
1514 if (selector & 1 << 2) {
1515 struct desc_struct desc;
1518 memset (dt, 0, sizeof *dt);
1519 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1523 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1524 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1526 ops->get_gdt(ctxt, dt);
1529 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1530 u16 selector, ulong *desc_addr_p)
1533 u16 index = selector >> 3;
1536 get_descriptor_table_ptr(ctxt, selector, &dt);
1538 if (dt.size < index * 8 + 7)
1539 return emulate_gp(ctxt, selector & 0xfffc);
1541 addr = dt.address + index * 8;
1543 #ifdef CONFIG_X86_64
1544 if (addr >> 32 != 0) {
1547 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1548 if (!(efer & EFER_LMA))
1553 *desc_addr_p = addr;
1554 return X86EMUL_CONTINUE;
1557 /* allowed just for 8 bytes segments */
1558 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1559 u16 selector, struct desc_struct *desc,
1564 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1565 if (rc != X86EMUL_CONTINUE)
1568 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1571 /* allowed just for 8 bytes segments */
1572 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1573 u16 selector, struct desc_struct *desc)
1578 rc = get_descriptor_ptr(ctxt, selector, &addr);
1579 if (rc != X86EMUL_CONTINUE)
1582 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1585 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1586 u16 selector, int seg, u8 cpl,
1587 enum x86_transfer_type transfer,
1588 struct desc_struct *desc)
1590 struct desc_struct seg_desc, old_desc;
1592 unsigned err_vec = GP_VECTOR;
1594 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1600 memset(&seg_desc, 0, sizeof seg_desc);
1602 if (ctxt->mode == X86EMUL_MODE_REAL) {
1603 /* set real mode segment descriptor (keep limit etc. for
1605 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1606 set_desc_base(&seg_desc, selector << 4);
1608 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1609 /* VM86 needs a clean new segment descriptor */
1610 set_desc_base(&seg_desc, selector << 4);
1611 set_desc_limit(&seg_desc, 0xffff);
1621 /* TR should be in GDT only */
1622 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1625 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1626 if (null_selector) {
1627 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1630 if (seg == VCPU_SREG_SS) {
1631 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1635 * ctxt->ops->set_segment expects the CPL to be in
1636 * SS.DPL, so fake an expand-up 32-bit data segment.
1646 /* Skip all following checks */
1650 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1651 if (ret != X86EMUL_CONTINUE)
1654 err_code = selector & 0xfffc;
1655 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1658 /* can't load system descriptor into segment selector */
1659 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1660 if (transfer == X86_TRANSFER_CALL_JMP)
1661 return X86EMUL_UNHANDLEABLE;
1666 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1675 * segment is not a writable data segment or segment
1676 * selector's RPL != CPL or segment selector's RPL != CPL
1678 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1682 if (!(seg_desc.type & 8))
1685 if (seg_desc.type & 4) {
1691 if (rpl > cpl || dpl != cpl)
1694 /* in long-mode d/b must be clear if l is set */
1695 if (seg_desc.d && seg_desc.l) {
1698 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1699 if (efer & EFER_LMA)
1703 /* CS(RPL) <- CPL */
1704 selector = (selector & 0xfffc) | cpl;
1707 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1709 old_desc = seg_desc;
1710 seg_desc.type |= 2; /* busy */
1711 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1712 sizeof(seg_desc), &ctxt->exception);
1713 if (ret != X86EMUL_CONTINUE)
1716 case VCPU_SREG_LDTR:
1717 if (seg_desc.s || seg_desc.type != 2)
1720 default: /* DS, ES, FS, or GS */
1722 * segment is not a data or readable code segment or
1723 * ((segment is a data or nonconforming code segment)
1724 * and (both RPL and CPL > DPL))
1726 if ((seg_desc.type & 0xa) == 0x8 ||
1727 (((seg_desc.type & 0xc) != 0xc) &&
1728 (rpl > dpl && cpl > dpl)))
1734 /* mark segment as accessed */
1735 if (!(seg_desc.type & 1)) {
1737 ret = write_segment_descriptor(ctxt, selector,
1739 if (ret != X86EMUL_CONTINUE)
1742 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1743 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1744 if (ret != X86EMUL_CONTINUE)
1746 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1747 ((u64)base3 << 32)))
1748 return emulate_gp(ctxt, 0);
1751 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1754 return X86EMUL_CONTINUE;
1756 return emulate_exception(ctxt, err_vec, err_code, true);
1759 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1760 u16 selector, int seg)
1762 u8 cpl = ctxt->ops->cpl(ctxt);
1765 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1766 * they can load it at CPL<3 (Intel's manual says only LSS can,
1769 * However, the Intel manual says that putting IST=1/DPL=3 in
1770 * an interrupt gate will result in SS=3 (the AMD manual instead
1771 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1772 * and only forbid it here.
1774 if (seg == VCPU_SREG_SS && selector == 3 &&
1775 ctxt->mode == X86EMUL_MODE_PROT64)
1776 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1778 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1779 X86_TRANSFER_NONE, NULL);
1782 static void write_register_operand(struct operand *op)
1784 return assign_register(op->addr.reg, op->val, op->bytes);
1787 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1791 write_register_operand(op);
1794 if (ctxt->lock_prefix)
1795 return segmented_cmpxchg(ctxt,
1801 return segmented_write(ctxt,
1807 return segmented_write(ctxt,
1810 op->bytes * op->count);
1813 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1816 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1824 return X86EMUL_CONTINUE;
1827 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1829 struct segmented_address addr;
1831 rsp_increment(ctxt, -bytes);
1832 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1833 addr.seg = VCPU_SREG_SS;
1835 return segmented_write(ctxt, addr, data, bytes);
1838 static int em_push(struct x86_emulate_ctxt *ctxt)
1840 /* Disable writeback. */
1841 ctxt->dst.type = OP_NONE;
1842 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1845 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1846 void *dest, int len)
1849 struct segmented_address addr;
1851 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1852 addr.seg = VCPU_SREG_SS;
1853 rc = segmented_read(ctxt, addr, dest, len);
1854 if (rc != X86EMUL_CONTINUE)
1857 rsp_increment(ctxt, len);
1861 static int em_pop(struct x86_emulate_ctxt *ctxt)
1863 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1866 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1867 void *dest, int len)
1870 unsigned long val, change_mask;
1871 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1872 int cpl = ctxt->ops->cpl(ctxt);
1874 rc = emulate_pop(ctxt, &val, len);
1875 if (rc != X86EMUL_CONTINUE)
1878 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1879 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1880 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1881 X86_EFLAGS_AC | X86_EFLAGS_ID;
1883 switch(ctxt->mode) {
1884 case X86EMUL_MODE_PROT64:
1885 case X86EMUL_MODE_PROT32:
1886 case X86EMUL_MODE_PROT16:
1888 change_mask |= X86_EFLAGS_IOPL;
1890 change_mask |= X86_EFLAGS_IF;
1892 case X86EMUL_MODE_VM86:
1894 return emulate_gp(ctxt, 0);
1895 change_mask |= X86_EFLAGS_IF;
1897 default: /* real mode */
1898 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1902 *(unsigned long *)dest =
1903 (ctxt->eflags & ~change_mask) | (val & change_mask);
1908 static int em_popf(struct x86_emulate_ctxt *ctxt)
1910 ctxt->dst.type = OP_REG;
1911 ctxt->dst.addr.reg = &ctxt->eflags;
1912 ctxt->dst.bytes = ctxt->op_bytes;
1913 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1916 static int em_enter(struct x86_emulate_ctxt *ctxt)
1919 unsigned frame_size = ctxt->src.val;
1920 unsigned nesting_level = ctxt->src2.val & 31;
1924 return X86EMUL_UNHANDLEABLE;
1926 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1927 rc = push(ctxt, &rbp, stack_size(ctxt));
1928 if (rc != X86EMUL_CONTINUE)
1930 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1932 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1933 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1935 return X86EMUL_CONTINUE;
1938 static int em_leave(struct x86_emulate_ctxt *ctxt)
1940 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1942 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1945 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1947 int seg = ctxt->src2.val;
1949 ctxt->src.val = get_segment_selector(ctxt, seg);
1950 if (ctxt->op_bytes == 4) {
1951 rsp_increment(ctxt, -2);
1955 return em_push(ctxt);
1958 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1960 int seg = ctxt->src2.val;
1961 unsigned long selector;
1964 rc = emulate_pop(ctxt, &selector, 2);
1965 if (rc != X86EMUL_CONTINUE)
1968 if (ctxt->modrm_reg == VCPU_SREG_SS)
1969 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1970 if (ctxt->op_bytes > 2)
1971 rsp_increment(ctxt, ctxt->op_bytes - 2);
1973 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1977 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1979 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1980 int rc = X86EMUL_CONTINUE;
1981 int reg = VCPU_REGS_RAX;
1983 while (reg <= VCPU_REGS_RDI) {
1984 (reg == VCPU_REGS_RSP) ?
1985 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1988 if (rc != X86EMUL_CONTINUE)
1997 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1999 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2000 return em_push(ctxt);
2003 static int em_popa(struct x86_emulate_ctxt *ctxt)
2005 int rc = X86EMUL_CONTINUE;
2006 int reg = VCPU_REGS_RDI;
2009 while (reg >= VCPU_REGS_RAX) {
2010 if (reg == VCPU_REGS_RSP) {
2011 rsp_increment(ctxt, ctxt->op_bytes);
2015 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2016 if (rc != X86EMUL_CONTINUE)
2018 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2024 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2026 const struct x86_emulate_ops *ops = ctxt->ops;
2033 /* TODO: Add limit checks */
2034 ctxt->src.val = ctxt->eflags;
2036 if (rc != X86EMUL_CONTINUE)
2039 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2041 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2043 if (rc != X86EMUL_CONTINUE)
2046 ctxt->src.val = ctxt->_eip;
2048 if (rc != X86EMUL_CONTINUE)
2051 ops->get_idt(ctxt, &dt);
2053 eip_addr = dt.address + (irq << 2);
2054 cs_addr = dt.address + (irq << 2) + 2;
2056 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2057 if (rc != X86EMUL_CONTINUE)
2060 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2061 if (rc != X86EMUL_CONTINUE)
2064 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2065 if (rc != X86EMUL_CONTINUE)
2073 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2077 invalidate_registers(ctxt);
2078 rc = __emulate_int_real(ctxt, irq);
2079 if (rc == X86EMUL_CONTINUE)
2080 writeback_registers(ctxt);
2084 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2086 switch(ctxt->mode) {
2087 case X86EMUL_MODE_REAL:
2088 return __emulate_int_real(ctxt, irq);
2089 case X86EMUL_MODE_VM86:
2090 case X86EMUL_MODE_PROT16:
2091 case X86EMUL_MODE_PROT32:
2092 case X86EMUL_MODE_PROT64:
2094 /* Protected mode interrupts unimplemented yet */
2095 return X86EMUL_UNHANDLEABLE;
2099 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2101 int rc = X86EMUL_CONTINUE;
2102 unsigned long temp_eip = 0;
2103 unsigned long temp_eflags = 0;
2104 unsigned long cs = 0;
2105 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2106 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2107 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2108 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2109 X86_EFLAGS_AC | X86_EFLAGS_ID |
2111 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2114 /* TODO: Add stack limit check */
2116 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2118 if (rc != X86EMUL_CONTINUE)
2121 if (temp_eip & ~0xffff)
2122 return emulate_gp(ctxt, 0);
2124 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2126 if (rc != X86EMUL_CONTINUE)
2129 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2131 if (rc != X86EMUL_CONTINUE)
2134 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2136 if (rc != X86EMUL_CONTINUE)
2139 ctxt->_eip = temp_eip;
2141 if (ctxt->op_bytes == 4)
2142 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2143 else if (ctxt->op_bytes == 2) {
2144 ctxt->eflags &= ~0xffff;
2145 ctxt->eflags |= temp_eflags;
2148 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2149 ctxt->eflags |= X86_EFLAGS_FIXED;
2150 ctxt->ops->set_nmi_mask(ctxt, false);
2155 static int em_iret(struct x86_emulate_ctxt *ctxt)
2157 switch(ctxt->mode) {
2158 case X86EMUL_MODE_REAL:
2159 return emulate_iret_real(ctxt);
2160 case X86EMUL_MODE_VM86:
2161 case X86EMUL_MODE_PROT16:
2162 case X86EMUL_MODE_PROT32:
2163 case X86EMUL_MODE_PROT64:
2165 /* iret from protected mode unimplemented yet */
2166 return X86EMUL_UNHANDLEABLE;
2170 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2174 struct desc_struct new_desc;
2175 u8 cpl = ctxt->ops->cpl(ctxt);
2177 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2179 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2180 X86_TRANSFER_CALL_JMP,
2182 if (rc != X86EMUL_CONTINUE)
2185 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2186 /* Error handling is not implemented. */
2187 if (rc != X86EMUL_CONTINUE)
2188 return X86EMUL_UNHANDLEABLE;
2193 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2195 return assign_eip_near(ctxt, ctxt->src.val);
2198 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2203 old_eip = ctxt->_eip;
2204 rc = assign_eip_near(ctxt, ctxt->src.val);
2205 if (rc != X86EMUL_CONTINUE)
2207 ctxt->src.val = old_eip;
2212 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2214 u64 old = ctxt->dst.orig_val64;
2216 if (ctxt->dst.bytes == 16)
2217 return X86EMUL_UNHANDLEABLE;
2219 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2220 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2221 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2222 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2223 ctxt->eflags &= ~X86_EFLAGS_ZF;
2225 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2226 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2228 ctxt->eflags |= X86_EFLAGS_ZF;
2230 return X86EMUL_CONTINUE;
2233 static int em_ret(struct x86_emulate_ctxt *ctxt)
2238 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2239 if (rc != X86EMUL_CONTINUE)
2242 return assign_eip_near(ctxt, eip);
2245 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2248 unsigned long eip, cs;
2249 int cpl = ctxt->ops->cpl(ctxt);
2250 struct desc_struct new_desc;
2252 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2253 if (rc != X86EMUL_CONTINUE)
2255 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2256 if (rc != X86EMUL_CONTINUE)
2258 /* Outer-privilege level return is not implemented */
2259 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2260 return X86EMUL_UNHANDLEABLE;
2261 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2264 if (rc != X86EMUL_CONTINUE)
2266 rc = assign_eip_far(ctxt, eip, &new_desc);
2267 /* Error handling is not implemented. */
2268 if (rc != X86EMUL_CONTINUE)
2269 return X86EMUL_UNHANDLEABLE;
2274 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2278 rc = em_ret_far(ctxt);
2279 if (rc != X86EMUL_CONTINUE)
2281 rsp_increment(ctxt, ctxt->src.val);
2282 return X86EMUL_CONTINUE;
2285 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2287 /* Save real source value, then compare EAX against destination. */
2288 ctxt->dst.orig_val = ctxt->dst.val;
2289 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2290 ctxt->src.orig_val = ctxt->src.val;
2291 ctxt->src.val = ctxt->dst.orig_val;
2292 fastop(ctxt, em_cmp);
2294 if (ctxt->eflags & X86_EFLAGS_ZF) {
2295 /* Success: write back to memory; no update of EAX */
2296 ctxt->src.type = OP_NONE;
2297 ctxt->dst.val = ctxt->src.orig_val;
2299 /* Failure: write the value we saw to EAX. */
2300 ctxt->src.type = OP_REG;
2301 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2302 ctxt->src.val = ctxt->dst.orig_val;
2303 /* Create write-cycle to dest by writing the same value */
2304 ctxt->dst.val = ctxt->dst.orig_val;
2306 return X86EMUL_CONTINUE;
2309 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2311 int seg = ctxt->src2.val;
2315 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2317 rc = load_segment_descriptor(ctxt, sel, seg);
2318 if (rc != X86EMUL_CONTINUE)
2321 ctxt->dst.val = ctxt->src.val;
2325 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2327 u32 eax, ebx, ecx, edx;
2331 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2332 return edx & bit(X86_FEATURE_LM);
2335 #define GET_SMSTATE(type, smbase, offset) \
2338 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2340 if (r != X86EMUL_CONTINUE) \
2341 return X86EMUL_UNHANDLEABLE; \
2345 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2347 desc->g = (flags >> 23) & 1;
2348 desc->d = (flags >> 22) & 1;
2349 desc->l = (flags >> 21) & 1;
2350 desc->avl = (flags >> 20) & 1;
2351 desc->p = (flags >> 15) & 1;
2352 desc->dpl = (flags >> 13) & 3;
2353 desc->s = (flags >> 12) & 1;
2354 desc->type = (flags >> 8) & 15;
2357 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2359 struct desc_struct desc;
2363 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2366 offset = 0x7f84 + n * 12;
2368 offset = 0x7f2c + (n - 3) * 12;
2370 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2371 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2372 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2373 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2374 return X86EMUL_CONTINUE;
2377 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2379 struct desc_struct desc;
2384 offset = 0x7e00 + n * 16;
2386 selector = GET_SMSTATE(u16, smbase, offset);
2387 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2388 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2389 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2390 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2392 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2393 return X86EMUL_CONTINUE;
2396 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2397 u64 cr0, u64 cr3, u64 cr4)
2402 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2404 if (cr4 & X86_CR4_PCIDE) {
2409 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2411 return X86EMUL_UNHANDLEABLE;
2414 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2415 * Then enable protected mode. However, PCID cannot be enabled
2416 * if EFER.LMA=0, so set it separately.
2418 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2420 return X86EMUL_UNHANDLEABLE;
2422 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2424 return X86EMUL_UNHANDLEABLE;
2426 if (cr4 & X86_CR4_PCIDE) {
2427 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2429 return X86EMUL_UNHANDLEABLE;
2431 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2433 return X86EMUL_UNHANDLEABLE;
2438 return X86EMUL_CONTINUE;
2441 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2443 struct desc_struct desc;
2446 u32 val, cr0, cr3, cr4;
2449 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2450 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2451 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2452 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2454 for (i = 0; i < 8; i++)
2455 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2457 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2458 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2459 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2460 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2462 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2463 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2464 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2465 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2466 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2468 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2469 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2470 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2471 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2472 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2474 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2475 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2476 ctxt->ops->set_gdt(ctxt, &dt);
2478 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2479 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2480 ctxt->ops->set_idt(ctxt, &dt);
2482 for (i = 0; i < 6; i++) {
2483 int r = rsm_load_seg_32(ctxt, smbase, i);
2484 if (r != X86EMUL_CONTINUE)
2488 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2490 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2492 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2495 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2497 struct desc_struct desc;
2499 u64 val, cr0, cr3, cr4;
2504 for (i = 0; i < 16; i++)
2505 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2507 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2508 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2510 val = GET_SMSTATE(u32, smbase, 0x7f68);
2511 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2512 val = GET_SMSTATE(u32, smbase, 0x7f60);
2513 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2515 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2516 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2517 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2518 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2519 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2520 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2522 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2523 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2524 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2525 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2526 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2527 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2529 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2530 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2531 ctxt->ops->set_idt(ctxt, &dt);
2533 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2534 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2535 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2536 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2537 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2538 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2540 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2541 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2542 ctxt->ops->set_gdt(ctxt, &dt);
2544 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2545 if (r != X86EMUL_CONTINUE)
2548 for (i = 0; i < 6; i++) {
2549 r = rsm_load_seg_64(ctxt, smbase, i);
2550 if (r != X86EMUL_CONTINUE)
2554 return X86EMUL_CONTINUE;
2557 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2559 unsigned long cr0, cr4, efer;
2563 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2564 return emulate_ud(ctxt);
2567 * Get back to real mode, to prepare a safe state in which to load
2568 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2569 * supports long mode.
2571 if (emulator_has_longmode(ctxt)) {
2572 struct desc_struct cs_desc;
2574 /* Zero CR4.PCIDE before CR0.PG. */
2575 cr4 = ctxt->ops->get_cr(ctxt, 4);
2576 if (cr4 & X86_CR4_PCIDE)
2577 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2579 /* A 32-bit code segment is required to clear EFER.LMA. */
2580 memset(&cs_desc, 0, sizeof(cs_desc));
2582 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2583 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2586 /* For the 64-bit case, this will clear EFER.LMA. */
2587 cr0 = ctxt->ops->get_cr(ctxt, 0);
2588 if (cr0 & X86_CR0_PE)
2589 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2591 if (emulator_has_longmode(ctxt)) {
2592 /* Clear CR4.PAE before clearing EFER.LME. */
2593 cr4 = ctxt->ops->get_cr(ctxt, 4);
2594 if (cr4 & X86_CR4_PAE)
2595 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2597 /* And finally go back to 32-bit mode. */
2599 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2602 smbase = ctxt->ops->get_smbase(ctxt);
2603 if (emulator_has_longmode(ctxt))
2604 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2606 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2608 if (ret != X86EMUL_CONTINUE) {
2609 /* FIXME: should triple fault */
2610 return X86EMUL_UNHANDLEABLE;
2613 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2614 ctxt->ops->set_nmi_mask(ctxt, false);
2616 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2617 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2618 return X86EMUL_CONTINUE;
2622 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2623 struct desc_struct *cs, struct desc_struct *ss)
2625 cs->l = 0; /* will be adjusted later */
2626 set_desc_base(cs, 0); /* flat segment */
2627 cs->g = 1; /* 4kb granularity */
2628 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2629 cs->type = 0x0b; /* Read, Execute, Accessed */
2631 cs->dpl = 0; /* will be adjusted later */
2636 set_desc_base(ss, 0); /* flat segment */
2637 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2638 ss->g = 1; /* 4kb granularity */
2640 ss->type = 0x03; /* Read/Write, Accessed */
2641 ss->d = 1; /* 32bit stack segment */
2648 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2650 u32 eax, ebx, ecx, edx;
2653 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2654 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2655 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2656 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2659 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2661 const struct x86_emulate_ops *ops = ctxt->ops;
2662 u32 eax, ebx, ecx, edx;
2665 * syscall should always be enabled in longmode - so only become
2666 * vendor specific (cpuid) if other modes are active...
2668 if (ctxt->mode == X86EMUL_MODE_PROT64)
2673 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2675 * Intel ("GenuineIntel")
2676 * remark: Intel CPUs only support "syscall" in 64bit
2677 * longmode. Also an 64bit guest with a
2678 * 32bit compat-app running will #UD !! While this
2679 * behaviour can be fixed (by emulating) into AMD
2680 * response - CPUs of AMD can't behave like Intel.
2682 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2683 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2684 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2687 /* AMD ("AuthenticAMD") */
2688 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2689 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2690 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2693 /* AMD ("AMDisbetter!") */
2694 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2695 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2696 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2699 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2703 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2705 const struct x86_emulate_ops *ops = ctxt->ops;
2706 struct desc_struct cs, ss;
2711 /* syscall is not available in real mode */
2712 if (ctxt->mode == X86EMUL_MODE_REAL ||
2713 ctxt->mode == X86EMUL_MODE_VM86)
2714 return emulate_ud(ctxt);
2716 if (!(em_syscall_is_enabled(ctxt)))
2717 return emulate_ud(ctxt);
2719 ops->get_msr(ctxt, MSR_EFER, &efer);
2720 setup_syscalls_segments(ctxt, &cs, &ss);
2722 if (!(efer & EFER_SCE))
2723 return emulate_ud(ctxt);
2725 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2727 cs_sel = (u16)(msr_data & 0xfffc);
2728 ss_sel = (u16)(msr_data + 8);
2730 if (efer & EFER_LMA) {
2734 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2735 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2737 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2738 if (efer & EFER_LMA) {
2739 #ifdef CONFIG_X86_64
2740 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2743 ctxt->mode == X86EMUL_MODE_PROT64 ?
2744 MSR_LSTAR : MSR_CSTAR, &msr_data);
2745 ctxt->_eip = msr_data;
2747 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2748 ctxt->eflags &= ~msr_data;
2749 ctxt->eflags |= X86_EFLAGS_FIXED;
2753 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2754 ctxt->_eip = (u32)msr_data;
2756 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2759 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2760 return X86EMUL_CONTINUE;
2763 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2765 const struct x86_emulate_ops *ops = ctxt->ops;
2766 struct desc_struct cs, ss;
2771 ops->get_msr(ctxt, MSR_EFER, &efer);
2772 /* inject #GP if in real mode */
2773 if (ctxt->mode == X86EMUL_MODE_REAL)
2774 return emulate_gp(ctxt, 0);
2777 * Not recognized on AMD in compat mode (but is recognized in legacy
2780 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2781 && !vendor_intel(ctxt))
2782 return emulate_ud(ctxt);
2784 /* sysenter/sysexit have not been tested in 64bit mode. */
2785 if (ctxt->mode == X86EMUL_MODE_PROT64)
2786 return X86EMUL_UNHANDLEABLE;
2788 setup_syscalls_segments(ctxt, &cs, &ss);
2790 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2791 if ((msr_data & 0xfffc) == 0x0)
2792 return emulate_gp(ctxt, 0);
2794 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2795 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2796 ss_sel = cs_sel + 8;
2797 if (efer & EFER_LMA) {
2802 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2803 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2805 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2806 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2808 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2809 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2812 return X86EMUL_CONTINUE;
2815 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2817 const struct x86_emulate_ops *ops = ctxt->ops;
2818 struct desc_struct cs, ss;
2819 u64 msr_data, rcx, rdx;
2821 u16 cs_sel = 0, ss_sel = 0;
2823 /* inject #GP if in real mode or Virtual 8086 mode */
2824 if (ctxt->mode == X86EMUL_MODE_REAL ||
2825 ctxt->mode == X86EMUL_MODE_VM86)
2826 return emulate_gp(ctxt, 0);
2828 setup_syscalls_segments(ctxt, &cs, &ss);
2830 if ((ctxt->rex_prefix & 0x8) != 0x0)
2831 usermode = X86EMUL_MODE_PROT64;
2833 usermode = X86EMUL_MODE_PROT32;
2835 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2836 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2840 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2842 case X86EMUL_MODE_PROT32:
2843 cs_sel = (u16)(msr_data + 16);
2844 if ((msr_data & 0xfffc) == 0x0)
2845 return emulate_gp(ctxt, 0);
2846 ss_sel = (u16)(msr_data + 24);
2850 case X86EMUL_MODE_PROT64:
2851 cs_sel = (u16)(msr_data + 32);
2852 if (msr_data == 0x0)
2853 return emulate_gp(ctxt, 0);
2854 ss_sel = cs_sel + 8;
2857 if (is_noncanonical_address(rcx) ||
2858 is_noncanonical_address(rdx))
2859 return emulate_gp(ctxt, 0);
2862 cs_sel |= SEGMENT_RPL_MASK;
2863 ss_sel |= SEGMENT_RPL_MASK;
2865 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2866 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2869 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2871 return X86EMUL_CONTINUE;
2874 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2877 if (ctxt->mode == X86EMUL_MODE_REAL)
2879 if (ctxt->mode == X86EMUL_MODE_VM86)
2881 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2882 return ctxt->ops->cpl(ctxt) > iopl;
2885 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2888 const struct x86_emulate_ops *ops = ctxt->ops;
2889 struct desc_struct tr_seg;
2892 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2893 unsigned mask = (1 << len) - 1;
2896 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2899 if (desc_limit_scaled(&tr_seg) < 103)
2901 base = get_desc_base(&tr_seg);
2902 #ifdef CONFIG_X86_64
2903 base |= ((u64)base3) << 32;
2905 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2906 if (r != X86EMUL_CONTINUE)
2908 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2910 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2911 if (r != X86EMUL_CONTINUE)
2913 if ((perm >> bit_idx) & mask)
2918 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2924 if (emulator_bad_iopl(ctxt))
2925 if (!emulator_io_port_access_allowed(ctxt, port, len))
2928 ctxt->perm_ok = true;
2933 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2936 * Intel CPUs mask the counter and pointers in quite strange
2937 * manner when ECX is zero due to REP-string optimizations.
2939 #ifdef CONFIG_X86_64
2940 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2943 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2946 case 0xa4: /* movsb */
2947 case 0xa5: /* movsd/w */
2948 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2950 case 0xaa: /* stosb */
2951 case 0xab: /* stosd/w */
2952 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2957 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2958 struct tss_segment_16 *tss)
2960 tss->ip = ctxt->_eip;
2961 tss->flag = ctxt->eflags;
2962 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2963 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2964 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2965 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2966 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2967 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2968 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2969 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2971 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2972 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2973 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2974 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2975 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2978 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2979 struct tss_segment_16 *tss)
2984 ctxt->_eip = tss->ip;
2985 ctxt->eflags = tss->flag | 2;
2986 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2987 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2988 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2989 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2990 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2991 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2992 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2993 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2996 * SDM says that segment selectors are loaded before segment
2999 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3000 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3001 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3002 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3003 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3008 * Now load segment descriptors. If fault happens at this stage
3009 * it is handled in a context of new task
3011 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3012 X86_TRANSFER_TASK_SWITCH, NULL);
3013 if (ret != X86EMUL_CONTINUE)
3015 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3016 X86_TRANSFER_TASK_SWITCH, NULL);
3017 if (ret != X86EMUL_CONTINUE)
3019 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3020 X86_TRANSFER_TASK_SWITCH, NULL);
3021 if (ret != X86EMUL_CONTINUE)
3023 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3024 X86_TRANSFER_TASK_SWITCH, NULL);
3025 if (ret != X86EMUL_CONTINUE)
3027 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3028 X86_TRANSFER_TASK_SWITCH, NULL);
3029 if (ret != X86EMUL_CONTINUE)
3032 return X86EMUL_CONTINUE;
3035 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3036 u16 tss_selector, u16 old_tss_sel,
3037 ulong old_tss_base, struct desc_struct *new_desc)
3039 struct tss_segment_16 tss_seg;
3041 u32 new_tss_base = get_desc_base(new_desc);
3043 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3044 if (ret != X86EMUL_CONTINUE)
3047 save_state_to_tss16(ctxt, &tss_seg);
3049 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3050 if (ret != X86EMUL_CONTINUE)
3053 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3054 if (ret != X86EMUL_CONTINUE)
3057 if (old_tss_sel != 0xffff) {
3058 tss_seg.prev_task_link = old_tss_sel;
3060 ret = linear_write_system(ctxt, new_tss_base,
3061 &tss_seg.prev_task_link,
3062 sizeof tss_seg.prev_task_link);
3063 if (ret != X86EMUL_CONTINUE)
3067 return load_state_from_tss16(ctxt, &tss_seg);
3070 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3071 struct tss_segment_32 *tss)
3073 /* CR3 and ldt selector are not saved intentionally */
3074 tss->eip = ctxt->_eip;
3075 tss->eflags = ctxt->eflags;
3076 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3077 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3078 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3079 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3080 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3081 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3082 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3083 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3085 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3086 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3087 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3088 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3089 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3090 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3093 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3094 struct tss_segment_32 *tss)
3099 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3100 return emulate_gp(ctxt, 0);
3101 ctxt->_eip = tss->eip;
3102 ctxt->eflags = tss->eflags | 2;
3104 /* General purpose registers */
3105 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3106 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3107 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3108 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3109 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3110 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3111 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3112 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3115 * SDM says that segment selectors are loaded before segment
3116 * descriptors. This is important because CPL checks will
3119 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3120 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3121 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3122 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3123 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3124 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3125 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3128 * If we're switching between Protected Mode and VM86, we need to make
3129 * sure to update the mode before loading the segment descriptors so
3130 * that the selectors are interpreted correctly.
3132 if (ctxt->eflags & X86_EFLAGS_VM) {
3133 ctxt->mode = X86EMUL_MODE_VM86;
3136 ctxt->mode = X86EMUL_MODE_PROT32;
3141 * Now load segment descriptors. If fault happenes at this stage
3142 * it is handled in a context of new task
3144 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3145 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3146 if (ret != X86EMUL_CONTINUE)
3148 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3149 X86_TRANSFER_TASK_SWITCH, NULL);
3150 if (ret != X86EMUL_CONTINUE)
3152 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3153 X86_TRANSFER_TASK_SWITCH, NULL);
3154 if (ret != X86EMUL_CONTINUE)
3156 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3157 X86_TRANSFER_TASK_SWITCH, NULL);
3158 if (ret != X86EMUL_CONTINUE)
3160 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3161 X86_TRANSFER_TASK_SWITCH, NULL);
3162 if (ret != X86EMUL_CONTINUE)
3164 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3165 X86_TRANSFER_TASK_SWITCH, NULL);
3166 if (ret != X86EMUL_CONTINUE)
3168 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3169 X86_TRANSFER_TASK_SWITCH, NULL);
3174 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3175 u16 tss_selector, u16 old_tss_sel,
3176 ulong old_tss_base, struct desc_struct *new_desc)
3178 struct tss_segment_32 tss_seg;
3180 u32 new_tss_base = get_desc_base(new_desc);
3181 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3182 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3184 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3185 if (ret != X86EMUL_CONTINUE)
3188 save_state_to_tss32(ctxt, &tss_seg);
3190 /* Only GP registers and segment selectors are saved */
3191 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3192 ldt_sel_offset - eip_offset);
3193 if (ret != X86EMUL_CONTINUE)
3196 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3197 if (ret != X86EMUL_CONTINUE)
3200 if (old_tss_sel != 0xffff) {
3201 tss_seg.prev_task_link = old_tss_sel;
3203 ret = linear_write_system(ctxt, new_tss_base,
3204 &tss_seg.prev_task_link,
3205 sizeof tss_seg.prev_task_link);
3206 if (ret != X86EMUL_CONTINUE)
3210 return load_state_from_tss32(ctxt, &tss_seg);
3213 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3214 u16 tss_selector, int idt_index, int reason,
3215 bool has_error_code, u32 error_code)
3217 const struct x86_emulate_ops *ops = ctxt->ops;
3218 struct desc_struct curr_tss_desc, next_tss_desc;
3220 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3221 ulong old_tss_base =
3222 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3224 ulong desc_addr, dr7;
3226 /* FIXME: old_tss_base == ~0 ? */
3228 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3229 if (ret != X86EMUL_CONTINUE)
3231 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3232 if (ret != X86EMUL_CONTINUE)
3235 /* FIXME: check that next_tss_desc is tss */
3238 * Check privileges. The three cases are task switch caused by...
3240 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3241 * 2. Exception/IRQ/iret: No check is performed
3242 * 3. jmp/call to TSS/task-gate: No check is performed since the
3243 * hardware checks it before exiting.
3245 if (reason == TASK_SWITCH_GATE) {
3246 if (idt_index != -1) {
3247 /* Software interrupts */
3248 struct desc_struct task_gate_desc;
3251 ret = read_interrupt_descriptor(ctxt, idt_index,
3253 if (ret != X86EMUL_CONTINUE)
3256 dpl = task_gate_desc.dpl;
3257 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3258 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3262 desc_limit = desc_limit_scaled(&next_tss_desc);
3263 if (!next_tss_desc.p ||
3264 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3265 desc_limit < 0x2b)) {
3266 return emulate_ts(ctxt, tss_selector & 0xfffc);
3269 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3270 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3271 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3274 if (reason == TASK_SWITCH_IRET)
3275 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3277 /* set back link to prev task only if NT bit is set in eflags
3278 note that old_tss_sel is not used after this point */
3279 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3280 old_tss_sel = 0xffff;
3282 if (next_tss_desc.type & 8)
3283 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3284 old_tss_base, &next_tss_desc);
3286 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3287 old_tss_base, &next_tss_desc);
3288 if (ret != X86EMUL_CONTINUE)
3291 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3292 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3294 if (reason != TASK_SWITCH_IRET) {
3295 next_tss_desc.type |= (1 << 1); /* set busy flag */
3296 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3299 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3300 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3302 if (has_error_code) {
3303 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3304 ctxt->lock_prefix = 0;
3305 ctxt->src.val = (unsigned long) error_code;
3306 ret = em_push(ctxt);
3309 ops->get_dr(ctxt, 7, &dr7);
3310 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3315 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3316 u16 tss_selector, int idt_index, int reason,
3317 bool has_error_code, u32 error_code)
3321 invalidate_registers(ctxt);
3322 ctxt->_eip = ctxt->eip;
3323 ctxt->dst.type = OP_NONE;
3325 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3326 has_error_code, error_code);
3328 if (rc == X86EMUL_CONTINUE) {
3329 ctxt->eip = ctxt->_eip;
3330 writeback_registers(ctxt);
3333 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3336 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3339 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3341 register_address_increment(ctxt, reg, df * op->bytes);
3342 op->addr.mem.ea = register_address(ctxt, reg);
3345 static int em_das(struct x86_emulate_ctxt *ctxt)
3348 bool af, cf, old_cf;
3350 cf = ctxt->eflags & X86_EFLAGS_CF;
3356 af = ctxt->eflags & X86_EFLAGS_AF;
3357 if ((al & 0x0f) > 9 || af) {
3359 cf = old_cf | (al >= 250);
3364 if (old_al > 0x99 || old_cf) {
3370 /* Set PF, ZF, SF */
3371 ctxt->src.type = OP_IMM;
3373 ctxt->src.bytes = 1;
3374 fastop(ctxt, em_or);
3375 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3377 ctxt->eflags |= X86_EFLAGS_CF;
3379 ctxt->eflags |= X86_EFLAGS_AF;
3380 return X86EMUL_CONTINUE;
3383 static int em_aam(struct x86_emulate_ctxt *ctxt)
3387 if (ctxt->src.val == 0)
3388 return emulate_de(ctxt);
3390 al = ctxt->dst.val & 0xff;
3391 ah = al / ctxt->src.val;
3392 al %= ctxt->src.val;
3394 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3396 /* Set PF, ZF, SF */
3397 ctxt->src.type = OP_IMM;
3399 ctxt->src.bytes = 1;
3400 fastop(ctxt, em_or);
3402 return X86EMUL_CONTINUE;
3405 static int em_aad(struct x86_emulate_ctxt *ctxt)
3407 u8 al = ctxt->dst.val & 0xff;
3408 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3410 al = (al + (ah * ctxt->src.val)) & 0xff;
3412 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3414 /* Set PF, ZF, SF */
3415 ctxt->src.type = OP_IMM;
3417 ctxt->src.bytes = 1;
3418 fastop(ctxt, em_or);
3420 return X86EMUL_CONTINUE;
3423 static int em_call(struct x86_emulate_ctxt *ctxt)
3426 long rel = ctxt->src.val;
3428 ctxt->src.val = (unsigned long)ctxt->_eip;
3429 rc = jmp_rel(ctxt, rel);
3430 if (rc != X86EMUL_CONTINUE)
3432 return em_push(ctxt);
3435 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3440 struct desc_struct old_desc, new_desc;
3441 const struct x86_emulate_ops *ops = ctxt->ops;
3442 int cpl = ctxt->ops->cpl(ctxt);
3443 enum x86emul_mode prev_mode = ctxt->mode;
3445 old_eip = ctxt->_eip;
3446 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3448 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3449 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3450 X86_TRANSFER_CALL_JMP, &new_desc);
3451 if (rc != X86EMUL_CONTINUE)
3454 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3455 if (rc != X86EMUL_CONTINUE)
3458 ctxt->src.val = old_cs;
3460 if (rc != X86EMUL_CONTINUE)
3463 ctxt->src.val = old_eip;
3465 /* If we failed, we tainted the memory, but the very least we should
3467 if (rc != X86EMUL_CONTINUE) {
3468 pr_warn_once("faulting far call emulation tainted memory\n");
3473 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3474 ctxt->mode = prev_mode;
3479 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3484 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3485 if (rc != X86EMUL_CONTINUE)
3487 rc = assign_eip_near(ctxt, eip);
3488 if (rc != X86EMUL_CONTINUE)
3490 rsp_increment(ctxt, ctxt->src.val);
3491 return X86EMUL_CONTINUE;
3494 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3496 /* Write back the register source. */
3497 ctxt->src.val = ctxt->dst.val;
3498 write_register_operand(&ctxt->src);
3500 /* Write back the memory destination with implicit LOCK prefix. */
3501 ctxt->dst.val = ctxt->src.orig_val;
3502 ctxt->lock_prefix = 1;
3503 return X86EMUL_CONTINUE;
3506 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3508 ctxt->dst.val = ctxt->src2.val;
3509 return fastop(ctxt, em_imul);
3512 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3514 ctxt->dst.type = OP_REG;
3515 ctxt->dst.bytes = ctxt->src.bytes;
3516 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3517 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3519 return X86EMUL_CONTINUE;
3522 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3526 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3527 return emulate_ud(ctxt);
3528 ctxt->dst.val = tsc_aux;
3529 return X86EMUL_CONTINUE;
3532 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3536 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3537 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3538 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3539 return X86EMUL_CONTINUE;
3542 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3546 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3547 return emulate_gp(ctxt, 0);
3548 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3549 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3550 return X86EMUL_CONTINUE;
3553 static int em_mov(struct x86_emulate_ctxt *ctxt)
3555 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3556 return X86EMUL_CONTINUE;
3559 #define FFL(x) bit(X86_FEATURE_##x)
3561 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3563 u32 ebx, ecx, edx, eax = 1;
3567 * Check MOVBE is set in the guest-visible CPUID leaf.
3569 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3570 if (!(ecx & FFL(MOVBE)))
3571 return emulate_ud(ctxt);
3573 switch (ctxt->op_bytes) {
3576 * From MOVBE definition: "...When the operand size is 16 bits,
3577 * the upper word of the destination register remains unchanged
3580 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3581 * rules so we have to do the operation almost per hand.
3583 tmp = (u16)ctxt->src.val;
3584 ctxt->dst.val &= ~0xffffUL;
3585 ctxt->dst.val |= (unsigned long)swab16(tmp);
3588 ctxt->dst.val = swab32((u32)ctxt->src.val);
3591 ctxt->dst.val = swab64(ctxt->src.val);
3596 return X86EMUL_CONTINUE;
3599 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3601 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3602 return emulate_gp(ctxt, 0);
3604 /* Disable writeback. */
3605 ctxt->dst.type = OP_NONE;
3606 return X86EMUL_CONTINUE;
3609 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3613 if (ctxt->mode == X86EMUL_MODE_PROT64)
3614 val = ctxt->src.val & ~0ULL;
3616 val = ctxt->src.val & ~0U;
3618 /* #UD condition is already handled. */
3619 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3620 return emulate_gp(ctxt, 0);
3622 /* Disable writeback. */
3623 ctxt->dst.type = OP_NONE;
3624 return X86EMUL_CONTINUE;
3627 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3631 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3632 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3633 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3634 return emulate_gp(ctxt, 0);
3636 return X86EMUL_CONTINUE;
3639 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3643 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3644 return emulate_gp(ctxt, 0);
3646 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3647 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3648 return X86EMUL_CONTINUE;
3651 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3653 if (ctxt->modrm_reg > VCPU_SREG_GS)
3654 return emulate_ud(ctxt);
3656 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3657 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3658 ctxt->dst.bytes = 2;
3659 return X86EMUL_CONTINUE;
3662 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3664 u16 sel = ctxt->src.val;
3666 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3667 return emulate_ud(ctxt);
3669 if (ctxt->modrm_reg == VCPU_SREG_SS)
3670 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3672 /* Disable writeback. */
3673 ctxt->dst.type = OP_NONE;
3674 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3677 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3679 u16 sel = ctxt->src.val;
3681 /* Disable writeback. */
3682 ctxt->dst.type = OP_NONE;
3683 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3686 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3688 u16 sel = ctxt->src.val;
3690 /* Disable writeback. */
3691 ctxt->dst.type = OP_NONE;
3692 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3695 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3700 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3701 if (rc == X86EMUL_CONTINUE)
3702 ctxt->ops->invlpg(ctxt, linear);
3703 /* Disable writeback. */
3704 ctxt->dst.type = OP_NONE;
3705 return X86EMUL_CONTINUE;
3708 static int em_clts(struct x86_emulate_ctxt *ctxt)
3712 cr0 = ctxt->ops->get_cr(ctxt, 0);
3714 ctxt->ops->set_cr(ctxt, 0, cr0);
3715 return X86EMUL_CONTINUE;
3718 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3720 int rc = ctxt->ops->fix_hypercall(ctxt);
3722 if (rc != X86EMUL_CONTINUE)
3725 /* Let the processor re-execute the fixed hypercall */
3726 ctxt->_eip = ctxt->eip;
3727 /* Disable writeback. */
3728 ctxt->dst.type = OP_NONE;
3729 return X86EMUL_CONTINUE;
3732 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3733 void (*get)(struct x86_emulate_ctxt *ctxt,
3734 struct desc_ptr *ptr))
3736 struct desc_ptr desc_ptr;
3738 if (ctxt->mode == X86EMUL_MODE_PROT64)
3740 get(ctxt, &desc_ptr);
3741 if (ctxt->op_bytes == 2) {
3743 desc_ptr.address &= 0x00ffffff;
3745 /* Disable writeback. */
3746 ctxt->dst.type = OP_NONE;
3747 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3748 &desc_ptr, 2 + ctxt->op_bytes);
3751 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3753 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3756 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3758 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3761 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3763 struct desc_ptr desc_ptr;
3766 if (ctxt->mode == X86EMUL_MODE_PROT64)
3768 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3769 &desc_ptr.size, &desc_ptr.address,
3771 if (rc != X86EMUL_CONTINUE)
3773 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3774 is_noncanonical_address(desc_ptr.address))
3775 return emulate_gp(ctxt, 0);
3777 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3779 ctxt->ops->set_idt(ctxt, &desc_ptr);
3780 /* Disable writeback. */
3781 ctxt->dst.type = OP_NONE;
3782 return X86EMUL_CONTINUE;
3785 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3787 return em_lgdt_lidt(ctxt, true);
3790 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3792 return em_lgdt_lidt(ctxt, false);
3795 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3797 if (ctxt->dst.type == OP_MEM)
3798 ctxt->dst.bytes = 2;
3799 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3800 return X86EMUL_CONTINUE;
3803 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3805 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3806 | (ctxt->src.val & 0x0f));
3807 ctxt->dst.type = OP_NONE;
3808 return X86EMUL_CONTINUE;
3811 static int em_loop(struct x86_emulate_ctxt *ctxt)
3813 int rc = X86EMUL_CONTINUE;
3815 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3816 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3817 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3818 rc = jmp_rel(ctxt, ctxt->src.val);
3823 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3825 int rc = X86EMUL_CONTINUE;
3827 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3828 rc = jmp_rel(ctxt, ctxt->src.val);
3833 static int em_in(struct x86_emulate_ctxt *ctxt)
3835 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3837 return X86EMUL_IO_NEEDED;
3839 return X86EMUL_CONTINUE;
3842 static int em_out(struct x86_emulate_ctxt *ctxt)
3844 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3846 /* Disable writeback. */
3847 ctxt->dst.type = OP_NONE;
3848 return X86EMUL_CONTINUE;
3851 static int em_cli(struct x86_emulate_ctxt *ctxt)
3853 if (emulator_bad_iopl(ctxt))
3854 return emulate_gp(ctxt, 0);
3856 ctxt->eflags &= ~X86_EFLAGS_IF;
3857 return X86EMUL_CONTINUE;
3860 static int em_sti(struct x86_emulate_ctxt *ctxt)
3862 if (emulator_bad_iopl(ctxt))
3863 return emulate_gp(ctxt, 0);
3865 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3866 ctxt->eflags |= X86_EFLAGS_IF;
3867 return X86EMUL_CONTINUE;
3870 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3872 u32 eax, ebx, ecx, edx;
3874 eax = reg_read(ctxt, VCPU_REGS_RAX);
3875 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3876 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3877 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3878 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3879 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3880 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3881 return X86EMUL_CONTINUE;
3884 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3888 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3890 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3892 ctxt->eflags &= ~0xffUL;
3893 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3894 return X86EMUL_CONTINUE;
3897 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3899 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3900 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3901 return X86EMUL_CONTINUE;
3904 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3906 switch (ctxt->op_bytes) {
3907 #ifdef CONFIG_X86_64
3909 asm("bswap %0" : "+r"(ctxt->dst.val));
3913 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3916 return X86EMUL_CONTINUE;
3919 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3921 /* emulating clflush regardless of cpuid */
3922 return X86EMUL_CONTINUE;
3925 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3927 /* emulating clflushopt regardless of cpuid */
3928 return X86EMUL_CONTINUE;
3931 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3933 ctxt->dst.val = (s32) ctxt->src.val;
3934 return X86EMUL_CONTINUE;
3937 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3939 u32 eax = 1, ebx, ecx = 0, edx;
3941 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3942 if (!(edx & FFL(FXSR)))
3943 return emulate_ud(ctxt);
3945 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3946 return emulate_nm(ctxt);
3949 * Don't emulate a case that should never be hit, instead of working
3950 * around a lack of fxsave64/fxrstor64 on old compilers.
3952 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3953 return X86EMUL_UNHANDLEABLE;
3955 return X86EMUL_CONTINUE;
3959 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3962 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3963 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3965 * 3) 64-bit mode with REX.W prefix
3966 * - like (2), but XMM 8-15 are being saved and restored
3967 * 4) 64-bit mode without REX.W prefix
3968 * - like (3), but FIP and FDP are 64 bit
3970 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3971 * desired result. (4) is not emulated.
3973 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3974 * and FPU DS) should match.
3976 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
3978 struct fxregs_state fx_state;
3982 rc = check_fxsr(ctxt);
3983 if (rc != X86EMUL_CONTINUE)
3986 ctxt->ops->get_fpu(ctxt);
3988 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
3990 ctxt->ops->put_fpu(ctxt);
3992 if (rc != X86EMUL_CONTINUE)
3995 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
3996 size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
3998 size = offsetof(struct fxregs_state, xmm_space[0]);
4000 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4003 static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
4004 struct fxregs_state *new)
4006 int rc = X86EMUL_CONTINUE;
4007 struct fxregs_state old;
4009 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
4010 if (rc != X86EMUL_CONTINUE)
4014 * 64 bit host will restore XMM 8-15, which is not correct on non-64
4015 * bit guests. Load the current values in order to preserve 64 bit
4016 * XMMs after fxrstor.
4018 #ifdef CONFIG_X86_64
4019 /* XXX: accessing XMM 8-15 very awkwardly */
4020 memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
4024 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
4025 * does save and restore MXCSR.
4027 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
4028 memcpy(new->xmm_space, old.xmm_space, 8 * 16);
4033 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4035 struct fxregs_state fx_state;
4038 rc = check_fxsr(ctxt);
4039 if (rc != X86EMUL_CONTINUE)
4042 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
4043 if (rc != X86EMUL_CONTINUE)
4046 if (fx_state.mxcsr >> 16)
4047 return emulate_gp(ctxt, 0);
4049 ctxt->ops->get_fpu(ctxt);
4051 if (ctxt->mode < X86EMUL_MODE_PROT64)
4052 rc = fxrstor_fixup(ctxt, &fx_state);
4054 if (rc == X86EMUL_CONTINUE)
4055 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4057 ctxt->ops->put_fpu(ctxt);
4062 static bool valid_cr(int nr)
4074 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4076 if (!valid_cr(ctxt->modrm_reg))
4077 return emulate_ud(ctxt);
4079 return X86EMUL_CONTINUE;
4082 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4084 u64 new_val = ctxt->src.val64;
4085 int cr = ctxt->modrm_reg;
4088 static u64 cr_reserved_bits[] = {
4089 0xffffffff00000000ULL,
4090 0, 0, 0, /* CR3 checked later */
4097 return emulate_ud(ctxt);
4099 if (new_val & cr_reserved_bits[cr])
4100 return emulate_gp(ctxt, 0);
4105 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4106 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4107 return emulate_gp(ctxt, 0);
4109 cr4 = ctxt->ops->get_cr(ctxt, 4);
4110 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4112 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4113 !(cr4 & X86_CR4_PAE))
4114 return emulate_gp(ctxt, 0);
4121 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4122 if (efer & EFER_LMA)
4123 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
4126 return emulate_gp(ctxt, 0);
4131 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4133 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4134 return emulate_gp(ctxt, 0);
4140 return X86EMUL_CONTINUE;
4143 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4147 ctxt->ops->get_dr(ctxt, 7, &dr7);
4149 /* Check if DR7.Global_Enable is set */
4150 return dr7 & (1 << 13);
4153 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4155 int dr = ctxt->modrm_reg;
4159 return emulate_ud(ctxt);
4161 cr4 = ctxt->ops->get_cr(ctxt, 4);
4162 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4163 return emulate_ud(ctxt);
4165 if (check_dr7_gd(ctxt)) {
4168 ctxt->ops->get_dr(ctxt, 6, &dr6);
4170 dr6 |= DR6_BD | DR6_RTM;
4171 ctxt->ops->set_dr(ctxt, 6, dr6);
4172 return emulate_db(ctxt);
4175 return X86EMUL_CONTINUE;
4178 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4180 u64 new_val = ctxt->src.val64;
4181 int dr = ctxt->modrm_reg;
4183 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4184 return emulate_gp(ctxt, 0);
4186 return check_dr_read(ctxt);
4189 static int check_svme(struct x86_emulate_ctxt *ctxt)
4193 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4195 if (!(efer & EFER_SVME))
4196 return emulate_ud(ctxt);
4198 return X86EMUL_CONTINUE;
4201 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4203 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4205 /* Valid physical address? */
4206 if (rax & 0xffff000000000000ULL)
4207 return emulate_gp(ctxt, 0);
4209 return check_svme(ctxt);
4212 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4214 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4216 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4217 return emulate_ud(ctxt);
4219 return X86EMUL_CONTINUE;
4222 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4224 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4225 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4227 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4228 ctxt->ops->check_pmc(ctxt, rcx))
4229 return emulate_gp(ctxt, 0);
4231 return X86EMUL_CONTINUE;
4234 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4236 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4237 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4238 return emulate_gp(ctxt, 0);
4240 return X86EMUL_CONTINUE;
4243 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4245 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4246 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4247 return emulate_gp(ctxt, 0);
4249 return X86EMUL_CONTINUE;
4252 #define D(_y) { .flags = (_y) }
4253 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4254 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4255 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4256 #define N D(NotImpl)
4257 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4258 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4259 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4260 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4261 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4262 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4263 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4264 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4265 #define II(_f, _e, _i) \
4266 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4267 #define IIP(_f, _e, _i, _p) \
4268 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4269 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4270 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4272 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4273 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4274 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4275 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4276 #define I2bvIP(_f, _e, _i, _p) \
4277 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4279 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4280 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4281 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4283 static const struct opcode group7_rm0[] = {
4285 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4289 static const struct opcode group7_rm1[] = {
4290 DI(SrcNone | Priv, monitor),
4291 DI(SrcNone | Priv, mwait),
4295 static const struct opcode group7_rm3[] = {
4296 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4297 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4298 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4299 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4300 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4301 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4302 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4303 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4306 static const struct opcode group7_rm7[] = {
4308 DIP(SrcNone, rdtscp, check_rdtsc),
4312 static const struct opcode group1[] = {
4314 F(Lock | PageTable, em_or),
4317 F(Lock | PageTable, em_and),
4323 static const struct opcode group1A[] = {
4324 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4327 static const struct opcode group2[] = {
4328 F(DstMem | ModRM, em_rol),
4329 F(DstMem | ModRM, em_ror),
4330 F(DstMem | ModRM, em_rcl),
4331 F(DstMem | ModRM, em_rcr),
4332 F(DstMem | ModRM, em_shl),
4333 F(DstMem | ModRM, em_shr),
4334 F(DstMem | ModRM, em_shl),
4335 F(DstMem | ModRM, em_sar),
4338 static const struct opcode group3[] = {
4339 F(DstMem | SrcImm | NoWrite, em_test),
4340 F(DstMem | SrcImm | NoWrite, em_test),
4341 F(DstMem | SrcNone | Lock, em_not),
4342 F(DstMem | SrcNone | Lock, em_neg),
4343 F(DstXacc | Src2Mem, em_mul_ex),
4344 F(DstXacc | Src2Mem, em_imul_ex),
4345 F(DstXacc | Src2Mem, em_div_ex),
4346 F(DstXacc | Src2Mem, em_idiv_ex),
4349 static const struct opcode group4[] = {
4350 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4351 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4355 static const struct opcode group5[] = {
4356 F(DstMem | SrcNone | Lock, em_inc),
4357 F(DstMem | SrcNone | Lock, em_dec),
4358 I(SrcMem | NearBranch, em_call_near_abs),
4359 I(SrcMemFAddr | ImplicitOps, em_call_far),
4360 I(SrcMem | NearBranch, em_jmp_abs),
4361 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4362 I(SrcMem | Stack, em_push), D(Undefined),
4365 static const struct opcode group6[] = {
4366 DI(Prot | DstMem, sldt),
4367 DI(Prot | DstMem, str),
4368 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4369 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4373 static const struct group_dual group7 = { {
4374 II(Mov | DstMem, em_sgdt, sgdt),
4375 II(Mov | DstMem, em_sidt, sidt),
4376 II(SrcMem | Priv, em_lgdt, lgdt),
4377 II(SrcMem | Priv, em_lidt, lidt),
4378 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4379 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4380 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4384 N, EXT(0, group7_rm3),
4385 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4386 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4390 static const struct opcode group8[] = {
4392 F(DstMem | SrcImmByte | NoWrite, em_bt),
4393 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4394 F(DstMem | SrcImmByte | Lock, em_btr),
4395 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4399 * The "memory" destination is actually always a register, since we come
4400 * from the register case of group9.
4402 static const struct gprefix pfx_0f_c7_7 = {
4403 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4407 static const struct group_dual group9 = { {
4408 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4410 N, N, N, N, N, N, N,
4411 GP(0, &pfx_0f_c7_7),
4414 static const struct opcode group11[] = {
4415 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4419 static const struct gprefix pfx_0f_ae_7 = {
4420 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4423 static const struct group_dual group15 = { {
4424 I(ModRM | Aligned16, em_fxsave),
4425 I(ModRM | Aligned16, em_fxrstor),
4426 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4428 N, N, N, N, N, N, N, N,
4431 static const struct gprefix pfx_0f_6f_0f_7f = {
4432 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4435 static const struct instr_dual instr_dual_0f_2b = {
4439 static const struct gprefix pfx_0f_2b = {
4440 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4443 static const struct gprefix pfx_0f_28_0f_29 = {
4444 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4447 static const struct gprefix pfx_0f_e7 = {
4448 N, I(Sse, em_mov), N, N,
4451 static const struct escape escape_d9 = { {
4452 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4455 N, N, N, N, N, N, N, N,
4457 N, N, N, N, N, N, N, N,
4459 N, N, N, N, N, N, N, N,
4461 N, N, N, N, N, N, N, N,
4463 N, N, N, N, N, N, N, N,
4465 N, N, N, N, N, N, N, N,
4467 N, N, N, N, N, N, N, N,
4469 N, N, N, N, N, N, N, N,
4472 static const struct escape escape_db = { {
4473 N, N, N, N, N, N, N, N,
4476 N, N, N, N, N, N, N, N,
4478 N, N, N, N, N, N, N, N,
4480 N, N, N, N, N, N, N, N,
4482 N, N, N, N, N, N, N, N,
4484 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4486 N, N, N, N, N, N, N, N,
4488 N, N, N, N, N, N, N, N,
4490 N, N, N, N, N, N, N, N,
4493 static const struct escape escape_dd = { {
4494 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4497 N, N, N, N, N, N, N, N,
4499 N, N, N, N, N, N, N, N,
4501 N, N, N, N, N, N, N, N,
4503 N, N, N, N, N, N, N, N,
4505 N, N, N, N, N, N, N, N,
4507 N, N, N, N, N, N, N, N,
4509 N, N, N, N, N, N, N, N,
4511 N, N, N, N, N, N, N, N,
4514 static const struct instr_dual instr_dual_0f_c3 = {
4515 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4518 static const struct mode_dual mode_dual_63 = {
4519 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4522 static const struct opcode opcode_table[256] = {
4524 F6ALU(Lock, em_add),
4525 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4526 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4528 F6ALU(Lock | PageTable, em_or),
4529 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4532 F6ALU(Lock, em_adc),
4533 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4534 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4536 F6ALU(Lock, em_sbb),
4537 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4538 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4540 F6ALU(Lock | PageTable, em_and), N, N,
4542 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4544 F6ALU(Lock, em_xor), N, N,
4546 F6ALU(NoWrite, em_cmp), N, N,
4548 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4550 X8(I(SrcReg | Stack, em_push)),
4552 X8(I(DstReg | Stack, em_pop)),
4554 I(ImplicitOps | Stack | No64, em_pusha),
4555 I(ImplicitOps | Stack | No64, em_popa),
4556 N, MD(ModRM, &mode_dual_63),
4559 I(SrcImm | Mov | Stack, em_push),
4560 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4561 I(SrcImmByte | Mov | Stack, em_push),
4562 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4563 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4564 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4566 X16(D(SrcImmByte | NearBranch)),
4568 G(ByteOp | DstMem | SrcImm, group1),
4569 G(DstMem | SrcImm, group1),
4570 G(ByteOp | DstMem | SrcImm | No64, group1),
4571 G(DstMem | SrcImmByte, group1),
4572 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4573 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4575 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4576 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4577 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4578 D(ModRM | SrcMem | NoAccess | DstReg),
4579 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4582 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4584 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4585 I(SrcImmFAddr | No64, em_call_far), N,
4586 II(ImplicitOps | Stack, em_pushf, pushf),
4587 II(ImplicitOps | Stack, em_popf, popf),
4588 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4590 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4591 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4592 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4593 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4595 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4596 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4597 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4598 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4600 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4602 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4604 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4605 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4606 I(ImplicitOps | NearBranch, em_ret),
4607 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4608 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4609 G(ByteOp, group11), G(0, group11),
4611 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4612 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4613 I(ImplicitOps, em_ret_far),
4614 D(ImplicitOps), DI(SrcImmByte, intn),
4615 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4617 G(Src2One | ByteOp, group2), G(Src2One, group2),
4618 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4619 I(DstAcc | SrcImmUByte | No64, em_aam),
4620 I(DstAcc | SrcImmUByte | No64, em_aad),
4621 F(DstAcc | ByteOp | No64, em_salc),
4622 I(DstAcc | SrcXLat | ByteOp, em_mov),
4624 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4626 X3(I(SrcImmByte | NearBranch, em_loop)),
4627 I(SrcImmByte | NearBranch, em_jcxz),
4628 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4629 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4631 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4632 I(SrcImmFAddr | No64, em_jmp_far),
4633 D(SrcImmByte | ImplicitOps | NearBranch),
4634 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4635 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4637 N, DI(ImplicitOps, icebp), N, N,
4638 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4639 G(ByteOp, group3), G(0, group3),
4641 D(ImplicitOps), D(ImplicitOps),
4642 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4643 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4646 static const struct opcode twobyte_table[256] = {
4648 G(0, group6), GD(0, &group7), N, N,
4649 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4650 II(ImplicitOps | Priv, em_clts, clts), N,
4651 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4652 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4654 N, N, N, N, N, N, N, N,
4655 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4656 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4658 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4659 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4660 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4662 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4665 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4666 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4667 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4670 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4671 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4672 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4673 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4674 I(ImplicitOps | EmulateOnUD, em_sysenter),
4675 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4677 N, N, N, N, N, N, N, N,
4679 X16(D(DstReg | SrcMem | ModRM)),
4681 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4686 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4691 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4693 X16(D(SrcImm | NearBranch)),
4695 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4697 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4698 II(ImplicitOps, em_cpuid, cpuid),
4699 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4700 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4701 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4703 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4704 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4705 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4706 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4707 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4708 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4710 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4711 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4712 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4713 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4714 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4715 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4719 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4720 I(DstReg | SrcMem | ModRM, em_bsf_c),
4721 I(DstReg | SrcMem | ModRM, em_bsr_c),
4722 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4724 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4725 N, ID(0, &instr_dual_0f_c3),
4726 N, N, N, GD(0, &group9),
4728 X8(I(DstReg, em_bswap)),
4730 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4732 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4733 N, N, N, N, N, N, N, N,
4735 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4738 static const struct instr_dual instr_dual_0f_38_f0 = {
4739 I(DstReg | SrcMem | Mov, em_movbe), N
4742 static const struct instr_dual instr_dual_0f_38_f1 = {
4743 I(DstMem | SrcReg | Mov, em_movbe), N
4746 static const struct gprefix three_byte_0f_38_f0 = {
4747 ID(0, &instr_dual_0f_38_f0), N, N, N
4750 static const struct gprefix three_byte_0f_38_f1 = {
4751 ID(0, &instr_dual_0f_38_f1), N, N, N
4755 * Insns below are selected by the prefix which indexed by the third opcode
4758 static const struct opcode opcode_map_0f_38[256] = {
4760 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4762 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4764 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4765 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4786 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4790 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4796 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4797 unsigned size, bool sign_extension)
4799 int rc = X86EMUL_CONTINUE;
4803 op->addr.mem.ea = ctxt->_eip;
4804 /* NB. Immediates are sign-extended as necessary. */
4805 switch (op->bytes) {
4807 op->val = insn_fetch(s8, ctxt);
4810 op->val = insn_fetch(s16, ctxt);
4813 op->val = insn_fetch(s32, ctxt);
4816 op->val = insn_fetch(s64, ctxt);
4819 if (!sign_extension) {
4820 switch (op->bytes) {
4828 op->val &= 0xffffffff;
4836 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4839 int rc = X86EMUL_CONTINUE;
4843 decode_register_operand(ctxt, op);
4846 rc = decode_imm(ctxt, op, 1, false);
4849 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4853 if (ctxt->d & BitOp)
4854 fetch_bit_operand(ctxt);
4855 op->orig_val = op->val;
4858 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4862 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4863 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4864 fetch_register_operand(op);
4865 op->orig_val = op->val;
4869 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4870 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4871 fetch_register_operand(op);
4872 op->orig_val = op->val;
4875 if (ctxt->d & ByteOp) {
4880 op->bytes = ctxt->op_bytes;
4881 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4882 fetch_register_operand(op);
4883 op->orig_val = op->val;
4887 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4889 register_address(ctxt, VCPU_REGS_RDI);
4890 op->addr.mem.seg = VCPU_SREG_ES;
4897 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4898 fetch_register_operand(op);
4903 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4906 rc = decode_imm(ctxt, op, 1, true);
4914 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4917 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4920 ctxt->memop.bytes = 1;
4921 if (ctxt->memop.type == OP_REG) {
4922 ctxt->memop.addr.reg = decode_register(ctxt,
4923 ctxt->modrm_rm, true);
4924 fetch_register_operand(&ctxt->memop);
4928 ctxt->memop.bytes = 2;
4931 ctxt->memop.bytes = 4;
4934 rc = decode_imm(ctxt, op, 2, false);
4937 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4941 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4943 register_address(ctxt, VCPU_REGS_RSI);
4944 op->addr.mem.seg = ctxt->seg_override;
4950 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4953 reg_read(ctxt, VCPU_REGS_RBX) +
4954 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4955 op->addr.mem.seg = ctxt->seg_override;
4960 op->addr.mem.ea = ctxt->_eip;
4961 op->bytes = ctxt->op_bytes + 2;
4962 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4965 ctxt->memop.bytes = ctxt->op_bytes + 2;
4969 op->val = VCPU_SREG_ES;
4973 op->val = VCPU_SREG_CS;
4977 op->val = VCPU_SREG_SS;
4981 op->val = VCPU_SREG_DS;
4985 op->val = VCPU_SREG_FS;
4989 op->val = VCPU_SREG_GS;
4992 /* Special instructions do their own operand decoding. */
4994 op->type = OP_NONE; /* Disable writeback. */
5002 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5004 int rc = X86EMUL_CONTINUE;
5005 int mode = ctxt->mode;
5006 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5007 bool op_prefix = false;
5008 bool has_seg_override = false;
5009 struct opcode opcode;
5011 struct desc_struct desc;
5013 ctxt->memop.type = OP_NONE;
5014 ctxt->memopp = NULL;
5015 ctxt->_eip = ctxt->eip;
5016 ctxt->fetch.ptr = ctxt->fetch.data;
5017 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5018 ctxt->opcode_len = 1;
5019 ctxt->intercept = x86_intercept_none;
5021 memcpy(ctxt->fetch.data, insn, insn_len);
5023 rc = __do_insn_fetch_bytes(ctxt, 1);
5024 if (rc != X86EMUL_CONTINUE)
5029 case X86EMUL_MODE_REAL:
5030 case X86EMUL_MODE_VM86:
5031 def_op_bytes = def_ad_bytes = 2;
5032 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5034 def_op_bytes = def_ad_bytes = 4;
5036 case X86EMUL_MODE_PROT16:
5037 def_op_bytes = def_ad_bytes = 2;
5039 case X86EMUL_MODE_PROT32:
5040 def_op_bytes = def_ad_bytes = 4;
5042 #ifdef CONFIG_X86_64
5043 case X86EMUL_MODE_PROT64:
5049 return EMULATION_FAILED;
5052 ctxt->op_bytes = def_op_bytes;
5053 ctxt->ad_bytes = def_ad_bytes;
5055 /* Legacy prefixes. */
5057 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5058 case 0x66: /* operand-size override */
5060 /* switch between 2/4 bytes */
5061 ctxt->op_bytes = def_op_bytes ^ 6;
5063 case 0x67: /* address-size override */
5064 if (mode == X86EMUL_MODE_PROT64)
5065 /* switch between 4/8 bytes */
5066 ctxt->ad_bytes = def_ad_bytes ^ 12;
5068 /* switch between 2/4 bytes */
5069 ctxt->ad_bytes = def_ad_bytes ^ 6;
5071 case 0x26: /* ES override */
5072 has_seg_override = true;
5073 ctxt->seg_override = VCPU_SREG_ES;
5075 case 0x2e: /* CS override */
5076 has_seg_override = true;
5077 ctxt->seg_override = VCPU_SREG_CS;
5079 case 0x36: /* SS override */
5080 has_seg_override = true;
5081 ctxt->seg_override = VCPU_SREG_SS;
5083 case 0x3e: /* DS override */
5084 has_seg_override = true;
5085 ctxt->seg_override = VCPU_SREG_DS;
5087 case 0x64: /* FS override */
5088 has_seg_override = true;
5089 ctxt->seg_override = VCPU_SREG_FS;
5091 case 0x65: /* GS override */
5092 has_seg_override = true;
5093 ctxt->seg_override = VCPU_SREG_GS;
5095 case 0x40 ... 0x4f: /* REX */
5096 if (mode != X86EMUL_MODE_PROT64)
5098 ctxt->rex_prefix = ctxt->b;
5100 case 0xf0: /* LOCK */
5101 ctxt->lock_prefix = 1;
5103 case 0xf2: /* REPNE/REPNZ */
5104 case 0xf3: /* REP/REPE/REPZ */
5105 ctxt->rep_prefix = ctxt->b;
5111 /* Any legacy prefix after a REX prefix nullifies its effect. */
5113 ctxt->rex_prefix = 0;
5119 if (ctxt->rex_prefix & 8)
5120 ctxt->op_bytes = 8; /* REX.W */
5122 /* Opcode byte(s). */
5123 opcode = opcode_table[ctxt->b];
5124 /* Two-byte opcode? */
5125 if (ctxt->b == 0x0f) {
5126 ctxt->opcode_len = 2;
5127 ctxt->b = insn_fetch(u8, ctxt);
5128 opcode = twobyte_table[ctxt->b];
5130 /* 0F_38 opcode map */
5131 if (ctxt->b == 0x38) {
5132 ctxt->opcode_len = 3;
5133 ctxt->b = insn_fetch(u8, ctxt);
5134 opcode = opcode_map_0f_38[ctxt->b];
5137 ctxt->d = opcode.flags;
5139 if (ctxt->d & ModRM)
5140 ctxt->modrm = insn_fetch(u8, ctxt);
5142 /* vex-prefix instructions are not implemented */
5143 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5144 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5148 while (ctxt->d & GroupMask) {
5149 switch (ctxt->d & GroupMask) {
5151 goffset = (ctxt->modrm >> 3) & 7;
5152 opcode = opcode.u.group[goffset];
5155 goffset = (ctxt->modrm >> 3) & 7;
5156 if ((ctxt->modrm >> 6) == 3)
5157 opcode = opcode.u.gdual->mod3[goffset];
5159 opcode = opcode.u.gdual->mod012[goffset];
5162 goffset = ctxt->modrm & 7;
5163 opcode = opcode.u.group[goffset];
5166 if (ctxt->rep_prefix && op_prefix)
5167 return EMULATION_FAILED;
5168 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5169 switch (simd_prefix) {
5170 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5171 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5172 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5173 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5177 if (ctxt->modrm > 0xbf) {
5178 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5179 u32 index = array_index_nospec(
5180 ctxt->modrm - 0xc0, size);
5182 opcode = opcode.u.esc->high[index];
5184 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5188 if ((ctxt->modrm >> 6) == 3)
5189 opcode = opcode.u.idual->mod3;
5191 opcode = opcode.u.idual->mod012;
5194 if (ctxt->mode == X86EMUL_MODE_PROT64)
5195 opcode = opcode.u.mdual->mode64;
5197 opcode = opcode.u.mdual->mode32;
5200 return EMULATION_FAILED;
5203 ctxt->d &= ~(u64)GroupMask;
5204 ctxt->d |= opcode.flags;
5209 return EMULATION_FAILED;
5211 ctxt->execute = opcode.u.execute;
5213 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5214 return EMULATION_FAILED;
5216 if (unlikely(ctxt->d &
5217 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5220 * These are copied unconditionally here, and checked unconditionally
5221 * in x86_emulate_insn.
5223 ctxt->check_perm = opcode.check_perm;
5224 ctxt->intercept = opcode.intercept;
5226 if (ctxt->d & NotImpl)
5227 return EMULATION_FAILED;
5229 if (mode == X86EMUL_MODE_PROT64) {
5230 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5232 else if (ctxt->d & NearBranch)
5236 if (ctxt->d & Op3264) {
5237 if (mode == X86EMUL_MODE_PROT64)
5243 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5247 ctxt->op_bytes = 16;
5248 else if (ctxt->d & Mmx)
5252 /* ModRM and SIB bytes. */
5253 if (ctxt->d & ModRM) {
5254 rc = decode_modrm(ctxt, &ctxt->memop);
5255 if (!has_seg_override) {
5256 has_seg_override = true;
5257 ctxt->seg_override = ctxt->modrm_seg;
5259 } else if (ctxt->d & MemAbs)
5260 rc = decode_abs(ctxt, &ctxt->memop);
5261 if (rc != X86EMUL_CONTINUE)
5264 if (!has_seg_override)
5265 ctxt->seg_override = VCPU_SREG_DS;
5267 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5270 * Decode and fetch the source operand: register, memory
5273 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5274 if (rc != X86EMUL_CONTINUE)
5278 * Decode and fetch the second source operand: register, memory
5281 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5282 if (rc != X86EMUL_CONTINUE)
5285 /* Decode and fetch the destination operand: register or memory. */
5286 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5288 if (ctxt->rip_relative && likely(ctxt->memopp))
5289 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5290 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5293 if (rc == X86EMUL_PROPAGATE_FAULT)
5294 ctxt->have_exception = true;
5295 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5298 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5300 return ctxt->d & PageTable;
5303 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5305 /* The second termination condition only applies for REPE
5306 * and REPNE. Test if the repeat string operation prefix is
5307 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5308 * corresponding termination condition according to:
5309 * - if REPE/REPZ and ZF = 0 then done
5310 * - if REPNE/REPNZ and ZF = 1 then done
5312 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5313 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5314 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5315 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5316 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5317 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5323 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5327 ctxt->ops->get_fpu(ctxt);
5328 rc = asm_safe("fwait");
5329 ctxt->ops->put_fpu(ctxt);
5331 if (unlikely(rc != X86EMUL_CONTINUE))
5332 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5334 return X86EMUL_CONTINUE;
5337 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5340 if (op->type == OP_MM)
5341 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5344 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5346 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5347 if (!(ctxt->d & ByteOp))
5348 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5349 asm("push %[flags]; popf; " CALL_NOSPEC "; pushf; pop %[flags]\n"
5350 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5351 [thunk_target]"+S"(fop)
5352 : "c"(ctxt->src2.val));
5353 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5354 if (!fop) /* exception is returned in fop variable */
5355 return emulate_de(ctxt);
5356 return X86EMUL_CONTINUE;
5359 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5361 memset(&ctxt->rip_relative, 0,
5362 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5364 ctxt->io_read.pos = 0;
5365 ctxt->io_read.end = 0;
5366 ctxt->mem_read.end = 0;
5369 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5371 const struct x86_emulate_ops *ops = ctxt->ops;
5372 int rc = X86EMUL_CONTINUE;
5373 int saved_dst_type = ctxt->dst.type;
5374 unsigned emul_flags;
5376 ctxt->mem_read.pos = 0;
5378 /* LOCK prefix is allowed only with some instructions */
5379 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5380 rc = emulate_ud(ctxt);
5384 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5385 rc = emulate_ud(ctxt);
5389 emul_flags = ctxt->ops->get_hflags(ctxt);
5390 if (unlikely(ctxt->d &
5391 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5392 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5393 (ctxt->d & Undefined)) {
5394 rc = emulate_ud(ctxt);
5398 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5399 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5400 rc = emulate_ud(ctxt);
5404 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5405 rc = emulate_nm(ctxt);
5409 if (ctxt->d & Mmx) {
5410 rc = flush_pending_x87_faults(ctxt);
5411 if (rc != X86EMUL_CONTINUE)
5414 * Now that we know the fpu is exception safe, we can fetch
5417 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5418 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5419 if (!(ctxt->d & Mov))
5420 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5423 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5424 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5425 X86_ICPT_PRE_EXCEPT);
5426 if (rc != X86EMUL_CONTINUE)
5430 /* Instruction can only be executed in protected mode */
5431 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5432 rc = emulate_ud(ctxt);
5436 /* Privileged instruction can be executed only in CPL=0 */
5437 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5438 if (ctxt->d & PrivUD)
5439 rc = emulate_ud(ctxt);
5441 rc = emulate_gp(ctxt, 0);
5445 /* Do instruction specific permission checks */
5446 if (ctxt->d & CheckPerm) {
5447 rc = ctxt->check_perm(ctxt);
5448 if (rc != X86EMUL_CONTINUE)
5452 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5453 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5454 X86_ICPT_POST_EXCEPT);
5455 if (rc != X86EMUL_CONTINUE)
5459 if (ctxt->rep_prefix && (ctxt->d & String)) {
5460 /* All REP prefixes have the same first termination condition */
5461 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5462 string_registers_quirk(ctxt);
5463 ctxt->eip = ctxt->_eip;
5464 ctxt->eflags &= ~X86_EFLAGS_RF;
5470 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5471 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5472 ctxt->src.valptr, ctxt->src.bytes);
5473 if (rc != X86EMUL_CONTINUE)
5475 ctxt->src.orig_val64 = ctxt->src.val64;
5478 if (ctxt->src2.type == OP_MEM) {
5479 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5480 &ctxt->src2.val, ctxt->src2.bytes);
5481 if (rc != X86EMUL_CONTINUE)
5485 if ((ctxt->d & DstMask) == ImplicitOps)
5489 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5490 /* optimisation - avoid slow emulated read if Mov */
5491 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5492 &ctxt->dst.val, ctxt->dst.bytes);
5493 if (rc != X86EMUL_CONTINUE) {
5494 if (!(ctxt->d & NoWrite) &&
5495 rc == X86EMUL_PROPAGATE_FAULT &&
5496 ctxt->exception.vector == PF_VECTOR)
5497 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5501 /* Copy full 64-bit value for CMPXCHG8B. */
5502 ctxt->dst.orig_val64 = ctxt->dst.val64;
5506 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5507 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5508 X86_ICPT_POST_MEMACCESS);
5509 if (rc != X86EMUL_CONTINUE)
5513 if (ctxt->rep_prefix && (ctxt->d & String))
5514 ctxt->eflags |= X86_EFLAGS_RF;
5516 ctxt->eflags &= ~X86_EFLAGS_RF;
5518 if (ctxt->execute) {
5519 if (ctxt->d & Fastop) {
5520 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5521 rc = fastop(ctxt, fop);
5522 if (rc != X86EMUL_CONTINUE)
5526 rc = ctxt->execute(ctxt);
5527 if (rc != X86EMUL_CONTINUE)
5532 if (ctxt->opcode_len == 2)
5534 else if (ctxt->opcode_len == 3)
5535 goto threebyte_insn;
5538 case 0x70 ... 0x7f: /* jcc (short) */
5539 if (test_cc(ctxt->b, ctxt->eflags))
5540 rc = jmp_rel(ctxt, ctxt->src.val);
5542 case 0x8d: /* lea r16/r32, m */
5543 ctxt->dst.val = ctxt->src.addr.mem.ea;
5545 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5546 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5547 ctxt->dst.type = OP_NONE;
5551 case 0x98: /* cbw/cwde/cdqe */
5552 switch (ctxt->op_bytes) {
5553 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5554 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5555 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5558 case 0xcc: /* int3 */
5559 rc = emulate_int(ctxt, 3);
5561 case 0xcd: /* int n */
5562 rc = emulate_int(ctxt, ctxt->src.val);
5564 case 0xce: /* into */
5565 if (ctxt->eflags & X86_EFLAGS_OF)
5566 rc = emulate_int(ctxt, 4);
5568 case 0xe9: /* jmp rel */
5569 case 0xeb: /* jmp rel short */
5570 rc = jmp_rel(ctxt, ctxt->src.val);
5571 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5573 case 0xf4: /* hlt */
5574 ctxt->ops->halt(ctxt);
5576 case 0xf5: /* cmc */
5577 /* complement carry flag from eflags reg */
5578 ctxt->eflags ^= X86_EFLAGS_CF;
5580 case 0xf8: /* clc */
5581 ctxt->eflags &= ~X86_EFLAGS_CF;
5583 case 0xf9: /* stc */
5584 ctxt->eflags |= X86_EFLAGS_CF;
5586 case 0xfc: /* cld */
5587 ctxt->eflags &= ~X86_EFLAGS_DF;
5589 case 0xfd: /* std */
5590 ctxt->eflags |= X86_EFLAGS_DF;
5593 goto cannot_emulate;
5596 if (rc != X86EMUL_CONTINUE)
5600 if (ctxt->d & SrcWrite) {
5601 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5602 rc = writeback(ctxt, &ctxt->src);
5603 if (rc != X86EMUL_CONTINUE)
5606 if (!(ctxt->d & NoWrite)) {
5607 rc = writeback(ctxt, &ctxt->dst);
5608 if (rc != X86EMUL_CONTINUE)
5613 * restore dst type in case the decoding will be reused
5614 * (happens for string instruction )
5616 ctxt->dst.type = saved_dst_type;
5618 if ((ctxt->d & SrcMask) == SrcSI)
5619 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5621 if ((ctxt->d & DstMask) == DstDI)
5622 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5624 if (ctxt->rep_prefix && (ctxt->d & String)) {
5626 struct read_cache *r = &ctxt->io_read;
5627 if ((ctxt->d & SrcMask) == SrcSI)
5628 count = ctxt->src.count;
5630 count = ctxt->dst.count;
5631 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5633 if (!string_insn_completed(ctxt)) {
5635 * Re-enter guest when pio read ahead buffer is empty
5636 * or, if it is not used, after each 1024 iteration.
5638 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5639 (r->end == 0 || r->end != r->pos)) {
5641 * Reset read cache. Usually happens before
5642 * decode, but since instruction is restarted
5643 * we have to do it here.
5645 ctxt->mem_read.end = 0;
5646 writeback_registers(ctxt);
5647 return EMULATION_RESTART;
5649 goto done; /* skip rip writeback */
5651 ctxt->eflags &= ~X86_EFLAGS_RF;
5654 ctxt->eip = ctxt->_eip;
5657 if (rc == X86EMUL_PROPAGATE_FAULT) {
5658 WARN_ON(ctxt->exception.vector > 0x1f);
5659 ctxt->have_exception = true;
5661 if (rc == X86EMUL_INTERCEPTED)
5662 return EMULATION_INTERCEPTED;
5664 if (rc == X86EMUL_CONTINUE)
5665 writeback_registers(ctxt);
5667 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5671 case 0x09: /* wbinvd */
5672 (ctxt->ops->wbinvd)(ctxt);
5674 case 0x08: /* invd */
5675 case 0x0d: /* GrpP (prefetch) */
5676 case 0x18: /* Grp16 (prefetch/nop) */
5677 case 0x1f: /* nop */
5679 case 0x20: /* mov cr, reg */
5680 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5682 case 0x21: /* mov from dr to reg */
5683 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5685 case 0x40 ... 0x4f: /* cmov */
5686 if (test_cc(ctxt->b, ctxt->eflags))
5687 ctxt->dst.val = ctxt->src.val;
5688 else if (ctxt->op_bytes != 4)
5689 ctxt->dst.type = OP_NONE; /* no writeback */
5691 case 0x80 ... 0x8f: /* jnz rel, etc*/
5692 if (test_cc(ctxt->b, ctxt->eflags))
5693 rc = jmp_rel(ctxt, ctxt->src.val);
5695 case 0x90 ... 0x9f: /* setcc r/m8 */
5696 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5698 case 0xb6 ... 0xb7: /* movzx */
5699 ctxt->dst.bytes = ctxt->op_bytes;
5700 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5701 : (u16) ctxt->src.val;
5703 case 0xbe ... 0xbf: /* movsx */
5704 ctxt->dst.bytes = ctxt->op_bytes;
5705 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5706 (s16) ctxt->src.val;
5709 goto cannot_emulate;
5714 if (rc != X86EMUL_CONTINUE)
5720 return EMULATION_FAILED;
5723 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5725 invalidate_registers(ctxt);
5728 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5730 writeback_registers(ctxt);