1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/debugreg.h>
26 #include <asm/nospec-branch.h>
37 #define OpImplicit 1ull /* No generic decode */
38 #define OpReg 2ull /* Register */
39 #define OpMem 3ull /* Memory */
40 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
41 #define OpDI 5ull /* ES:DI/EDI/RDI */
42 #define OpMem64 6ull /* Memory, 64-bit */
43 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
44 #define OpDX 8ull /* DX register */
45 #define OpCL 9ull /* CL register (for shifts) */
46 #define OpImmByte 10ull /* 8-bit sign extended immediate */
47 #define OpOne 11ull /* Implied 1 */
48 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
49 #define OpMem16 13ull /* Memory operand (16-bit). */
50 #define OpMem32 14ull /* Memory operand (32-bit). */
51 #define OpImmU 15ull /* Immediate operand, zero extended */
52 #define OpSI 16ull /* SI/ESI/RSI */
53 #define OpImmFAddr 17ull /* Immediate far address */
54 #define OpMemFAddr 18ull /* Far address in memory */
55 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
56 #define OpES 20ull /* ES */
57 #define OpCS 21ull /* CS */
58 #define OpSS 22ull /* SS */
59 #define OpDS 23ull /* DS */
60 #define OpFS 24ull /* FS */
61 #define OpGS 25ull /* GS */
62 #define OpMem8 26ull /* 8-bit zero extended memory operand */
63 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
66 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
68 #define OpBits 5 /* Width of operand field */
69 #define OpMask ((1ull << OpBits) - 1)
72 * Opcode effective-address decode tables.
73 * Note that we only emulate instructions that have at least one memory
74 * operand (excluding implicit stack references). We assume that stack
75 * references and instruction fetches will never occur in special memory
76 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
80 /* Operand sizes: 8-bit operands or specified/overridden size. */
81 #define ByteOp (1<<0) /* 8-bit operands. */
82 /* Destination operand type. */
84 #define ImplicitOps (OpImplicit << DstShift)
85 #define DstReg (OpReg << DstShift)
86 #define DstMem (OpMem << DstShift)
87 #define DstAcc (OpAcc << DstShift)
88 #define DstDI (OpDI << DstShift)
89 #define DstMem64 (OpMem64 << DstShift)
90 #define DstMem16 (OpMem16 << DstShift)
91 #define DstImmUByte (OpImmUByte << DstShift)
92 #define DstDX (OpDX << DstShift)
93 #define DstAccLo (OpAccLo << DstShift)
94 #define DstMask (OpMask << DstShift)
95 /* Source operand type. */
97 #define SrcNone (OpNone << SrcShift)
98 #define SrcReg (OpReg << SrcShift)
99 #define SrcMem (OpMem << SrcShift)
100 #define SrcMem16 (OpMem16 << SrcShift)
101 #define SrcMem32 (OpMem32 << SrcShift)
102 #define SrcImm (OpImm << SrcShift)
103 #define SrcImmByte (OpImmByte << SrcShift)
104 #define SrcOne (OpOne << SrcShift)
105 #define SrcImmUByte (OpImmUByte << SrcShift)
106 #define SrcImmU (OpImmU << SrcShift)
107 #define SrcSI (OpSI << SrcShift)
108 #define SrcXLat (OpXLat << SrcShift)
109 #define SrcImmFAddr (OpImmFAddr << SrcShift)
110 #define SrcMemFAddr (OpMemFAddr << SrcShift)
111 #define SrcAcc (OpAcc << SrcShift)
112 #define SrcImmU16 (OpImmU16 << SrcShift)
113 #define SrcImm64 (OpImm64 << SrcShift)
114 #define SrcDX (OpDX << SrcShift)
115 #define SrcMem8 (OpMem8 << SrcShift)
116 #define SrcAccHi (OpAccHi << SrcShift)
117 #define SrcMask (OpMask << SrcShift)
118 #define BitOp (1<<11)
119 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
120 #define String (1<<13) /* String instruction (rep capable) */
121 #define Stack (1<<14) /* Stack instruction (push/pop) */
122 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
123 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
124 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
125 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
126 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
127 #define Escape (5<<15) /* Escape to coprocessor instruction */
128 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
129 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
130 #define Sse (1<<18) /* SSE Vector instruction */
131 /* Generic ModRM decode. */
132 #define ModRM (1<<19)
133 /* Destination is only written; never read. */
136 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
137 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
138 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
139 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
140 #define Undefined (1<<25) /* No Such Instruction */
141 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
142 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
144 #define PageTable (1 << 29) /* instruction used to write page table */
145 #define NotImpl (1 << 30) /* instruction is not implemented */
146 /* Source 2 operand type */
147 #define Src2Shift (31)
148 #define Src2None (OpNone << Src2Shift)
149 #define Src2Mem (OpMem << Src2Shift)
150 #define Src2CL (OpCL << Src2Shift)
151 #define Src2ImmByte (OpImmByte << Src2Shift)
152 #define Src2One (OpOne << Src2Shift)
153 #define Src2Imm (OpImm << Src2Shift)
154 #define Src2ES (OpES << Src2Shift)
155 #define Src2CS (OpCS << Src2Shift)
156 #define Src2SS (OpSS << Src2Shift)
157 #define Src2DS (OpDS << Src2Shift)
158 #define Src2FS (OpFS << Src2Shift)
159 #define Src2GS (OpGS << Src2Shift)
160 #define Src2Mask (OpMask << Src2Shift)
161 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
162 #define AlignMask ((u64)7 << 41)
163 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
164 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
165 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
166 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
167 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
168 #define NoWrite ((u64)1 << 45) /* No writeback */
169 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
170 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
171 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
172 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
173 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
174 #define NearBranch ((u64)1 << 52) /* Near branches */
175 #define No16 ((u64)1 << 53) /* No 16 bit operand */
176 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
177 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
181 #define X2(x...) x, x
182 #define X3(x...) X2(x), x
183 #define X4(x...) X2(x), X2(x)
184 #define X5(x...) X4(x), x
185 #define X6(x...) X4(x), X2(x)
186 #define X7(x...) X4(x), X3(x)
187 #define X8(x...) X4(x), X4(x)
188 #define X16(x...) X8(x), X8(x)
190 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
191 #define FASTOP_SIZE 8
197 int (*execute)(struct x86_emulate_ctxt *ctxt);
198 const struct opcode *group;
199 const struct group_dual *gdual;
200 const struct gprefix *gprefix;
201 const struct escape *esc;
202 const struct instr_dual *idual;
203 const struct mode_dual *mdual;
204 void (*fastop)(struct fastop *fake);
206 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
210 struct opcode mod012[8];
211 struct opcode mod3[8];
215 struct opcode pfx_no;
216 struct opcode pfx_66;
217 struct opcode pfx_f2;
218 struct opcode pfx_f3;
223 struct opcode high[64];
227 struct opcode mod012;
232 struct opcode mode32;
233 struct opcode mode64;
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
238 enum x86_transfer_type {
240 X86_TRANSFER_CALL_JMP,
242 X86_TRANSFER_TASK_SWITCH,
245 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
247 if (!(ctxt->regs_valid & (1 << nr))) {
248 ctxt->regs_valid |= 1 << nr;
249 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
251 return ctxt->_regs[nr];
254 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
256 ctxt->regs_valid |= 1 << nr;
257 ctxt->regs_dirty |= 1 << nr;
258 return &ctxt->_regs[nr];
261 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
264 return reg_write(ctxt, nr);
267 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
271 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
272 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
275 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
277 ctxt->regs_dirty = 0;
278 ctxt->regs_valid = 0;
282 * These EFLAGS bits are restored from saved value during emulation, and
283 * any changes are written back to the saved value after emulation.
285 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
286 X86_EFLAGS_PF|X86_EFLAGS_CF)
295 * fastop functions have a special calling convention:
300 * flags: rflags (in/out)
301 * ex: rsi (in:fastop pointer, out:zero if exception)
303 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
304 * different operand sizes can be reached by calculation, rather than a jump
305 * table (which would be bigger than the code).
307 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
309 #define __FOP_FUNC(name) \
310 ".align " __stringify(FASTOP_SIZE) " \n\t" \
311 ".type " name ", @function \n\t" \
314 #define FOP_FUNC(name) \
317 #define __FOP_RET(name) \
319 ".size " name ", .-" name "\n\t"
321 #define FOP_RET(name) \
324 #define FOP_START(op) \
325 extern void em_##op(struct fastop *fake); \
326 asm(".pushsection .text, \"ax\" \n\t" \
327 ".global em_" #op " \n\t" \
328 ".align " __stringify(FASTOP_SIZE) " \n\t" \
334 #define __FOPNOP(name) \
339 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
341 #define FOP1E(op, dst) \
342 __FOP_FUNC(#op "_" #dst) \
343 "10: " #op " %" #dst " \n\t" \
344 __FOP_RET(#op "_" #dst)
346 #define FOP1EEX(op, dst) \
347 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
349 #define FASTOP1(op) \
354 ON64(FOP1E(op##q, rax)) \
357 /* 1-operand, using src2 (for MUL/DIV r/m) */
358 #define FASTOP1SRC2(op, name) \
363 ON64(FOP1E(op, rcx)) \
366 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
367 #define FASTOP1SRC2EX(op, name) \
372 ON64(FOP1EEX(op, rcx)) \
375 #define FOP2E(op, dst, src) \
376 __FOP_FUNC(#op "_" #dst "_" #src) \
377 #op " %" #src ", %" #dst " \n\t" \
378 __FOP_RET(#op "_" #dst "_" #src)
380 #define FASTOP2(op) \
382 FOP2E(op##b, al, dl) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, word only */
389 #define FASTOP2W(op) \
392 FOP2E(op##w, ax, dx) \
393 FOP2E(op##l, eax, edx) \
394 ON64(FOP2E(op##q, rax, rdx)) \
397 /* 2 operand, src is CL */
398 #define FASTOP2CL(op) \
400 FOP2E(op##b, al, cl) \
401 FOP2E(op##w, ax, cl) \
402 FOP2E(op##l, eax, cl) \
403 ON64(FOP2E(op##q, rax, cl)) \
406 /* 2 operand, src and dest are reversed */
407 #define FASTOP2R(op, name) \
409 FOP2E(op##b, dl, al) \
410 FOP2E(op##w, dx, ax) \
411 FOP2E(op##l, edx, eax) \
412 ON64(FOP2E(op##q, rdx, rax)) \
415 #define FOP3E(op, dst, src, src2) \
416 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
417 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
418 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
420 /* 3-operand, word-only, src2=cl */
421 #define FASTOP3WCL(op) \
424 FOP3E(op##w, ax, dx, cl) \
425 FOP3E(op##l, eax, edx, cl) \
426 ON64(FOP3E(op##q, rax, rdx, cl)) \
429 /* Special case for SETcc - 1 instruction per cc */
432 * Depending on .config the SETcc functions look like:
434 * SETcc %al [3 bytes]
436 * INT3 [1 byte; CONFIG_SLS]
438 * Which gives possible sizes 4 or 5. When rounded up to the
439 * next power-of-two alignment they become 4 or 8.
441 #define SETCC_LENGTH (4 + IS_ENABLED(CONFIG_SLS))
442 #define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS))
443 static_assert(SETCC_LENGTH <= SETCC_ALIGN);
445 #define FOP_SETCC(op) \
446 ".align " __stringify(SETCC_ALIGN) " \n\t" \
447 ".type " #op ", @function \n\t" \
452 asm(".pushsection .fixup, \"ax\"\n"
453 "kvm_fastop_exception: xor %esi, %esi; " ASM_RET
477 "pushf; sbb %al, %al; popf \n\t"
482 * XXX: inoutclob user must know where the argument is being expanded.
483 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
485 #define asm_safe(insn, inoutclob...) \
489 asm volatile("1:" insn "\n" \
491 ".pushsection .fixup, \"ax\"\n" \
492 "3: movl $1, %[_fault]\n" \
495 _ASM_EXTABLE(1b, 3b) \
496 : [_fault] "+qm"(_fault) inoutclob ); \
498 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
501 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
502 enum x86_intercept intercept,
503 enum x86_intercept_stage stage)
505 struct x86_instruction_info info = {
506 .intercept = intercept,
507 .rep_prefix = ctxt->rep_prefix,
508 .modrm_mod = ctxt->modrm_mod,
509 .modrm_reg = ctxt->modrm_reg,
510 .modrm_rm = ctxt->modrm_rm,
511 .src_val = ctxt->src.val64,
512 .dst_val = ctxt->dst.val64,
513 .src_bytes = ctxt->src.bytes,
514 .dst_bytes = ctxt->dst.bytes,
515 .ad_bytes = ctxt->ad_bytes,
516 .next_rip = ctxt->eip,
519 return ctxt->ops->intercept(ctxt, &info, stage);
522 static void assign_masked(ulong *dest, ulong src, ulong mask)
524 *dest = (*dest & ~mask) | (src & mask);
527 static void assign_register(unsigned long *reg, u64 val, int bytes)
529 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
532 *(u8 *)reg = (u8)val;
535 *(u16 *)reg = (u16)val;
539 break; /* 64b: zero-extend */
546 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
548 return (1UL << (ctxt->ad_bytes << 3)) - 1;
551 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
554 struct desc_struct ss;
556 if (ctxt->mode == X86EMUL_MODE_PROT64)
558 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
559 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
562 static int stack_size(struct x86_emulate_ctxt *ctxt)
564 return (__fls(stack_mask(ctxt)) + 1) >> 3;
567 /* Access/update address held in a register, based on addressing mode. */
568 static inline unsigned long
569 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
571 if (ctxt->ad_bytes == sizeof(unsigned long))
574 return reg & ad_mask(ctxt);
577 static inline unsigned long
578 register_address(struct x86_emulate_ctxt *ctxt, int reg)
580 return address_mask(ctxt, reg_read(ctxt, reg));
583 static void masked_increment(ulong *reg, ulong mask, int inc)
585 assign_masked(reg, *reg + inc, mask);
589 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
591 ulong *preg = reg_rmw(ctxt, reg);
593 assign_register(preg, *preg + inc, ctxt->ad_bytes);
596 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
598 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
601 static u32 desc_limit_scaled(struct desc_struct *desc)
603 u32 limit = get_desc_limit(desc);
605 return desc->g ? (limit << 12) | 0xfff : limit;
608 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
610 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
613 return ctxt->ops->get_cached_segment_base(ctxt, seg);
616 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
617 u32 error, bool valid)
620 ctxt->exception.vector = vec;
621 ctxt->exception.error_code = error;
622 ctxt->exception.error_code_valid = valid;
623 return X86EMUL_PROPAGATE_FAULT;
626 static int emulate_db(struct x86_emulate_ctxt *ctxt)
628 return emulate_exception(ctxt, DB_VECTOR, 0, false);
631 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
633 return emulate_exception(ctxt, GP_VECTOR, err, true);
636 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
638 return emulate_exception(ctxt, SS_VECTOR, err, true);
641 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
643 return emulate_exception(ctxt, UD_VECTOR, 0, false);
646 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
648 return emulate_exception(ctxt, TS_VECTOR, err, true);
651 static int emulate_de(struct x86_emulate_ctxt *ctxt)
653 return emulate_exception(ctxt, DE_VECTOR, 0, false);
656 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
658 return emulate_exception(ctxt, NM_VECTOR, 0, false);
661 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
664 struct desc_struct desc;
666 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
670 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
675 struct desc_struct desc;
677 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
678 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
681 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
683 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
686 static inline bool emul_is_noncanonical_address(u64 la,
687 struct x86_emulate_ctxt *ctxt)
689 return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
693 * x86 defines three classes of vector instructions: explicitly
694 * aligned, explicitly unaligned, and the rest, which change behaviour
695 * depending on whether they're AVX encoded or not.
697 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
698 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
699 * 512 bytes of data must be aligned to a 16 byte boundary.
701 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
703 u64 alignment = ctxt->d & AlignMask;
705 if (likely(size < 16))
720 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
721 struct segmented_address addr,
722 unsigned *max_size, unsigned size,
723 bool write, bool fetch,
724 enum x86emul_mode mode, ulong *linear)
726 struct desc_struct desc;
733 la = seg_base(ctxt, addr.seg) + addr.ea;
736 case X86EMUL_MODE_PROT64:
738 va_bits = ctxt_virt_addr_bits(ctxt);
739 if (get_canonical(la, va_bits) != la)
742 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
743 if (size > *max_size)
747 *linear = la = (u32)la;
748 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
752 /* code segment in protected mode or read-only data segment */
753 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
754 || !(desc.type & 2)) && write)
756 /* unreadable code segment */
757 if (!fetch && (desc.type & 8) && !(desc.type & 2))
759 lim = desc_limit_scaled(&desc);
760 if (!(desc.type & 8) && (desc.type & 4)) {
761 /* expand-down segment */
764 lim = desc.d ? 0xffffffff : 0xffff;
768 if (lim == 0xffffffff)
771 *max_size = (u64)lim + 1 - addr.ea;
772 if (size > *max_size)
777 if (la & (insn_alignment(ctxt, size) - 1))
778 return emulate_gp(ctxt, 0);
779 return X86EMUL_CONTINUE;
781 if (addr.seg == VCPU_SREG_SS)
782 return emulate_ss(ctxt, 0);
784 return emulate_gp(ctxt, 0);
787 static int linearize(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 unsigned size, bool write,
793 return __linearize(ctxt, addr, &max_size, size, write, false,
797 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
798 enum x86emul_mode mode)
803 struct segmented_address addr = { .seg = VCPU_SREG_CS,
806 if (ctxt->op_bytes != sizeof(unsigned long))
807 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
808 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
809 if (rc == X86EMUL_CONTINUE)
810 ctxt->_eip = addr.ea;
814 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
816 return assign_eip(ctxt, dst, ctxt->mode);
819 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
820 const struct desc_struct *cs_desc)
822 enum x86emul_mode mode = ctxt->mode;
826 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
830 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
832 mode = X86EMUL_MODE_PROT64;
834 mode = X86EMUL_MODE_PROT32; /* temporary value */
837 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
838 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
839 rc = assign_eip(ctxt, dst, mode);
840 if (rc == X86EMUL_CONTINUE)
845 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
847 return assign_eip_near(ctxt, ctxt->_eip + rel);
850 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
851 void *data, unsigned size)
853 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
856 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
857 ulong linear, void *data,
860 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
863 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
864 struct segmented_address addr,
871 rc = linearize(ctxt, addr, size, false, &linear);
872 if (rc != X86EMUL_CONTINUE)
874 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
877 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
878 struct segmented_address addr,
885 rc = linearize(ctxt, addr, size, true, &linear);
886 if (rc != X86EMUL_CONTINUE)
888 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
892 * Prefetch the remaining bytes of the instruction without crossing page
893 * boundary if they are not in fetch_cache yet.
895 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
898 unsigned size, max_size;
899 unsigned long linear;
900 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
901 struct segmented_address addr = { .seg = VCPU_SREG_CS,
902 .ea = ctxt->eip + cur_size };
905 * We do not know exactly how many bytes will be needed, and
906 * __linearize is expensive, so fetch as much as possible. We
907 * just have to avoid going beyond the 15 byte limit, the end
908 * of the segment, or the end of the page.
910 * __linearize is called with size 0 so that it does not do any
911 * boundary check itself. Instead, we use max_size to check
914 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
916 if (unlikely(rc != X86EMUL_CONTINUE))
919 size = min_t(unsigned, 15UL ^ cur_size, max_size);
920 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
923 * One instruction can only straddle two pages,
924 * and one has been loaded at the beginning of
925 * x86_decode_insn. So, if not enough bytes
926 * still, we must have hit the 15-byte boundary.
928 if (unlikely(size < op_size))
929 return emulate_gp(ctxt, 0);
931 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
932 size, &ctxt->exception);
933 if (unlikely(rc != X86EMUL_CONTINUE))
935 ctxt->fetch.end += size;
936 return X86EMUL_CONTINUE;
939 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
942 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
944 if (unlikely(done_size < size))
945 return __do_insn_fetch_bytes(ctxt, size - done_size);
947 return X86EMUL_CONTINUE;
950 /* Fetch next part of the instruction being emulated. */
951 #define insn_fetch(_type, _ctxt) \
954 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
955 if (rc != X86EMUL_CONTINUE) \
957 ctxt->_eip += sizeof(_type); \
958 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
959 ctxt->fetch.ptr += sizeof(_type); \
963 #define insn_fetch_arr(_arr, _size, _ctxt) \
965 rc = do_insn_fetch_bytes(_ctxt, _size); \
966 if (rc != X86EMUL_CONTINUE) \
968 ctxt->_eip += (_size); \
969 memcpy(_arr, ctxt->fetch.ptr, _size); \
970 ctxt->fetch.ptr += (_size); \
974 * Given the 'reg' portion of a ModRM byte, and a register block, return a
975 * pointer into the block that addresses the relevant register.
976 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
978 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
982 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
984 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
985 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
987 p = reg_rmw(ctxt, modrm_reg);
991 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
992 struct segmented_address addr,
993 u16 *size, unsigned long *address, int op_bytes)
1000 rc = segmented_read_std(ctxt, addr, size, 2);
1001 if (rc != X86EMUL_CONTINUE)
1004 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1018 FASTOP1SRC2(mul, mul_ex);
1019 FASTOP1SRC2(imul, imul_ex);
1020 FASTOP1SRC2EX(div, div_ex);
1021 FASTOP1SRC2EX(idiv, idiv_ex);
1050 FASTOP2R(cmp, cmp_r);
1052 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1054 /* If src is zero, do not writeback, but update flags */
1055 if (ctxt->src.val == 0)
1056 ctxt->dst.type = OP_NONE;
1057 return fastop(ctxt, em_bsf);
1060 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1062 /* If src is zero, do not writeback, but update flags */
1063 if (ctxt->src.val == 0)
1064 ctxt->dst.type = OP_NONE;
1065 return fastop(ctxt, em_bsr);
1068 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1071 void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf);
1073 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1074 asm("push %[flags]; popf; " CALL_NOSPEC
1075 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1079 static void fetch_register_operand(struct operand *op)
1081 switch (op->bytes) {
1083 op->val = *(u8 *)op->addr.reg;
1086 op->val = *(u16 *)op->addr.reg;
1089 op->val = *(u32 *)op->addr.reg;
1092 op->val = *(u64 *)op->addr.reg;
1097 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1099 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1100 return emulate_nm(ctxt);
1103 asm volatile("fninit");
1105 return X86EMUL_CONTINUE;
1108 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1112 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1113 return emulate_nm(ctxt);
1116 asm volatile("fnstcw %0": "+m"(fcw));
1119 ctxt->dst.val = fcw;
1121 return X86EMUL_CONTINUE;
1124 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1128 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1129 return emulate_nm(ctxt);
1132 asm volatile("fnstsw %0": "+m"(fsw));
1135 ctxt->dst.val = fsw;
1137 return X86EMUL_CONTINUE;
1140 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1143 unsigned reg = ctxt->modrm_reg;
1145 if (!(ctxt->d & ModRM))
1146 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1148 if (ctxt->d & Sse) {
1152 kvm_read_sse_reg(reg, &op->vec_val);
1155 if (ctxt->d & Mmx) {
1164 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1165 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1167 fetch_register_operand(op);
1168 op->orig_val = op->val;
1171 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1173 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1174 ctxt->modrm_seg = VCPU_SREG_SS;
1177 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1181 int index_reg, base_reg, scale;
1182 int rc = X86EMUL_CONTINUE;
1185 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1186 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1187 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1189 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1190 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1191 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1192 ctxt->modrm_seg = VCPU_SREG_DS;
1194 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1196 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1197 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1199 if (ctxt->d & Sse) {
1202 op->addr.xmm = ctxt->modrm_rm;
1203 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
1206 if (ctxt->d & Mmx) {
1209 op->addr.mm = ctxt->modrm_rm & 7;
1212 fetch_register_operand(op);
1218 if (ctxt->ad_bytes == 2) {
1219 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1220 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1221 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1222 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1224 /* 16-bit ModR/M decode. */
1225 switch (ctxt->modrm_mod) {
1227 if (ctxt->modrm_rm == 6)
1228 modrm_ea += insn_fetch(u16, ctxt);
1231 modrm_ea += insn_fetch(s8, ctxt);
1234 modrm_ea += insn_fetch(u16, ctxt);
1237 switch (ctxt->modrm_rm) {
1239 modrm_ea += bx + si;
1242 modrm_ea += bx + di;
1245 modrm_ea += bp + si;
1248 modrm_ea += bp + di;
1257 if (ctxt->modrm_mod != 0)
1264 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1265 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1266 ctxt->modrm_seg = VCPU_SREG_SS;
1267 modrm_ea = (u16)modrm_ea;
1269 /* 32/64-bit ModR/M decode. */
1270 if ((ctxt->modrm_rm & 7) == 4) {
1271 sib = insn_fetch(u8, ctxt);
1272 index_reg |= (sib >> 3) & 7;
1273 base_reg |= sib & 7;
1276 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1277 modrm_ea += insn_fetch(s32, ctxt);
1279 modrm_ea += reg_read(ctxt, base_reg);
1280 adjust_modrm_seg(ctxt, base_reg);
1281 /* Increment ESP on POP [ESP] */
1282 if ((ctxt->d & IncSP) &&
1283 base_reg == VCPU_REGS_RSP)
1284 modrm_ea += ctxt->op_bytes;
1287 modrm_ea += reg_read(ctxt, index_reg) << scale;
1288 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1289 modrm_ea += insn_fetch(s32, ctxt);
1290 if (ctxt->mode == X86EMUL_MODE_PROT64)
1291 ctxt->rip_relative = 1;
1293 base_reg = ctxt->modrm_rm;
1294 modrm_ea += reg_read(ctxt, base_reg);
1295 adjust_modrm_seg(ctxt, base_reg);
1297 switch (ctxt->modrm_mod) {
1299 modrm_ea += insn_fetch(s8, ctxt);
1302 modrm_ea += insn_fetch(s32, ctxt);
1306 op->addr.mem.ea = modrm_ea;
1307 if (ctxt->ad_bytes != 8)
1308 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1314 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1317 int rc = X86EMUL_CONTINUE;
1320 switch (ctxt->ad_bytes) {
1322 op->addr.mem.ea = insn_fetch(u16, ctxt);
1325 op->addr.mem.ea = insn_fetch(u32, ctxt);
1328 op->addr.mem.ea = insn_fetch(u64, ctxt);
1335 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1339 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1340 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1342 if (ctxt->src.bytes == 2)
1343 sv = (s16)ctxt->src.val & (s16)mask;
1344 else if (ctxt->src.bytes == 4)
1345 sv = (s32)ctxt->src.val & (s32)mask;
1347 sv = (s64)ctxt->src.val & (s64)mask;
1349 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1350 ctxt->dst.addr.mem.ea + (sv >> 3));
1353 /* only subword offset */
1354 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1357 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1358 unsigned long addr, void *dest, unsigned size)
1361 struct read_cache *mc = &ctxt->mem_read;
1363 if (mc->pos < mc->end)
1366 WARN_ON((mc->end + size) >= sizeof(mc->data));
1368 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1370 if (rc != X86EMUL_CONTINUE)
1376 memcpy(dest, mc->data + mc->pos, size);
1378 return X86EMUL_CONTINUE;
1381 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1382 struct segmented_address addr,
1389 rc = linearize(ctxt, addr, size, false, &linear);
1390 if (rc != X86EMUL_CONTINUE)
1392 return read_emulated(ctxt, linear, data, size);
1395 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1396 struct segmented_address addr,
1403 rc = linearize(ctxt, addr, size, true, &linear);
1404 if (rc != X86EMUL_CONTINUE)
1406 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1410 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1411 struct segmented_address addr,
1412 const void *orig_data, const void *data,
1418 rc = linearize(ctxt, addr, size, true, &linear);
1419 if (rc != X86EMUL_CONTINUE)
1421 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1422 size, &ctxt->exception);
1425 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1426 unsigned int size, unsigned short port,
1429 struct read_cache *rc = &ctxt->io_read;
1431 if (rc->pos == rc->end) { /* refill pio read ahead */
1432 unsigned int in_page, n;
1433 unsigned int count = ctxt->rep_prefix ?
1434 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1435 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1436 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1437 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1438 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1441 rc->pos = rc->end = 0;
1442 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1447 if (ctxt->rep_prefix && (ctxt->d & String) &&
1448 !(ctxt->eflags & X86_EFLAGS_DF)) {
1449 ctxt->dst.data = rc->data + rc->pos;
1450 ctxt->dst.type = OP_MEM_STR;
1451 ctxt->dst.count = (rc->end - rc->pos) / size;
1454 memcpy(dest, rc->data + rc->pos, size);
1460 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1461 u16 index, struct desc_struct *desc)
1466 ctxt->ops->get_idt(ctxt, &dt);
1468 if (dt.size < index * 8 + 7)
1469 return emulate_gp(ctxt, index << 3 | 0x2);
1471 addr = dt.address + index * 8;
1472 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1475 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1476 u16 selector, struct desc_ptr *dt)
1478 const struct x86_emulate_ops *ops = ctxt->ops;
1481 if (selector & 1 << 2) {
1482 struct desc_struct desc;
1485 memset(dt, 0, sizeof(*dt));
1486 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1490 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1491 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1493 ops->get_gdt(ctxt, dt);
1496 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1497 u16 selector, ulong *desc_addr_p)
1500 u16 index = selector >> 3;
1503 get_descriptor_table_ptr(ctxt, selector, &dt);
1505 if (dt.size < index * 8 + 7)
1506 return emulate_gp(ctxt, selector & 0xfffc);
1508 addr = dt.address + index * 8;
1510 #ifdef CONFIG_X86_64
1511 if (addr >> 32 != 0) {
1514 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1515 if (!(efer & EFER_LMA))
1520 *desc_addr_p = addr;
1521 return X86EMUL_CONTINUE;
1524 /* allowed just for 8 bytes segments */
1525 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1526 u16 selector, struct desc_struct *desc,
1531 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1532 if (rc != X86EMUL_CONTINUE)
1535 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1538 /* allowed just for 8 bytes segments */
1539 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1540 u16 selector, struct desc_struct *desc)
1545 rc = get_descriptor_ptr(ctxt, selector, &addr);
1546 if (rc != X86EMUL_CONTINUE)
1549 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1552 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1553 u16 selector, int seg, u8 cpl,
1554 enum x86_transfer_type transfer,
1555 struct desc_struct *desc)
1557 struct desc_struct seg_desc, old_desc;
1559 unsigned err_vec = GP_VECTOR;
1561 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1567 memset(&seg_desc, 0, sizeof(seg_desc));
1569 if (ctxt->mode == X86EMUL_MODE_REAL) {
1570 /* set real mode segment descriptor (keep limit etc. for
1572 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1573 set_desc_base(&seg_desc, selector << 4);
1575 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1576 /* VM86 needs a clean new segment descriptor */
1577 set_desc_base(&seg_desc, selector << 4);
1578 set_desc_limit(&seg_desc, 0xffff);
1588 /* TR should be in GDT only */
1589 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1592 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1593 if (null_selector) {
1594 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1597 if (seg == VCPU_SREG_SS) {
1598 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1602 * ctxt->ops->set_segment expects the CPL to be in
1603 * SS.DPL, so fake an expand-up 32-bit data segment.
1613 /* Skip all following checks */
1617 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1618 if (ret != X86EMUL_CONTINUE)
1621 err_code = selector & 0xfffc;
1622 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1625 /* can't load system descriptor into segment selector */
1626 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1627 if (transfer == X86_TRANSFER_CALL_JMP)
1628 return X86EMUL_UNHANDLEABLE;
1637 * segment is not a writable data segment or segment
1638 * selector's RPL != CPL or segment selector's RPL != CPL
1640 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1644 if (!(seg_desc.type & 8))
1647 if (seg_desc.type & 4) {
1653 if (rpl > cpl || dpl != cpl)
1656 /* in long-mode d/b must be clear if l is set */
1657 if (seg_desc.d && seg_desc.l) {
1660 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1661 if (efer & EFER_LMA)
1665 /* CS(RPL) <- CPL */
1666 selector = (selector & 0xfffc) | cpl;
1669 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1672 err_vec = NP_VECTOR;
1675 old_desc = seg_desc;
1676 seg_desc.type |= 2; /* busy */
1677 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1678 sizeof(seg_desc), &ctxt->exception);
1679 if (ret != X86EMUL_CONTINUE)
1682 case VCPU_SREG_LDTR:
1683 if (seg_desc.s || seg_desc.type != 2)
1686 default: /* DS, ES, FS, or GS */
1688 * segment is not a data or readable code segment or
1689 * ((segment is a data or nonconforming code segment)
1690 * and (both RPL and CPL > DPL))
1692 if ((seg_desc.type & 0xa) == 0x8 ||
1693 (((seg_desc.type & 0xc) != 0xc) &&
1694 (rpl > dpl && cpl > dpl)))
1700 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1705 /* mark segment as accessed */
1706 if (!(seg_desc.type & 1)) {
1708 ret = write_segment_descriptor(ctxt, selector,
1710 if (ret != X86EMUL_CONTINUE)
1713 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1714 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1715 if (ret != X86EMUL_CONTINUE)
1717 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1718 ((u64)base3 << 32), ctxt))
1719 return emulate_gp(ctxt, 0);
1722 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1725 return X86EMUL_CONTINUE;
1727 return emulate_exception(ctxt, err_vec, err_code, true);
1730 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1731 u16 selector, int seg)
1733 u8 cpl = ctxt->ops->cpl(ctxt);
1736 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1737 * they can load it at CPL<3 (Intel's manual says only LSS can,
1740 * However, the Intel manual says that putting IST=1/DPL=3 in
1741 * an interrupt gate will result in SS=3 (the AMD manual instead
1742 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1743 * and only forbid it here.
1745 if (seg == VCPU_SREG_SS && selector == 3 &&
1746 ctxt->mode == X86EMUL_MODE_PROT64)
1747 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1749 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1750 X86_TRANSFER_NONE, NULL);
1753 static void write_register_operand(struct operand *op)
1755 return assign_register(op->addr.reg, op->val, op->bytes);
1758 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1762 write_register_operand(op);
1765 if (ctxt->lock_prefix)
1766 return segmented_cmpxchg(ctxt,
1772 return segmented_write(ctxt,
1778 return segmented_write(ctxt,
1781 op->bytes * op->count);
1784 kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
1787 kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
1795 return X86EMUL_CONTINUE;
1798 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1800 struct segmented_address addr;
1802 rsp_increment(ctxt, -bytes);
1803 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1804 addr.seg = VCPU_SREG_SS;
1806 return segmented_write(ctxt, addr, data, bytes);
1809 static int em_push(struct x86_emulate_ctxt *ctxt)
1811 /* Disable writeback. */
1812 ctxt->dst.type = OP_NONE;
1813 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1816 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1817 void *dest, int len)
1820 struct segmented_address addr;
1822 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1823 addr.seg = VCPU_SREG_SS;
1824 rc = segmented_read(ctxt, addr, dest, len);
1825 if (rc != X86EMUL_CONTINUE)
1828 rsp_increment(ctxt, len);
1832 static int em_pop(struct x86_emulate_ctxt *ctxt)
1834 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1837 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1838 void *dest, int len)
1841 unsigned long val, change_mask;
1842 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1843 int cpl = ctxt->ops->cpl(ctxt);
1845 rc = emulate_pop(ctxt, &val, len);
1846 if (rc != X86EMUL_CONTINUE)
1849 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1850 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1851 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1852 X86_EFLAGS_AC | X86_EFLAGS_ID;
1854 switch(ctxt->mode) {
1855 case X86EMUL_MODE_PROT64:
1856 case X86EMUL_MODE_PROT32:
1857 case X86EMUL_MODE_PROT16:
1859 change_mask |= X86_EFLAGS_IOPL;
1861 change_mask |= X86_EFLAGS_IF;
1863 case X86EMUL_MODE_VM86:
1865 return emulate_gp(ctxt, 0);
1866 change_mask |= X86_EFLAGS_IF;
1868 default: /* real mode */
1869 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1873 *(unsigned long *)dest =
1874 (ctxt->eflags & ~change_mask) | (val & change_mask);
1879 static int em_popf(struct x86_emulate_ctxt *ctxt)
1881 ctxt->dst.type = OP_REG;
1882 ctxt->dst.addr.reg = &ctxt->eflags;
1883 ctxt->dst.bytes = ctxt->op_bytes;
1884 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1887 static int em_enter(struct x86_emulate_ctxt *ctxt)
1890 unsigned frame_size = ctxt->src.val;
1891 unsigned nesting_level = ctxt->src2.val & 31;
1895 return X86EMUL_UNHANDLEABLE;
1897 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1898 rc = push(ctxt, &rbp, stack_size(ctxt));
1899 if (rc != X86EMUL_CONTINUE)
1901 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1903 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1904 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1906 return X86EMUL_CONTINUE;
1909 static int em_leave(struct x86_emulate_ctxt *ctxt)
1911 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1913 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1916 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1918 int seg = ctxt->src2.val;
1920 ctxt->src.val = get_segment_selector(ctxt, seg);
1921 if (ctxt->op_bytes == 4) {
1922 rsp_increment(ctxt, -2);
1926 return em_push(ctxt);
1929 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1931 int seg = ctxt->src2.val;
1932 unsigned long selector;
1935 rc = emulate_pop(ctxt, &selector, 2);
1936 if (rc != X86EMUL_CONTINUE)
1939 if (ctxt->modrm_reg == VCPU_SREG_SS)
1940 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1941 if (ctxt->op_bytes > 2)
1942 rsp_increment(ctxt, ctxt->op_bytes - 2);
1944 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1948 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1950 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1951 int rc = X86EMUL_CONTINUE;
1952 int reg = VCPU_REGS_RAX;
1954 while (reg <= VCPU_REGS_RDI) {
1955 (reg == VCPU_REGS_RSP) ?
1956 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1959 if (rc != X86EMUL_CONTINUE)
1968 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1970 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1971 return em_push(ctxt);
1974 static int em_popa(struct x86_emulate_ctxt *ctxt)
1976 int rc = X86EMUL_CONTINUE;
1977 int reg = VCPU_REGS_RDI;
1980 while (reg >= VCPU_REGS_RAX) {
1981 if (reg == VCPU_REGS_RSP) {
1982 rsp_increment(ctxt, ctxt->op_bytes);
1986 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1987 if (rc != X86EMUL_CONTINUE)
1989 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1995 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1997 const struct x86_emulate_ops *ops = ctxt->ops;
2004 /* TODO: Add limit checks */
2005 ctxt->src.val = ctxt->eflags;
2007 if (rc != X86EMUL_CONTINUE)
2010 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2012 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2014 if (rc != X86EMUL_CONTINUE)
2017 ctxt->src.val = ctxt->_eip;
2019 if (rc != X86EMUL_CONTINUE)
2022 ops->get_idt(ctxt, &dt);
2024 eip_addr = dt.address + (irq << 2);
2025 cs_addr = dt.address + (irq << 2) + 2;
2027 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2028 if (rc != X86EMUL_CONTINUE)
2031 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2032 if (rc != X86EMUL_CONTINUE)
2035 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2036 if (rc != X86EMUL_CONTINUE)
2044 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2048 invalidate_registers(ctxt);
2049 rc = __emulate_int_real(ctxt, irq);
2050 if (rc == X86EMUL_CONTINUE)
2051 writeback_registers(ctxt);
2055 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2057 switch(ctxt->mode) {
2058 case X86EMUL_MODE_REAL:
2059 return __emulate_int_real(ctxt, irq);
2060 case X86EMUL_MODE_VM86:
2061 case X86EMUL_MODE_PROT16:
2062 case X86EMUL_MODE_PROT32:
2063 case X86EMUL_MODE_PROT64:
2065 /* Protected mode interrupts unimplemented yet */
2066 return X86EMUL_UNHANDLEABLE;
2070 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2072 int rc = X86EMUL_CONTINUE;
2073 unsigned long temp_eip = 0;
2074 unsigned long temp_eflags = 0;
2075 unsigned long cs = 0;
2076 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2077 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2078 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2079 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2080 X86_EFLAGS_AC | X86_EFLAGS_ID |
2082 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2085 /* TODO: Add stack limit check */
2087 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2089 if (rc != X86EMUL_CONTINUE)
2092 if (temp_eip & ~0xffff)
2093 return emulate_gp(ctxt, 0);
2095 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2097 if (rc != X86EMUL_CONTINUE)
2100 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2102 if (rc != X86EMUL_CONTINUE)
2105 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2107 if (rc != X86EMUL_CONTINUE)
2110 ctxt->_eip = temp_eip;
2112 if (ctxt->op_bytes == 4)
2113 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2114 else if (ctxt->op_bytes == 2) {
2115 ctxt->eflags &= ~0xffff;
2116 ctxt->eflags |= temp_eflags;
2119 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2120 ctxt->eflags |= X86_EFLAGS_FIXED;
2121 ctxt->ops->set_nmi_mask(ctxt, false);
2126 static int em_iret(struct x86_emulate_ctxt *ctxt)
2128 switch(ctxt->mode) {
2129 case X86EMUL_MODE_REAL:
2130 return emulate_iret_real(ctxt);
2131 case X86EMUL_MODE_VM86:
2132 case X86EMUL_MODE_PROT16:
2133 case X86EMUL_MODE_PROT32:
2134 case X86EMUL_MODE_PROT64:
2136 /* iret from protected mode unimplemented yet */
2137 return X86EMUL_UNHANDLEABLE;
2141 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2145 struct desc_struct new_desc;
2146 u8 cpl = ctxt->ops->cpl(ctxt);
2148 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2150 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2151 X86_TRANSFER_CALL_JMP,
2153 if (rc != X86EMUL_CONTINUE)
2156 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2157 /* Error handling is not implemented. */
2158 if (rc != X86EMUL_CONTINUE)
2159 return X86EMUL_UNHANDLEABLE;
2164 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2166 return assign_eip_near(ctxt, ctxt->src.val);
2169 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2174 old_eip = ctxt->_eip;
2175 rc = assign_eip_near(ctxt, ctxt->src.val);
2176 if (rc != X86EMUL_CONTINUE)
2178 ctxt->src.val = old_eip;
2183 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2185 u64 old = ctxt->dst.orig_val64;
2187 if (ctxt->dst.bytes == 16)
2188 return X86EMUL_UNHANDLEABLE;
2190 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2191 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2192 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2193 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2194 ctxt->eflags &= ~X86_EFLAGS_ZF;
2196 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2197 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2199 ctxt->eflags |= X86_EFLAGS_ZF;
2201 return X86EMUL_CONTINUE;
2204 static int em_ret(struct x86_emulate_ctxt *ctxt)
2209 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2210 if (rc != X86EMUL_CONTINUE)
2213 return assign_eip_near(ctxt, eip);
2216 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2219 unsigned long eip, cs;
2220 int cpl = ctxt->ops->cpl(ctxt);
2221 struct desc_struct new_desc;
2223 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2224 if (rc != X86EMUL_CONTINUE)
2226 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2227 if (rc != X86EMUL_CONTINUE)
2229 /* Outer-privilege level return is not implemented */
2230 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2231 return X86EMUL_UNHANDLEABLE;
2232 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2235 if (rc != X86EMUL_CONTINUE)
2237 rc = assign_eip_far(ctxt, eip, &new_desc);
2238 /* Error handling is not implemented. */
2239 if (rc != X86EMUL_CONTINUE)
2240 return X86EMUL_UNHANDLEABLE;
2245 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2249 rc = em_ret_far(ctxt);
2250 if (rc != X86EMUL_CONTINUE)
2252 rsp_increment(ctxt, ctxt->src.val);
2253 return X86EMUL_CONTINUE;
2256 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2258 /* Save real source value, then compare EAX against destination. */
2259 ctxt->dst.orig_val = ctxt->dst.val;
2260 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2261 ctxt->src.orig_val = ctxt->src.val;
2262 ctxt->src.val = ctxt->dst.orig_val;
2263 fastop(ctxt, em_cmp);
2265 if (ctxt->eflags & X86_EFLAGS_ZF) {
2266 /* Success: write back to memory; no update of EAX */
2267 ctxt->src.type = OP_NONE;
2268 ctxt->dst.val = ctxt->src.orig_val;
2270 /* Failure: write the value we saw to EAX. */
2271 ctxt->src.type = OP_REG;
2272 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2273 ctxt->src.val = ctxt->dst.orig_val;
2274 /* Create write-cycle to dest by writing the same value */
2275 ctxt->dst.val = ctxt->dst.orig_val;
2277 return X86EMUL_CONTINUE;
2280 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2282 int seg = ctxt->src2.val;
2286 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2288 rc = load_segment_descriptor(ctxt, sel, seg);
2289 if (rc != X86EMUL_CONTINUE)
2292 ctxt->dst.val = ctxt->src.val;
2296 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2298 #ifdef CONFIG_X86_64
2299 return ctxt->ops->guest_has_long_mode(ctxt);
2305 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2307 desc->g = (flags >> 23) & 1;
2308 desc->d = (flags >> 22) & 1;
2309 desc->l = (flags >> 21) & 1;
2310 desc->avl = (flags >> 20) & 1;
2311 desc->p = (flags >> 15) & 1;
2312 desc->dpl = (flags >> 13) & 3;
2313 desc->s = (flags >> 12) & 1;
2314 desc->type = (flags >> 8) & 15;
2317 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2320 struct desc_struct desc;
2324 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2327 offset = 0x7f84 + n * 12;
2329 offset = 0x7f2c + (n - 3) * 12;
2331 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2332 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2333 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2334 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2335 return X86EMUL_CONTINUE;
2338 #ifdef CONFIG_X86_64
2339 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2342 struct desc_struct desc;
2347 offset = 0x7e00 + n * 16;
2349 selector = GET_SMSTATE(u16, smstate, offset);
2350 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2351 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2352 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2353 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2355 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2356 return X86EMUL_CONTINUE;
2360 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2361 u64 cr0, u64 cr3, u64 cr4)
2366 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2368 if (cr4 & X86_CR4_PCIDE) {
2373 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2375 return X86EMUL_UNHANDLEABLE;
2378 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2379 * Then enable protected mode. However, PCID cannot be enabled
2380 * if EFER.LMA=0, so set it separately.
2382 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2384 return X86EMUL_UNHANDLEABLE;
2386 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2388 return X86EMUL_UNHANDLEABLE;
2390 if (cr4 & X86_CR4_PCIDE) {
2391 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2393 return X86EMUL_UNHANDLEABLE;
2395 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2397 return X86EMUL_UNHANDLEABLE;
2402 return X86EMUL_CONTINUE;
2405 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2406 const char *smstate)
2408 struct desc_struct desc;
2411 u32 val, cr0, cr3, cr4;
2414 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2415 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2416 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2417 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2419 for (i = 0; i < 8; i++)
2420 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2422 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2424 if (ctxt->ops->set_dr(ctxt, 6, val))
2425 return X86EMUL_UNHANDLEABLE;
2427 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2429 if (ctxt->ops->set_dr(ctxt, 7, val))
2430 return X86EMUL_UNHANDLEABLE;
2432 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2433 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2434 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2435 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2436 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2438 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2439 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2440 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2441 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2442 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2444 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2445 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2446 ctxt->ops->set_gdt(ctxt, &dt);
2448 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2449 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2450 ctxt->ops->set_idt(ctxt, &dt);
2452 for (i = 0; i < 6; i++) {
2453 int r = rsm_load_seg_32(ctxt, smstate, i);
2454 if (r != X86EMUL_CONTINUE)
2458 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2460 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2462 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2465 #ifdef CONFIG_X86_64
2466 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2467 const char *smstate)
2469 struct desc_struct desc;
2471 u64 val, cr0, cr3, cr4;
2476 for (i = 0; i < 16; i++)
2477 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2479 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2480 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2482 val = GET_SMSTATE(u64, smstate, 0x7f68);
2484 if (ctxt->ops->set_dr(ctxt, 6, val))
2485 return X86EMUL_UNHANDLEABLE;
2487 val = GET_SMSTATE(u64, smstate, 0x7f60);
2489 if (ctxt->ops->set_dr(ctxt, 7, val))
2490 return X86EMUL_UNHANDLEABLE;
2492 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2493 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2494 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2495 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2496 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2498 if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
2499 return X86EMUL_UNHANDLEABLE;
2501 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2502 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2503 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2504 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2505 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2506 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2508 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2509 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2510 ctxt->ops->set_idt(ctxt, &dt);
2512 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2513 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2514 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2515 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2516 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2517 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2519 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2520 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2521 ctxt->ops->set_gdt(ctxt, &dt);
2523 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2524 if (r != X86EMUL_CONTINUE)
2527 for (i = 0; i < 6; i++) {
2528 r = rsm_load_seg_64(ctxt, smstate, i);
2529 if (r != X86EMUL_CONTINUE)
2533 return X86EMUL_CONTINUE;
2537 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2539 unsigned long cr0, cr4, efer;
2544 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2545 return emulate_ud(ctxt);
2547 smbase = ctxt->ops->get_smbase(ctxt);
2549 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2550 if (ret != X86EMUL_CONTINUE)
2551 return X86EMUL_UNHANDLEABLE;
2553 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2554 ctxt->ops->set_nmi_mask(ctxt, false);
2556 ctxt->ops->exiting_smm(ctxt);
2559 * Get back to real mode, to prepare a safe state in which to load
2560 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2561 * supports long mode.
2563 if (emulator_has_longmode(ctxt)) {
2564 struct desc_struct cs_desc;
2566 /* Zero CR4.PCIDE before CR0.PG. */
2567 cr4 = ctxt->ops->get_cr(ctxt, 4);
2568 if (cr4 & X86_CR4_PCIDE)
2569 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2571 /* A 32-bit code segment is required to clear EFER.LMA. */
2572 memset(&cs_desc, 0, sizeof(cs_desc));
2574 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2575 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2578 /* For the 64-bit case, this will clear EFER.LMA. */
2579 cr0 = ctxt->ops->get_cr(ctxt, 0);
2580 if (cr0 & X86_CR0_PE)
2581 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2583 if (emulator_has_longmode(ctxt)) {
2584 /* Clear CR4.PAE before clearing EFER.LME. */
2585 cr4 = ctxt->ops->get_cr(ctxt, 4);
2586 if (cr4 & X86_CR4_PAE)
2587 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2589 /* And finally go back to 32-bit mode. */
2591 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2595 * Give leave_smm() a chance to make ISA-specific changes to the vCPU
2596 * state (e.g. enter guest mode) before loading state from the SMM
2599 if (ctxt->ops->leave_smm(ctxt, buf))
2600 goto emulate_shutdown;
2602 #ifdef CONFIG_X86_64
2603 if (emulator_has_longmode(ctxt))
2604 ret = rsm_load_state_64(ctxt, buf);
2607 ret = rsm_load_state_32(ctxt, buf);
2609 if (ret != X86EMUL_CONTINUE)
2610 goto emulate_shutdown;
2613 * Note, the ctxt->ops callbacks are responsible for handling side
2614 * effects when writing MSRs and CRs, e.g. MMU context resets, CPUID
2615 * runtime updates, etc... If that changes, e.g. this flow is moved
2616 * out of the emulator to make it look more like enter_smm(), then
2617 * those side effects need to be explicitly handled for both success
2620 return X86EMUL_CONTINUE;
2623 ctxt->ops->triple_fault(ctxt);
2624 return X86EMUL_CONTINUE;
2628 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2629 struct desc_struct *cs, struct desc_struct *ss)
2631 cs->l = 0; /* will be adjusted later */
2632 set_desc_base(cs, 0); /* flat segment */
2633 cs->g = 1; /* 4kb granularity */
2634 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2635 cs->type = 0x0b; /* Read, Execute, Accessed */
2637 cs->dpl = 0; /* will be adjusted later */
2642 set_desc_base(ss, 0); /* flat segment */
2643 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2644 ss->g = 1; /* 4kb granularity */
2646 ss->type = 0x03; /* Read/Write, Accessed */
2647 ss->d = 1; /* 32bit stack segment */
2654 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2656 u32 eax, ebx, ecx, edx;
2659 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2660 return is_guest_vendor_intel(ebx, ecx, edx);
2663 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2665 const struct x86_emulate_ops *ops = ctxt->ops;
2666 u32 eax, ebx, ecx, edx;
2669 * syscall should always be enabled in longmode - so only become
2670 * vendor specific (cpuid) if other modes are active...
2672 if (ctxt->mode == X86EMUL_MODE_PROT64)
2677 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2679 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2680 * 64bit guest with a 32bit compat-app running will #UD !! While this
2681 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2682 * AMD can't behave like Intel.
2684 if (is_guest_vendor_intel(ebx, ecx, edx))
2687 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2688 is_guest_vendor_hygon(ebx, ecx, edx))
2692 * default: (not Intel, not AMD, not Hygon), apply Intel's
2698 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2700 const struct x86_emulate_ops *ops = ctxt->ops;
2701 struct desc_struct cs, ss;
2706 /* syscall is not available in real mode */
2707 if (ctxt->mode == X86EMUL_MODE_REAL ||
2708 ctxt->mode == X86EMUL_MODE_VM86)
2709 return emulate_ud(ctxt);
2711 if (!(em_syscall_is_enabled(ctxt)))
2712 return emulate_ud(ctxt);
2714 ops->get_msr(ctxt, MSR_EFER, &efer);
2715 if (!(efer & EFER_SCE))
2716 return emulate_ud(ctxt);
2718 setup_syscalls_segments(ctxt, &cs, &ss);
2719 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2721 cs_sel = (u16)(msr_data & 0xfffc);
2722 ss_sel = (u16)(msr_data + 8);
2724 if (efer & EFER_LMA) {
2728 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2729 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2731 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2732 if (efer & EFER_LMA) {
2733 #ifdef CONFIG_X86_64
2734 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2737 ctxt->mode == X86EMUL_MODE_PROT64 ?
2738 MSR_LSTAR : MSR_CSTAR, &msr_data);
2739 ctxt->_eip = msr_data;
2741 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2742 ctxt->eflags &= ~msr_data;
2743 ctxt->eflags |= X86_EFLAGS_FIXED;
2747 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2748 ctxt->_eip = (u32)msr_data;
2750 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2753 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2754 return X86EMUL_CONTINUE;
2757 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2759 const struct x86_emulate_ops *ops = ctxt->ops;
2760 struct desc_struct cs, ss;
2765 ops->get_msr(ctxt, MSR_EFER, &efer);
2766 /* inject #GP if in real mode */
2767 if (ctxt->mode == X86EMUL_MODE_REAL)
2768 return emulate_gp(ctxt, 0);
2771 * Not recognized on AMD in compat mode (but is recognized in legacy
2774 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2775 && !vendor_intel(ctxt))
2776 return emulate_ud(ctxt);
2778 /* sysenter/sysexit have not been tested in 64bit mode. */
2779 if (ctxt->mode == X86EMUL_MODE_PROT64)
2780 return X86EMUL_UNHANDLEABLE;
2782 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2783 if ((msr_data & 0xfffc) == 0x0)
2784 return emulate_gp(ctxt, 0);
2786 setup_syscalls_segments(ctxt, &cs, &ss);
2787 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2788 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2789 ss_sel = cs_sel + 8;
2790 if (efer & EFER_LMA) {
2795 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2796 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2798 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2799 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2801 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2802 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2804 if (efer & EFER_LMA)
2805 ctxt->mode = X86EMUL_MODE_PROT64;
2807 return X86EMUL_CONTINUE;
2810 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2812 const struct x86_emulate_ops *ops = ctxt->ops;
2813 struct desc_struct cs, ss;
2814 u64 msr_data, rcx, rdx;
2816 u16 cs_sel = 0, ss_sel = 0;
2818 /* inject #GP if in real mode or Virtual 8086 mode */
2819 if (ctxt->mode == X86EMUL_MODE_REAL ||
2820 ctxt->mode == X86EMUL_MODE_VM86)
2821 return emulate_gp(ctxt, 0);
2823 setup_syscalls_segments(ctxt, &cs, &ss);
2825 if ((ctxt->rex_prefix & 0x8) != 0x0)
2826 usermode = X86EMUL_MODE_PROT64;
2828 usermode = X86EMUL_MODE_PROT32;
2830 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2831 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2835 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2837 case X86EMUL_MODE_PROT32:
2838 cs_sel = (u16)(msr_data + 16);
2839 if ((msr_data & 0xfffc) == 0x0)
2840 return emulate_gp(ctxt, 0);
2841 ss_sel = (u16)(msr_data + 24);
2845 case X86EMUL_MODE_PROT64:
2846 cs_sel = (u16)(msr_data + 32);
2847 if (msr_data == 0x0)
2848 return emulate_gp(ctxt, 0);
2849 ss_sel = cs_sel + 8;
2852 if (emul_is_noncanonical_address(rcx, ctxt) ||
2853 emul_is_noncanonical_address(rdx, ctxt))
2854 return emulate_gp(ctxt, 0);
2857 cs_sel |= SEGMENT_RPL_MASK;
2858 ss_sel |= SEGMENT_RPL_MASK;
2860 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2861 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2864 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2866 return X86EMUL_CONTINUE;
2869 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2872 if (ctxt->mode == X86EMUL_MODE_REAL)
2874 if (ctxt->mode == X86EMUL_MODE_VM86)
2876 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2877 return ctxt->ops->cpl(ctxt) > iopl;
2880 #define VMWARE_PORT_VMPORT (0x5658)
2881 #define VMWARE_PORT_VMRPC (0x5659)
2883 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2886 const struct x86_emulate_ops *ops = ctxt->ops;
2887 struct desc_struct tr_seg;
2890 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2891 unsigned mask = (1 << len) - 1;
2895 * VMware allows access to these ports even if denied
2896 * by TSS I/O permission bitmap. Mimic behavior.
2898 if (enable_vmware_backdoor &&
2899 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2902 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2905 if (desc_limit_scaled(&tr_seg) < 103)
2907 base = get_desc_base(&tr_seg);
2908 #ifdef CONFIG_X86_64
2909 base |= ((u64)base3) << 32;
2911 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2912 if (r != X86EMUL_CONTINUE)
2914 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2916 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2917 if (r != X86EMUL_CONTINUE)
2919 if ((perm >> bit_idx) & mask)
2924 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2930 if (emulator_bad_iopl(ctxt))
2931 if (!emulator_io_port_access_allowed(ctxt, port, len))
2934 ctxt->perm_ok = true;
2939 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2942 * Intel CPUs mask the counter and pointers in quite strange
2943 * manner when ECX is zero due to REP-string optimizations.
2945 #ifdef CONFIG_X86_64
2946 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2949 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2952 case 0xa4: /* movsb */
2953 case 0xa5: /* movsd/w */
2954 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2956 case 0xaa: /* stosb */
2957 case 0xab: /* stosd/w */
2958 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2963 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2964 struct tss_segment_16 *tss)
2966 tss->ip = ctxt->_eip;
2967 tss->flag = ctxt->eflags;
2968 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2969 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2970 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2971 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2972 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2973 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2974 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2975 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2977 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2978 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2979 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2980 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2981 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2984 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2985 struct tss_segment_16 *tss)
2990 ctxt->_eip = tss->ip;
2991 ctxt->eflags = tss->flag | 2;
2992 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2993 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2994 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2995 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2996 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2997 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2998 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2999 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3002 * SDM says that segment selectors are loaded before segment
3005 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3006 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3007 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3008 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3009 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3014 * Now load segment descriptors. If fault happens at this stage
3015 * it is handled in a context of new task
3017 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3018 X86_TRANSFER_TASK_SWITCH, NULL);
3019 if (ret != X86EMUL_CONTINUE)
3021 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3022 X86_TRANSFER_TASK_SWITCH, NULL);
3023 if (ret != X86EMUL_CONTINUE)
3025 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3026 X86_TRANSFER_TASK_SWITCH, NULL);
3027 if (ret != X86EMUL_CONTINUE)
3029 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3030 X86_TRANSFER_TASK_SWITCH, NULL);
3031 if (ret != X86EMUL_CONTINUE)
3033 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3034 X86_TRANSFER_TASK_SWITCH, NULL);
3035 if (ret != X86EMUL_CONTINUE)
3038 return X86EMUL_CONTINUE;
3041 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3042 u16 tss_selector, u16 old_tss_sel,
3043 ulong old_tss_base, struct desc_struct *new_desc)
3045 struct tss_segment_16 tss_seg;
3047 u32 new_tss_base = get_desc_base(new_desc);
3049 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3050 if (ret != X86EMUL_CONTINUE)
3053 save_state_to_tss16(ctxt, &tss_seg);
3055 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3056 if (ret != X86EMUL_CONTINUE)
3059 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3060 if (ret != X86EMUL_CONTINUE)
3063 if (old_tss_sel != 0xffff) {
3064 tss_seg.prev_task_link = old_tss_sel;
3066 ret = linear_write_system(ctxt, new_tss_base,
3067 &tss_seg.prev_task_link,
3068 sizeof(tss_seg.prev_task_link));
3069 if (ret != X86EMUL_CONTINUE)
3073 return load_state_from_tss16(ctxt, &tss_seg);
3076 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3077 struct tss_segment_32 *tss)
3079 /* CR3 and ldt selector are not saved intentionally */
3080 tss->eip = ctxt->_eip;
3081 tss->eflags = ctxt->eflags;
3082 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3083 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3084 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3085 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3086 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3087 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3088 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3089 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3091 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3092 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3093 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3094 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3095 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3096 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3099 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3100 struct tss_segment_32 *tss)
3105 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3106 return emulate_gp(ctxt, 0);
3107 ctxt->_eip = tss->eip;
3108 ctxt->eflags = tss->eflags | 2;
3110 /* General purpose registers */
3111 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3112 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3113 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3114 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3115 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3116 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3117 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3118 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3121 * SDM says that segment selectors are loaded before segment
3122 * descriptors. This is important because CPL checks will
3125 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3126 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3127 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3128 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3129 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3130 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3131 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3134 * If we're switching between Protected Mode and VM86, we need to make
3135 * sure to update the mode before loading the segment descriptors so
3136 * that the selectors are interpreted correctly.
3138 if (ctxt->eflags & X86_EFLAGS_VM) {
3139 ctxt->mode = X86EMUL_MODE_VM86;
3142 ctxt->mode = X86EMUL_MODE_PROT32;
3147 * Now load segment descriptors. If fault happens at this stage
3148 * it is handled in a context of new task
3150 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3151 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3152 if (ret != X86EMUL_CONTINUE)
3154 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3155 X86_TRANSFER_TASK_SWITCH, NULL);
3156 if (ret != X86EMUL_CONTINUE)
3158 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3159 X86_TRANSFER_TASK_SWITCH, NULL);
3160 if (ret != X86EMUL_CONTINUE)
3162 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3163 X86_TRANSFER_TASK_SWITCH, NULL);
3164 if (ret != X86EMUL_CONTINUE)
3166 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3167 X86_TRANSFER_TASK_SWITCH, NULL);
3168 if (ret != X86EMUL_CONTINUE)
3170 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3171 X86_TRANSFER_TASK_SWITCH, NULL);
3172 if (ret != X86EMUL_CONTINUE)
3174 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3175 X86_TRANSFER_TASK_SWITCH, NULL);
3180 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3181 u16 tss_selector, u16 old_tss_sel,
3182 ulong old_tss_base, struct desc_struct *new_desc)
3184 struct tss_segment_32 tss_seg;
3186 u32 new_tss_base = get_desc_base(new_desc);
3187 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3188 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3190 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3191 if (ret != X86EMUL_CONTINUE)
3194 save_state_to_tss32(ctxt, &tss_seg);
3196 /* Only GP registers and segment selectors are saved */
3197 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3198 ldt_sel_offset - eip_offset);
3199 if (ret != X86EMUL_CONTINUE)
3202 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3203 if (ret != X86EMUL_CONTINUE)
3206 if (old_tss_sel != 0xffff) {
3207 tss_seg.prev_task_link = old_tss_sel;
3209 ret = linear_write_system(ctxt, new_tss_base,
3210 &tss_seg.prev_task_link,
3211 sizeof(tss_seg.prev_task_link));
3212 if (ret != X86EMUL_CONTINUE)
3216 return load_state_from_tss32(ctxt, &tss_seg);
3219 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3220 u16 tss_selector, int idt_index, int reason,
3221 bool has_error_code, u32 error_code)
3223 const struct x86_emulate_ops *ops = ctxt->ops;
3224 struct desc_struct curr_tss_desc, next_tss_desc;
3226 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3227 ulong old_tss_base =
3228 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3230 ulong desc_addr, dr7;
3232 /* FIXME: old_tss_base == ~0 ? */
3234 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3235 if (ret != X86EMUL_CONTINUE)
3237 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3238 if (ret != X86EMUL_CONTINUE)
3241 /* FIXME: check that next_tss_desc is tss */
3244 * Check privileges. The three cases are task switch caused by...
3246 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3247 * 2. Exception/IRQ/iret: No check is performed
3248 * 3. jmp/call to TSS/task-gate: No check is performed since the
3249 * hardware checks it before exiting.
3251 if (reason == TASK_SWITCH_GATE) {
3252 if (idt_index != -1) {
3253 /* Software interrupts */
3254 struct desc_struct task_gate_desc;
3257 ret = read_interrupt_descriptor(ctxt, idt_index,
3259 if (ret != X86EMUL_CONTINUE)
3262 dpl = task_gate_desc.dpl;
3263 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3264 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3268 desc_limit = desc_limit_scaled(&next_tss_desc);
3269 if (!next_tss_desc.p ||
3270 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3271 desc_limit < 0x2b)) {
3272 return emulate_ts(ctxt, tss_selector & 0xfffc);
3275 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3276 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3277 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3280 if (reason == TASK_SWITCH_IRET)
3281 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3283 /* set back link to prev task only if NT bit is set in eflags
3284 note that old_tss_sel is not used after this point */
3285 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3286 old_tss_sel = 0xffff;
3288 if (next_tss_desc.type & 8)
3289 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3290 old_tss_base, &next_tss_desc);
3292 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3293 old_tss_base, &next_tss_desc);
3294 if (ret != X86EMUL_CONTINUE)
3297 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3298 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3300 if (reason != TASK_SWITCH_IRET) {
3301 next_tss_desc.type |= (1 << 1); /* set busy flag */
3302 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3305 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3306 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3308 if (has_error_code) {
3309 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3310 ctxt->lock_prefix = 0;
3311 ctxt->src.val = (unsigned long) error_code;
3312 ret = em_push(ctxt);
3315 ops->get_dr(ctxt, 7, &dr7);
3316 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3321 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3322 u16 tss_selector, int idt_index, int reason,
3323 bool has_error_code, u32 error_code)
3327 invalidate_registers(ctxt);
3328 ctxt->_eip = ctxt->eip;
3329 ctxt->dst.type = OP_NONE;
3331 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3332 has_error_code, error_code);
3334 if (rc == X86EMUL_CONTINUE) {
3335 ctxt->eip = ctxt->_eip;
3336 writeback_registers(ctxt);
3339 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3342 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3345 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3347 register_address_increment(ctxt, reg, df * op->bytes);
3348 op->addr.mem.ea = register_address(ctxt, reg);
3351 static int em_das(struct x86_emulate_ctxt *ctxt)
3354 bool af, cf, old_cf;
3356 cf = ctxt->eflags & X86_EFLAGS_CF;
3362 af = ctxt->eflags & X86_EFLAGS_AF;
3363 if ((al & 0x0f) > 9 || af) {
3365 cf = old_cf | (al >= 250);
3370 if (old_al > 0x99 || old_cf) {
3376 /* Set PF, ZF, SF */
3377 ctxt->src.type = OP_IMM;
3379 ctxt->src.bytes = 1;
3380 fastop(ctxt, em_or);
3381 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3383 ctxt->eflags |= X86_EFLAGS_CF;
3385 ctxt->eflags |= X86_EFLAGS_AF;
3386 return X86EMUL_CONTINUE;
3389 static int em_aam(struct x86_emulate_ctxt *ctxt)
3393 if (ctxt->src.val == 0)
3394 return emulate_de(ctxt);
3396 al = ctxt->dst.val & 0xff;
3397 ah = al / ctxt->src.val;
3398 al %= ctxt->src.val;
3400 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3402 /* Set PF, ZF, SF */
3403 ctxt->src.type = OP_IMM;
3405 ctxt->src.bytes = 1;
3406 fastop(ctxt, em_or);
3408 return X86EMUL_CONTINUE;
3411 static int em_aad(struct x86_emulate_ctxt *ctxt)
3413 u8 al = ctxt->dst.val & 0xff;
3414 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3416 al = (al + (ah * ctxt->src.val)) & 0xff;
3418 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3420 /* Set PF, ZF, SF */
3421 ctxt->src.type = OP_IMM;
3423 ctxt->src.bytes = 1;
3424 fastop(ctxt, em_or);
3426 return X86EMUL_CONTINUE;
3429 static int em_call(struct x86_emulate_ctxt *ctxt)
3432 long rel = ctxt->src.val;
3434 ctxt->src.val = (unsigned long)ctxt->_eip;
3435 rc = jmp_rel(ctxt, rel);
3436 if (rc != X86EMUL_CONTINUE)
3438 return em_push(ctxt);
3441 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3446 struct desc_struct old_desc, new_desc;
3447 const struct x86_emulate_ops *ops = ctxt->ops;
3448 int cpl = ctxt->ops->cpl(ctxt);
3449 enum x86emul_mode prev_mode = ctxt->mode;
3451 old_eip = ctxt->_eip;
3452 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3454 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3455 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3456 X86_TRANSFER_CALL_JMP, &new_desc);
3457 if (rc != X86EMUL_CONTINUE)
3460 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3461 if (rc != X86EMUL_CONTINUE)
3464 ctxt->src.val = old_cs;
3466 if (rc != X86EMUL_CONTINUE)
3469 ctxt->src.val = old_eip;
3471 /* If we failed, we tainted the memory, but the very least we should
3473 if (rc != X86EMUL_CONTINUE) {
3474 pr_warn_once("faulting far call emulation tainted memory\n");
3479 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3480 ctxt->mode = prev_mode;
3485 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3490 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3491 if (rc != X86EMUL_CONTINUE)
3493 rc = assign_eip_near(ctxt, eip);
3494 if (rc != X86EMUL_CONTINUE)
3496 rsp_increment(ctxt, ctxt->src.val);
3497 return X86EMUL_CONTINUE;
3500 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3502 /* Write back the register source. */
3503 ctxt->src.val = ctxt->dst.val;
3504 write_register_operand(&ctxt->src);
3506 /* Write back the memory destination with implicit LOCK prefix. */
3507 ctxt->dst.val = ctxt->src.orig_val;
3508 ctxt->lock_prefix = 1;
3509 return X86EMUL_CONTINUE;
3512 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3514 ctxt->dst.val = ctxt->src2.val;
3515 return fastop(ctxt, em_imul);
3518 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3520 ctxt->dst.type = OP_REG;
3521 ctxt->dst.bytes = ctxt->src.bytes;
3522 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3523 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3525 return X86EMUL_CONTINUE;
3528 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3532 if (!ctxt->ops->guest_has_rdpid(ctxt))
3533 return emulate_ud(ctxt);
3535 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
3536 ctxt->dst.val = tsc_aux;
3537 return X86EMUL_CONTINUE;
3540 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3544 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3545 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3546 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3547 return X86EMUL_CONTINUE;
3550 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3554 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3555 return emulate_gp(ctxt, 0);
3556 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3557 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3558 return X86EMUL_CONTINUE;
3561 static int em_mov(struct x86_emulate_ctxt *ctxt)
3563 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3564 return X86EMUL_CONTINUE;
3567 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3571 if (!ctxt->ops->guest_has_movbe(ctxt))
3572 return emulate_ud(ctxt);
3574 switch (ctxt->op_bytes) {
3577 * From MOVBE definition: "...When the operand size is 16 bits,
3578 * the upper word of the destination register remains unchanged
3581 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3582 * rules so we have to do the operation almost per hand.
3584 tmp = (u16)ctxt->src.val;
3585 ctxt->dst.val &= ~0xffffUL;
3586 ctxt->dst.val |= (unsigned long)swab16(tmp);
3589 ctxt->dst.val = swab32((u32)ctxt->src.val);
3592 ctxt->dst.val = swab64(ctxt->src.val);
3597 return X86EMUL_CONTINUE;
3600 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3602 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3603 return emulate_gp(ctxt, 0);
3605 /* Disable writeback. */
3606 ctxt->dst.type = OP_NONE;
3607 return X86EMUL_CONTINUE;
3610 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3614 if (ctxt->mode == X86EMUL_MODE_PROT64)
3615 val = ctxt->src.val & ~0ULL;
3617 val = ctxt->src.val & ~0U;
3619 /* #UD condition is already handled. */
3620 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3621 return emulate_gp(ctxt, 0);
3623 /* Disable writeback. */
3624 ctxt->dst.type = OP_NONE;
3625 return X86EMUL_CONTINUE;
3628 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3630 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3634 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3635 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3636 r = ctxt->ops->set_msr(ctxt, msr_index, msr_data);
3638 if (r == X86EMUL_IO_NEEDED)
3642 return emulate_gp(ctxt, 0);
3644 return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3647 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3649 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3653 r = ctxt->ops->get_msr(ctxt, msr_index, &msr_data);
3655 if (r == X86EMUL_IO_NEEDED)
3659 return emulate_gp(ctxt, 0);
3661 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3662 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3663 return X86EMUL_CONTINUE;
3666 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3668 if (segment > VCPU_SREG_GS &&
3669 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3670 ctxt->ops->cpl(ctxt) > 0)
3671 return emulate_gp(ctxt, 0);
3673 ctxt->dst.val = get_segment_selector(ctxt, segment);
3674 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3675 ctxt->dst.bytes = 2;
3676 return X86EMUL_CONTINUE;
3679 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3681 if (ctxt->modrm_reg > VCPU_SREG_GS)
3682 return emulate_ud(ctxt);
3684 return em_store_sreg(ctxt, ctxt->modrm_reg);
3687 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3689 u16 sel = ctxt->src.val;
3691 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3692 return emulate_ud(ctxt);
3694 if (ctxt->modrm_reg == VCPU_SREG_SS)
3695 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3697 /* Disable writeback. */
3698 ctxt->dst.type = OP_NONE;
3699 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3702 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3704 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3707 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3709 u16 sel = ctxt->src.val;
3711 /* Disable writeback. */
3712 ctxt->dst.type = OP_NONE;
3713 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3716 static int em_str(struct x86_emulate_ctxt *ctxt)
3718 return em_store_sreg(ctxt, VCPU_SREG_TR);
3721 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3723 u16 sel = ctxt->src.val;
3725 /* Disable writeback. */
3726 ctxt->dst.type = OP_NONE;
3727 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3730 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3735 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3736 if (rc == X86EMUL_CONTINUE)
3737 ctxt->ops->invlpg(ctxt, linear);
3738 /* Disable writeback. */
3739 ctxt->dst.type = OP_NONE;
3740 return X86EMUL_CONTINUE;
3743 static int em_clts(struct x86_emulate_ctxt *ctxt)
3747 cr0 = ctxt->ops->get_cr(ctxt, 0);
3749 ctxt->ops->set_cr(ctxt, 0, cr0);
3750 return X86EMUL_CONTINUE;
3753 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3755 int rc = ctxt->ops->fix_hypercall(ctxt);
3757 if (rc != X86EMUL_CONTINUE)
3760 /* Let the processor re-execute the fixed hypercall */
3761 ctxt->_eip = ctxt->eip;
3762 /* Disable writeback. */
3763 ctxt->dst.type = OP_NONE;
3764 return X86EMUL_CONTINUE;
3767 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3768 void (*get)(struct x86_emulate_ctxt *ctxt,
3769 struct desc_ptr *ptr))
3771 struct desc_ptr desc_ptr;
3773 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3774 ctxt->ops->cpl(ctxt) > 0)
3775 return emulate_gp(ctxt, 0);
3777 if (ctxt->mode == X86EMUL_MODE_PROT64)
3779 get(ctxt, &desc_ptr);
3780 if (ctxt->op_bytes == 2) {
3782 desc_ptr.address &= 0x00ffffff;
3784 /* Disable writeback. */
3785 ctxt->dst.type = OP_NONE;
3786 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3787 &desc_ptr, 2 + ctxt->op_bytes);
3790 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3792 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3795 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3797 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3800 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3802 struct desc_ptr desc_ptr;
3805 if (ctxt->mode == X86EMUL_MODE_PROT64)
3807 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3808 &desc_ptr.size, &desc_ptr.address,
3810 if (rc != X86EMUL_CONTINUE)
3812 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3813 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3814 return emulate_gp(ctxt, 0);
3816 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3818 ctxt->ops->set_idt(ctxt, &desc_ptr);
3819 /* Disable writeback. */
3820 ctxt->dst.type = OP_NONE;
3821 return X86EMUL_CONTINUE;
3824 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3826 return em_lgdt_lidt(ctxt, true);
3829 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3831 return em_lgdt_lidt(ctxt, false);
3834 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3836 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3837 ctxt->ops->cpl(ctxt) > 0)
3838 return emulate_gp(ctxt, 0);
3840 if (ctxt->dst.type == OP_MEM)
3841 ctxt->dst.bytes = 2;
3842 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3843 return X86EMUL_CONTINUE;
3846 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3848 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3849 | (ctxt->src.val & 0x0f));
3850 ctxt->dst.type = OP_NONE;
3851 return X86EMUL_CONTINUE;
3854 static int em_loop(struct x86_emulate_ctxt *ctxt)
3856 int rc = X86EMUL_CONTINUE;
3858 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3859 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3860 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3861 rc = jmp_rel(ctxt, ctxt->src.val);
3866 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3868 int rc = X86EMUL_CONTINUE;
3870 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3871 rc = jmp_rel(ctxt, ctxt->src.val);
3876 static int em_in(struct x86_emulate_ctxt *ctxt)
3878 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3880 return X86EMUL_IO_NEEDED;
3882 return X86EMUL_CONTINUE;
3885 static int em_out(struct x86_emulate_ctxt *ctxt)
3887 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3889 /* Disable writeback. */
3890 ctxt->dst.type = OP_NONE;
3891 return X86EMUL_CONTINUE;
3894 static int em_cli(struct x86_emulate_ctxt *ctxt)
3896 if (emulator_bad_iopl(ctxt))
3897 return emulate_gp(ctxt, 0);
3899 ctxt->eflags &= ~X86_EFLAGS_IF;
3900 return X86EMUL_CONTINUE;
3903 static int em_sti(struct x86_emulate_ctxt *ctxt)
3905 if (emulator_bad_iopl(ctxt))
3906 return emulate_gp(ctxt, 0);
3908 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3909 ctxt->eflags |= X86_EFLAGS_IF;
3910 return X86EMUL_CONTINUE;
3913 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3915 u32 eax, ebx, ecx, edx;
3918 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3919 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3920 ctxt->ops->cpl(ctxt)) {
3921 return emulate_gp(ctxt, 0);
3924 eax = reg_read(ctxt, VCPU_REGS_RAX);
3925 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3926 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3927 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3928 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3929 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3930 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3931 return X86EMUL_CONTINUE;
3934 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3938 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3940 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3942 ctxt->eflags &= ~0xffUL;
3943 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3944 return X86EMUL_CONTINUE;
3947 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3949 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3950 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3951 return X86EMUL_CONTINUE;
3954 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3956 switch (ctxt->op_bytes) {
3957 #ifdef CONFIG_X86_64
3959 asm("bswap %0" : "+r"(ctxt->dst.val));
3963 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3966 return X86EMUL_CONTINUE;
3969 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3971 /* emulating clflush regardless of cpuid */
3972 return X86EMUL_CONTINUE;
3975 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3977 /* emulating clflushopt regardless of cpuid */
3978 return X86EMUL_CONTINUE;
3981 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3983 ctxt->dst.val = (s32) ctxt->src.val;
3984 return X86EMUL_CONTINUE;
3987 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3989 if (!ctxt->ops->guest_has_fxsr(ctxt))
3990 return emulate_ud(ctxt);
3992 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3993 return emulate_nm(ctxt);
3996 * Don't emulate a case that should never be hit, instead of working
3997 * around a lack of fxsave64/fxrstor64 on old compilers.
3999 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4000 return X86EMUL_UNHANDLEABLE;
4002 return X86EMUL_CONTINUE;
4006 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4007 * and restore MXCSR.
4009 static size_t __fxstate_size(int nregs)
4011 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4014 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4017 if (ctxt->mode == X86EMUL_MODE_PROT64)
4018 return __fxstate_size(16);
4020 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4021 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4025 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4028 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4029 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4031 * 3) 64-bit mode with REX.W prefix
4032 * - like (2), but XMM 8-15 are being saved and restored
4033 * 4) 64-bit mode without REX.W prefix
4034 * - like (3), but FIP and FDP are 64 bit
4036 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4037 * desired result. (4) is not emulated.
4039 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4040 * and FPU DS) should match.
4042 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4044 struct fxregs_state fx_state;
4047 rc = check_fxsr(ctxt);
4048 if (rc != X86EMUL_CONTINUE)
4053 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4057 if (rc != X86EMUL_CONTINUE)
4060 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4061 fxstate_size(ctxt));
4065 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4066 * in the host registers (via FXSAVE) instead, so they won't be modified.
4067 * (preemption has to stay disabled until FXRSTOR).
4069 * Use noinline to keep the stack for other functions called by callers small.
4071 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4072 const size_t used_size)
4074 struct fxregs_state fx_tmp;
4077 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4078 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4079 __fxstate_size(16) - used_size);
4084 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4086 struct fxregs_state fx_state;
4090 rc = check_fxsr(ctxt);
4091 if (rc != X86EMUL_CONTINUE)
4094 size = fxstate_size(ctxt);
4095 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4096 if (rc != X86EMUL_CONTINUE)
4101 if (size < __fxstate_size(16)) {
4102 rc = fxregs_fixup(&fx_state, size);
4103 if (rc != X86EMUL_CONTINUE)
4107 if (fx_state.mxcsr >> 16) {
4108 rc = emulate_gp(ctxt, 0);
4112 if (rc == X86EMUL_CONTINUE)
4113 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4121 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4125 eax = reg_read(ctxt, VCPU_REGS_RAX);
4126 edx = reg_read(ctxt, VCPU_REGS_RDX);
4127 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4129 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4130 return emulate_gp(ctxt, 0);
4132 return X86EMUL_CONTINUE;
4135 static bool valid_cr(int nr)
4147 static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4149 if (!valid_cr(ctxt->modrm_reg))
4150 return emulate_ud(ctxt);
4152 return X86EMUL_CONTINUE;
4155 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4159 ctxt->ops->get_dr(ctxt, 7, &dr7);
4161 /* Check if DR7.Global_Enable is set */
4162 return dr7 & (1 << 13);
4165 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4167 int dr = ctxt->modrm_reg;
4171 return emulate_ud(ctxt);
4173 cr4 = ctxt->ops->get_cr(ctxt, 4);
4174 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4175 return emulate_ud(ctxt);
4177 if (check_dr7_gd(ctxt)) {
4180 ctxt->ops->get_dr(ctxt, 6, &dr6);
4181 dr6 &= ~DR_TRAP_BITS;
4182 dr6 |= DR6_BD | DR6_ACTIVE_LOW;
4183 ctxt->ops->set_dr(ctxt, 6, dr6);
4184 return emulate_db(ctxt);
4187 return X86EMUL_CONTINUE;
4190 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4192 u64 new_val = ctxt->src.val64;
4193 int dr = ctxt->modrm_reg;
4195 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4196 return emulate_gp(ctxt, 0);
4198 return check_dr_read(ctxt);
4201 static int check_svme(struct x86_emulate_ctxt *ctxt)
4205 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4207 if (!(efer & EFER_SVME))
4208 return emulate_ud(ctxt);
4210 return X86EMUL_CONTINUE;
4213 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4215 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4217 /* Valid physical address? */
4218 if (rax & 0xffff000000000000ULL)
4219 return emulate_gp(ctxt, 0);
4221 return check_svme(ctxt);
4224 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4226 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4228 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4229 return emulate_gp(ctxt, 0);
4231 return X86EMUL_CONTINUE;
4234 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4236 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4237 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4240 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4241 * in Ring3 when CR4.PCE=0.
4243 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4244 return X86EMUL_CONTINUE;
4246 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4247 ctxt->ops->check_pmc(ctxt, rcx))
4248 return emulate_gp(ctxt, 0);
4250 return X86EMUL_CONTINUE;
4253 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4255 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4256 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4257 return emulate_gp(ctxt, 0);
4259 return X86EMUL_CONTINUE;
4262 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4264 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4265 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4266 return emulate_gp(ctxt, 0);
4268 return X86EMUL_CONTINUE;
4271 #define D(_y) { .flags = (_y) }
4272 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4273 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4274 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4275 #define N D(NotImpl)
4276 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4277 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4278 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4279 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4280 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4281 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4282 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4283 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4284 #define II(_f, _e, _i) \
4285 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4286 #define IIP(_f, _e, _i, _p) \
4287 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4288 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4289 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4291 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4292 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4293 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4294 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4295 #define I2bvIP(_f, _e, _i, _p) \
4296 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4298 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4299 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4300 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4302 static const struct opcode group7_rm0[] = {
4304 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4308 static const struct opcode group7_rm1[] = {
4309 DI(SrcNone | Priv, monitor),
4310 DI(SrcNone | Priv, mwait),
4314 static const struct opcode group7_rm2[] = {
4316 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4320 static const struct opcode group7_rm3[] = {
4321 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4322 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4323 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4324 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4325 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4326 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4327 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4328 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4331 static const struct opcode group7_rm7[] = {
4333 DIP(SrcNone, rdtscp, check_rdtsc),
4337 static const struct opcode group1[] = {
4339 F(Lock | PageTable, em_or),
4342 F(Lock | PageTable, em_and),
4348 static const struct opcode group1A[] = {
4349 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4352 static const struct opcode group2[] = {
4353 F(DstMem | ModRM, em_rol),
4354 F(DstMem | ModRM, em_ror),
4355 F(DstMem | ModRM, em_rcl),
4356 F(DstMem | ModRM, em_rcr),
4357 F(DstMem | ModRM, em_shl),
4358 F(DstMem | ModRM, em_shr),
4359 F(DstMem | ModRM, em_shl),
4360 F(DstMem | ModRM, em_sar),
4363 static const struct opcode group3[] = {
4364 F(DstMem | SrcImm | NoWrite, em_test),
4365 F(DstMem | SrcImm | NoWrite, em_test),
4366 F(DstMem | SrcNone | Lock, em_not),
4367 F(DstMem | SrcNone | Lock, em_neg),
4368 F(DstXacc | Src2Mem, em_mul_ex),
4369 F(DstXacc | Src2Mem, em_imul_ex),
4370 F(DstXacc | Src2Mem, em_div_ex),
4371 F(DstXacc | Src2Mem, em_idiv_ex),
4374 static const struct opcode group4[] = {
4375 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4376 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4380 static const struct opcode group5[] = {
4381 F(DstMem | SrcNone | Lock, em_inc),
4382 F(DstMem | SrcNone | Lock, em_dec),
4383 I(SrcMem | NearBranch, em_call_near_abs),
4384 I(SrcMemFAddr | ImplicitOps, em_call_far),
4385 I(SrcMem | NearBranch, em_jmp_abs),
4386 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4387 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4390 static const struct opcode group6[] = {
4391 II(Prot | DstMem, em_sldt, sldt),
4392 II(Prot | DstMem, em_str, str),
4393 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4394 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4398 static const struct group_dual group7 = { {
4399 II(Mov | DstMem, em_sgdt, sgdt),
4400 II(Mov | DstMem, em_sidt, sidt),
4401 II(SrcMem | Priv, em_lgdt, lgdt),
4402 II(SrcMem | Priv, em_lidt, lidt),
4403 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4404 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4405 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4411 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4412 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4416 static const struct opcode group8[] = {
4418 F(DstMem | SrcImmByte | NoWrite, em_bt),
4419 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4420 F(DstMem | SrcImmByte | Lock, em_btr),
4421 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4425 * The "memory" destination is actually always a register, since we come
4426 * from the register case of group9.
4428 static const struct gprefix pfx_0f_c7_7 = {
4429 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdpid),
4433 static const struct group_dual group9 = { {
4434 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4436 N, N, N, N, N, N, N,
4437 GP(0, &pfx_0f_c7_7),
4440 static const struct opcode group11[] = {
4441 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4445 static const struct gprefix pfx_0f_ae_7 = {
4446 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4449 static const struct group_dual group15 = { {
4450 I(ModRM | Aligned16, em_fxsave),
4451 I(ModRM | Aligned16, em_fxrstor),
4452 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4454 N, N, N, N, N, N, N, N,
4457 static const struct gprefix pfx_0f_6f_0f_7f = {
4458 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4461 static const struct instr_dual instr_dual_0f_2b = {
4465 static const struct gprefix pfx_0f_2b = {
4466 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4469 static const struct gprefix pfx_0f_10_0f_11 = {
4470 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4473 static const struct gprefix pfx_0f_28_0f_29 = {
4474 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4477 static const struct gprefix pfx_0f_e7 = {
4478 N, I(Sse, em_mov), N, N,
4481 static const struct escape escape_d9 = { {
4482 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4485 N, N, N, N, N, N, N, N,
4487 N, N, N, N, N, N, N, N,
4489 N, N, N, N, N, N, N, N,
4491 N, N, N, N, N, N, N, N,
4493 N, N, N, N, N, N, N, N,
4495 N, N, N, N, N, N, N, N,
4497 N, N, N, N, N, N, N, N,
4499 N, N, N, N, N, N, N, N,
4502 static const struct escape escape_db = { {
4503 N, N, N, N, N, N, N, N,
4506 N, N, N, N, N, N, N, N,
4508 N, N, N, N, N, N, N, N,
4510 N, N, N, N, N, N, N, N,
4512 N, N, N, N, N, N, N, N,
4514 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4516 N, N, N, N, N, N, N, N,
4518 N, N, N, N, N, N, N, N,
4520 N, N, N, N, N, N, N, N,
4523 static const struct escape escape_dd = { {
4524 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4527 N, N, N, N, N, N, N, N,
4529 N, N, N, N, N, N, N, N,
4531 N, N, N, N, N, N, N, N,
4533 N, N, N, N, N, N, N, N,
4535 N, N, N, N, N, N, N, N,
4537 N, N, N, N, N, N, N, N,
4539 N, N, N, N, N, N, N, N,
4541 N, N, N, N, N, N, N, N,
4544 static const struct instr_dual instr_dual_0f_c3 = {
4545 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4548 static const struct mode_dual mode_dual_63 = {
4549 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4552 static const struct opcode opcode_table[256] = {
4554 F6ALU(Lock, em_add),
4555 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4556 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4558 F6ALU(Lock | PageTable, em_or),
4559 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4562 F6ALU(Lock, em_adc),
4563 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4564 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4566 F6ALU(Lock, em_sbb),
4567 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4568 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4570 F6ALU(Lock | PageTable, em_and), N, N,
4572 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4574 F6ALU(Lock, em_xor), N, N,
4576 F6ALU(NoWrite, em_cmp), N, N,
4578 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4580 X8(I(SrcReg | Stack, em_push)),
4582 X8(I(DstReg | Stack, em_pop)),
4584 I(ImplicitOps | Stack | No64, em_pusha),
4585 I(ImplicitOps | Stack | No64, em_popa),
4586 N, MD(ModRM, &mode_dual_63),
4589 I(SrcImm | Mov | Stack, em_push),
4590 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4591 I(SrcImmByte | Mov | Stack, em_push),
4592 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4593 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4594 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4596 X16(D(SrcImmByte | NearBranch)),
4598 G(ByteOp | DstMem | SrcImm, group1),
4599 G(DstMem | SrcImm, group1),
4600 G(ByteOp | DstMem | SrcImm | No64, group1),
4601 G(DstMem | SrcImmByte, group1),
4602 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4603 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4605 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4606 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4607 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4608 D(ModRM | SrcMem | NoAccess | DstReg),
4609 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4612 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4614 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4615 I(SrcImmFAddr | No64, em_call_far), N,
4616 II(ImplicitOps | Stack, em_pushf, pushf),
4617 II(ImplicitOps | Stack, em_popf, popf),
4618 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4620 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4621 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4622 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4623 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4625 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4626 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4627 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4628 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4630 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4632 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4634 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4635 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4636 I(ImplicitOps | NearBranch, em_ret),
4637 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4638 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4639 G(ByteOp, group11), G(0, group11),
4641 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4642 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4643 I(ImplicitOps, em_ret_far),
4644 D(ImplicitOps), DI(SrcImmByte, intn),
4645 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4647 G(Src2One | ByteOp, group2), G(Src2One, group2),
4648 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4649 I(DstAcc | SrcImmUByte | No64, em_aam),
4650 I(DstAcc | SrcImmUByte | No64, em_aad),
4651 F(DstAcc | ByteOp | No64, em_salc),
4652 I(DstAcc | SrcXLat | ByteOp, em_mov),
4654 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4656 X3(I(SrcImmByte | NearBranch, em_loop)),
4657 I(SrcImmByte | NearBranch, em_jcxz),
4658 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4659 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4661 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4662 I(SrcImmFAddr | No64, em_jmp_far),
4663 D(SrcImmByte | ImplicitOps | NearBranch),
4664 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4665 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4667 N, DI(ImplicitOps, icebp), N, N,
4668 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4669 G(ByteOp, group3), G(0, group3),
4671 D(ImplicitOps), D(ImplicitOps),
4672 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4673 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4676 static const struct opcode twobyte_table[256] = {
4678 G(0, group6), GD(0, &group7), N, N,
4679 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4680 II(ImplicitOps | Priv, em_clts, clts), N,
4681 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4682 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4684 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4685 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4687 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4688 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4689 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4690 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4691 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4692 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4694 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
4695 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4696 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4698 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4701 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4702 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4703 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4706 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4707 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4708 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4709 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4710 I(ImplicitOps | EmulateOnUD, em_sysenter),
4711 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4713 N, N, N, N, N, N, N, N,
4715 X16(D(DstReg | SrcMem | ModRM)),
4717 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4722 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4727 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4729 X16(D(SrcImm | NearBranch)),
4731 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4733 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4734 II(ImplicitOps, em_cpuid, cpuid),
4735 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4736 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4737 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4739 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4740 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4741 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4742 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4743 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4744 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4746 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4747 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4748 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4749 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4750 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4751 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4755 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4756 I(DstReg | SrcMem | ModRM, em_bsf_c),
4757 I(DstReg | SrcMem | ModRM, em_bsr_c),
4758 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4760 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4761 N, ID(0, &instr_dual_0f_c3),
4762 N, N, N, GD(0, &group9),
4764 X8(I(DstReg, em_bswap)),
4766 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4768 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4769 N, N, N, N, N, N, N, N,
4771 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4774 static const struct instr_dual instr_dual_0f_38_f0 = {
4775 I(DstReg | SrcMem | Mov, em_movbe), N
4778 static const struct instr_dual instr_dual_0f_38_f1 = {
4779 I(DstMem | SrcReg | Mov, em_movbe), N
4782 static const struct gprefix three_byte_0f_38_f0 = {
4783 ID(0, &instr_dual_0f_38_f0), N, N, N
4786 static const struct gprefix three_byte_0f_38_f1 = {
4787 ID(0, &instr_dual_0f_38_f1), N, N, N
4791 * Insns below are selected by the prefix which indexed by the third opcode
4794 static const struct opcode opcode_map_0f_38[256] = {
4796 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4798 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4800 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4801 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4822 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4826 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4832 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4833 unsigned size, bool sign_extension)
4835 int rc = X86EMUL_CONTINUE;
4839 op->addr.mem.ea = ctxt->_eip;
4840 /* NB. Immediates are sign-extended as necessary. */
4841 switch (op->bytes) {
4843 op->val = insn_fetch(s8, ctxt);
4846 op->val = insn_fetch(s16, ctxt);
4849 op->val = insn_fetch(s32, ctxt);
4852 op->val = insn_fetch(s64, ctxt);
4855 if (!sign_extension) {
4856 switch (op->bytes) {
4864 op->val &= 0xffffffff;
4872 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4875 int rc = X86EMUL_CONTINUE;
4879 decode_register_operand(ctxt, op);
4882 rc = decode_imm(ctxt, op, 1, false);
4885 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4889 if (ctxt->d & BitOp)
4890 fetch_bit_operand(ctxt);
4891 op->orig_val = op->val;
4894 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4898 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4899 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4900 fetch_register_operand(op);
4901 op->orig_val = op->val;
4905 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4906 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4907 fetch_register_operand(op);
4908 op->orig_val = op->val;
4911 if (ctxt->d & ByteOp) {
4916 op->bytes = ctxt->op_bytes;
4917 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4918 fetch_register_operand(op);
4919 op->orig_val = op->val;
4923 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4925 register_address(ctxt, VCPU_REGS_RDI);
4926 op->addr.mem.seg = VCPU_SREG_ES;
4933 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4934 fetch_register_operand(op);
4939 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4942 rc = decode_imm(ctxt, op, 1, true);
4950 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4953 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4956 ctxt->memop.bytes = 1;
4957 if (ctxt->memop.type == OP_REG) {
4958 ctxt->memop.addr.reg = decode_register(ctxt,
4959 ctxt->modrm_rm, true);
4960 fetch_register_operand(&ctxt->memop);
4964 ctxt->memop.bytes = 2;
4967 ctxt->memop.bytes = 4;
4970 rc = decode_imm(ctxt, op, 2, false);
4973 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4977 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4979 register_address(ctxt, VCPU_REGS_RSI);
4980 op->addr.mem.seg = ctxt->seg_override;
4986 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4989 reg_read(ctxt, VCPU_REGS_RBX) +
4990 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4991 op->addr.mem.seg = ctxt->seg_override;
4996 op->addr.mem.ea = ctxt->_eip;
4997 op->bytes = ctxt->op_bytes + 2;
4998 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5001 ctxt->memop.bytes = ctxt->op_bytes + 2;
5005 op->val = VCPU_SREG_ES;
5009 op->val = VCPU_SREG_CS;
5013 op->val = VCPU_SREG_SS;
5017 op->val = VCPU_SREG_DS;
5021 op->val = VCPU_SREG_FS;
5025 op->val = VCPU_SREG_GS;
5028 /* Special instructions do their own operand decoding. */
5030 op->type = OP_NONE; /* Disable writeback. */
5038 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)
5040 int rc = X86EMUL_CONTINUE;
5041 int mode = ctxt->mode;
5042 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5043 bool op_prefix = false;
5044 bool has_seg_override = false;
5045 struct opcode opcode;
5047 struct desc_struct desc;
5049 ctxt->memop.type = OP_NONE;
5050 ctxt->memopp = NULL;
5051 ctxt->_eip = ctxt->eip;
5052 ctxt->fetch.ptr = ctxt->fetch.data;
5053 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5054 ctxt->opcode_len = 1;
5055 ctxt->intercept = x86_intercept_none;
5057 memcpy(ctxt->fetch.data, insn, insn_len);
5059 rc = __do_insn_fetch_bytes(ctxt, 1);
5060 if (rc != X86EMUL_CONTINUE)
5065 case X86EMUL_MODE_REAL:
5066 case X86EMUL_MODE_VM86:
5067 def_op_bytes = def_ad_bytes = 2;
5068 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5070 def_op_bytes = def_ad_bytes = 4;
5072 case X86EMUL_MODE_PROT16:
5073 def_op_bytes = def_ad_bytes = 2;
5075 case X86EMUL_MODE_PROT32:
5076 def_op_bytes = def_ad_bytes = 4;
5078 #ifdef CONFIG_X86_64
5079 case X86EMUL_MODE_PROT64:
5085 return EMULATION_FAILED;
5088 ctxt->op_bytes = def_op_bytes;
5089 ctxt->ad_bytes = def_ad_bytes;
5091 /* Legacy prefixes. */
5093 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5094 case 0x66: /* operand-size override */
5096 /* switch between 2/4 bytes */
5097 ctxt->op_bytes = def_op_bytes ^ 6;
5099 case 0x67: /* address-size override */
5100 if (mode == X86EMUL_MODE_PROT64)
5101 /* switch between 4/8 bytes */
5102 ctxt->ad_bytes = def_ad_bytes ^ 12;
5104 /* switch between 2/4 bytes */
5105 ctxt->ad_bytes = def_ad_bytes ^ 6;
5107 case 0x26: /* ES override */
5108 has_seg_override = true;
5109 ctxt->seg_override = VCPU_SREG_ES;
5111 case 0x2e: /* CS override */
5112 has_seg_override = true;
5113 ctxt->seg_override = VCPU_SREG_CS;
5115 case 0x36: /* SS override */
5116 has_seg_override = true;
5117 ctxt->seg_override = VCPU_SREG_SS;
5119 case 0x3e: /* DS override */
5120 has_seg_override = true;
5121 ctxt->seg_override = VCPU_SREG_DS;
5123 case 0x64: /* FS override */
5124 has_seg_override = true;
5125 ctxt->seg_override = VCPU_SREG_FS;
5127 case 0x65: /* GS override */
5128 has_seg_override = true;
5129 ctxt->seg_override = VCPU_SREG_GS;
5131 case 0x40 ... 0x4f: /* REX */
5132 if (mode != X86EMUL_MODE_PROT64)
5134 ctxt->rex_prefix = ctxt->b;
5136 case 0xf0: /* LOCK */
5137 ctxt->lock_prefix = 1;
5139 case 0xf2: /* REPNE/REPNZ */
5140 case 0xf3: /* REP/REPE/REPZ */
5141 ctxt->rep_prefix = ctxt->b;
5147 /* Any legacy prefix after a REX prefix nullifies its effect. */
5149 ctxt->rex_prefix = 0;
5155 if (ctxt->rex_prefix & 8)
5156 ctxt->op_bytes = 8; /* REX.W */
5158 /* Opcode byte(s). */
5159 opcode = opcode_table[ctxt->b];
5160 /* Two-byte opcode? */
5161 if (ctxt->b == 0x0f) {
5162 ctxt->opcode_len = 2;
5163 ctxt->b = insn_fetch(u8, ctxt);
5164 opcode = twobyte_table[ctxt->b];
5166 /* 0F_38 opcode map */
5167 if (ctxt->b == 0x38) {
5168 ctxt->opcode_len = 3;
5169 ctxt->b = insn_fetch(u8, ctxt);
5170 opcode = opcode_map_0f_38[ctxt->b];
5173 ctxt->d = opcode.flags;
5175 if (ctxt->d & ModRM)
5176 ctxt->modrm = insn_fetch(u8, ctxt);
5178 /* vex-prefix instructions are not implemented */
5179 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5180 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5184 while (ctxt->d & GroupMask) {
5185 switch (ctxt->d & GroupMask) {
5187 goffset = (ctxt->modrm >> 3) & 7;
5188 opcode = opcode.u.group[goffset];
5191 goffset = (ctxt->modrm >> 3) & 7;
5192 if ((ctxt->modrm >> 6) == 3)
5193 opcode = opcode.u.gdual->mod3[goffset];
5195 opcode = opcode.u.gdual->mod012[goffset];
5198 goffset = ctxt->modrm & 7;
5199 opcode = opcode.u.group[goffset];
5202 if (ctxt->rep_prefix && op_prefix)
5203 return EMULATION_FAILED;
5204 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5205 switch (simd_prefix) {
5206 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5207 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5208 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5209 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5213 if (ctxt->modrm > 0xbf) {
5214 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5215 u32 index = array_index_nospec(
5216 ctxt->modrm - 0xc0, size);
5218 opcode = opcode.u.esc->high[index];
5220 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5224 if ((ctxt->modrm >> 6) == 3)
5225 opcode = opcode.u.idual->mod3;
5227 opcode = opcode.u.idual->mod012;
5230 if (ctxt->mode == X86EMUL_MODE_PROT64)
5231 opcode = opcode.u.mdual->mode64;
5233 opcode = opcode.u.mdual->mode32;
5236 return EMULATION_FAILED;
5239 ctxt->d &= ~(u64)GroupMask;
5240 ctxt->d |= opcode.flags;
5245 return EMULATION_FAILED;
5247 ctxt->execute = opcode.u.execute;
5249 if (unlikely(emulation_type & EMULTYPE_TRAP_UD) &&
5250 likely(!(ctxt->d & EmulateOnUD)))
5251 return EMULATION_FAILED;
5253 if (unlikely(ctxt->d &
5254 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5257 * These are copied unconditionally here, and checked unconditionally
5258 * in x86_emulate_insn.
5260 ctxt->check_perm = opcode.check_perm;
5261 ctxt->intercept = opcode.intercept;
5263 if (ctxt->d & NotImpl)
5264 return EMULATION_FAILED;
5266 if (mode == X86EMUL_MODE_PROT64) {
5267 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5269 else if (ctxt->d & NearBranch)
5273 if (ctxt->d & Op3264) {
5274 if (mode == X86EMUL_MODE_PROT64)
5280 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5284 ctxt->op_bytes = 16;
5285 else if (ctxt->d & Mmx)
5289 /* ModRM and SIB bytes. */
5290 if (ctxt->d & ModRM) {
5291 rc = decode_modrm(ctxt, &ctxt->memop);
5292 if (!has_seg_override) {
5293 has_seg_override = true;
5294 ctxt->seg_override = ctxt->modrm_seg;
5296 } else if (ctxt->d & MemAbs)
5297 rc = decode_abs(ctxt, &ctxt->memop);
5298 if (rc != X86EMUL_CONTINUE)
5301 if (!has_seg_override)
5302 ctxt->seg_override = VCPU_SREG_DS;
5304 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5307 * Decode and fetch the source operand: register, memory
5310 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5311 if (rc != X86EMUL_CONTINUE)
5315 * Decode and fetch the second source operand: register, memory
5318 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5319 if (rc != X86EMUL_CONTINUE)
5322 /* Decode and fetch the destination operand: register or memory. */
5323 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5325 if (ctxt->rip_relative && likely(ctxt->memopp))
5326 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5327 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5330 if (rc == X86EMUL_PROPAGATE_FAULT)
5331 ctxt->have_exception = true;
5332 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5335 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5337 return ctxt->d & PageTable;
5340 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5342 /* The second termination condition only applies for REPE
5343 * and REPNE. Test if the repeat string operation prefix is
5344 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5345 * corresponding termination condition according to:
5346 * - if REPE/REPZ and ZF = 0 then done
5347 * - if REPNE/REPNZ and ZF = 1 then done
5349 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5350 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5351 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5352 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5353 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5354 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5360 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5365 rc = asm_safe("fwait");
5368 if (unlikely(rc != X86EMUL_CONTINUE))
5369 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5371 return X86EMUL_CONTINUE;
5374 static void fetch_possible_mmx_operand(struct operand *op)
5376 if (op->type == OP_MM)
5377 kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5380 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5382 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5384 if (!(ctxt->d & ByteOp))
5385 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5387 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5388 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5389 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5390 : "c"(ctxt->src2.val));
5392 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5393 if (!fop) /* exception is returned in fop variable */
5394 return emulate_de(ctxt);
5395 return X86EMUL_CONTINUE;
5398 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5400 memset(&ctxt->rip_relative, 0,
5401 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5403 ctxt->io_read.pos = 0;
5404 ctxt->io_read.end = 0;
5405 ctxt->mem_read.end = 0;
5408 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5410 const struct x86_emulate_ops *ops = ctxt->ops;
5411 int rc = X86EMUL_CONTINUE;
5412 int saved_dst_type = ctxt->dst.type;
5413 unsigned emul_flags;
5415 ctxt->mem_read.pos = 0;
5417 /* LOCK prefix is allowed only with some instructions */
5418 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5419 rc = emulate_ud(ctxt);
5423 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5424 rc = emulate_ud(ctxt);
5428 emul_flags = ctxt->ops->get_hflags(ctxt);
5429 if (unlikely(ctxt->d &
5430 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5431 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5432 (ctxt->d & Undefined)) {
5433 rc = emulate_ud(ctxt);
5437 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5438 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5439 rc = emulate_ud(ctxt);
5443 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5444 rc = emulate_nm(ctxt);
5448 if (ctxt->d & Mmx) {
5449 rc = flush_pending_x87_faults(ctxt);
5450 if (rc != X86EMUL_CONTINUE)
5453 * Now that we know the fpu is exception safe, we can fetch
5456 fetch_possible_mmx_operand(&ctxt->src);
5457 fetch_possible_mmx_operand(&ctxt->src2);
5458 if (!(ctxt->d & Mov))
5459 fetch_possible_mmx_operand(&ctxt->dst);
5462 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5463 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5464 X86_ICPT_PRE_EXCEPT);
5465 if (rc != X86EMUL_CONTINUE)
5469 /* Instruction can only be executed in protected mode */
5470 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5471 rc = emulate_ud(ctxt);
5475 /* Privileged instruction can be executed only in CPL=0 */
5476 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5477 if (ctxt->d & PrivUD)
5478 rc = emulate_ud(ctxt);
5480 rc = emulate_gp(ctxt, 0);
5484 /* Do instruction specific permission checks */
5485 if (ctxt->d & CheckPerm) {
5486 rc = ctxt->check_perm(ctxt);
5487 if (rc != X86EMUL_CONTINUE)
5491 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5492 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5493 X86_ICPT_POST_EXCEPT);
5494 if (rc != X86EMUL_CONTINUE)
5498 if (ctxt->rep_prefix && (ctxt->d & String)) {
5499 /* All REP prefixes have the same first termination condition */
5500 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5501 string_registers_quirk(ctxt);
5502 ctxt->eip = ctxt->_eip;
5503 ctxt->eflags &= ~X86_EFLAGS_RF;
5509 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5510 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5511 ctxt->src.valptr, ctxt->src.bytes);
5512 if (rc != X86EMUL_CONTINUE)
5514 ctxt->src.orig_val64 = ctxt->src.val64;
5517 if (ctxt->src2.type == OP_MEM) {
5518 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5519 &ctxt->src2.val, ctxt->src2.bytes);
5520 if (rc != X86EMUL_CONTINUE)
5524 if ((ctxt->d & DstMask) == ImplicitOps)
5528 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5529 /* optimisation - avoid slow emulated read if Mov */
5530 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5531 &ctxt->dst.val, ctxt->dst.bytes);
5532 if (rc != X86EMUL_CONTINUE) {
5533 if (!(ctxt->d & NoWrite) &&
5534 rc == X86EMUL_PROPAGATE_FAULT &&
5535 ctxt->exception.vector == PF_VECTOR)
5536 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5540 /* Copy full 64-bit value for CMPXCHG8B. */
5541 ctxt->dst.orig_val64 = ctxt->dst.val64;
5545 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5546 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5547 X86_ICPT_POST_MEMACCESS);
5548 if (rc != X86EMUL_CONTINUE)
5552 if (ctxt->rep_prefix && (ctxt->d & String))
5553 ctxt->eflags |= X86_EFLAGS_RF;
5555 ctxt->eflags &= ~X86_EFLAGS_RF;
5557 if (ctxt->execute) {
5558 if (ctxt->d & Fastop)
5559 rc = fastop(ctxt, ctxt->fop);
5561 rc = ctxt->execute(ctxt);
5562 if (rc != X86EMUL_CONTINUE)
5567 if (ctxt->opcode_len == 2)
5569 else if (ctxt->opcode_len == 3)
5570 goto threebyte_insn;
5573 case 0x70 ... 0x7f: /* jcc (short) */
5574 if (test_cc(ctxt->b, ctxt->eflags))
5575 rc = jmp_rel(ctxt, ctxt->src.val);
5577 case 0x8d: /* lea r16/r32, m */
5578 ctxt->dst.val = ctxt->src.addr.mem.ea;
5580 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5581 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5582 ctxt->dst.type = OP_NONE;
5586 case 0x98: /* cbw/cwde/cdqe */
5587 switch (ctxt->op_bytes) {
5588 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5589 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5590 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5593 case 0xcc: /* int3 */
5594 rc = emulate_int(ctxt, 3);
5596 case 0xcd: /* int n */
5597 rc = emulate_int(ctxt, ctxt->src.val);
5599 case 0xce: /* into */
5600 if (ctxt->eflags & X86_EFLAGS_OF)
5601 rc = emulate_int(ctxt, 4);
5603 case 0xe9: /* jmp rel */
5604 case 0xeb: /* jmp rel short */
5605 rc = jmp_rel(ctxt, ctxt->src.val);
5606 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5608 case 0xf4: /* hlt */
5609 ctxt->ops->halt(ctxt);
5611 case 0xf5: /* cmc */
5612 /* complement carry flag from eflags reg */
5613 ctxt->eflags ^= X86_EFLAGS_CF;
5615 case 0xf8: /* clc */
5616 ctxt->eflags &= ~X86_EFLAGS_CF;
5618 case 0xf9: /* stc */
5619 ctxt->eflags |= X86_EFLAGS_CF;
5621 case 0xfc: /* cld */
5622 ctxt->eflags &= ~X86_EFLAGS_DF;
5624 case 0xfd: /* std */
5625 ctxt->eflags |= X86_EFLAGS_DF;
5628 goto cannot_emulate;
5631 if (rc != X86EMUL_CONTINUE)
5635 if (ctxt->d & SrcWrite) {
5636 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5637 rc = writeback(ctxt, &ctxt->src);
5638 if (rc != X86EMUL_CONTINUE)
5641 if (!(ctxt->d & NoWrite)) {
5642 rc = writeback(ctxt, &ctxt->dst);
5643 if (rc != X86EMUL_CONTINUE)
5648 * restore dst type in case the decoding will be reused
5649 * (happens for string instruction )
5651 ctxt->dst.type = saved_dst_type;
5653 if ((ctxt->d & SrcMask) == SrcSI)
5654 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5656 if ((ctxt->d & DstMask) == DstDI)
5657 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5659 if (ctxt->rep_prefix && (ctxt->d & String)) {
5661 struct read_cache *r = &ctxt->io_read;
5662 if ((ctxt->d & SrcMask) == SrcSI)
5663 count = ctxt->src.count;
5665 count = ctxt->dst.count;
5666 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5668 if (!string_insn_completed(ctxt)) {
5670 * Re-enter guest when pio read ahead buffer is empty
5671 * or, if it is not used, after each 1024 iteration.
5673 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5674 (r->end == 0 || r->end != r->pos)) {
5676 * Reset read cache. Usually happens before
5677 * decode, but since instruction is restarted
5678 * we have to do it here.
5680 ctxt->mem_read.end = 0;
5681 writeback_registers(ctxt);
5682 return EMULATION_RESTART;
5684 goto done; /* skip rip writeback */
5686 ctxt->eflags &= ~X86_EFLAGS_RF;
5689 ctxt->eip = ctxt->_eip;
5690 if (ctxt->mode != X86EMUL_MODE_PROT64)
5691 ctxt->eip = (u32)ctxt->_eip;
5694 if (rc == X86EMUL_PROPAGATE_FAULT) {
5695 WARN_ON(ctxt->exception.vector > 0x1f);
5696 ctxt->have_exception = true;
5698 if (rc == X86EMUL_INTERCEPTED)
5699 return EMULATION_INTERCEPTED;
5701 if (rc == X86EMUL_CONTINUE)
5702 writeback_registers(ctxt);
5704 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5708 case 0x09: /* wbinvd */
5709 (ctxt->ops->wbinvd)(ctxt);
5711 case 0x08: /* invd */
5712 case 0x0d: /* GrpP (prefetch) */
5713 case 0x18: /* Grp16 (prefetch/nop) */
5714 case 0x1f: /* nop */
5716 case 0x20: /* mov cr, reg */
5717 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5719 case 0x21: /* mov from dr to reg */
5720 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5722 case 0x40 ... 0x4f: /* cmov */
5723 if (test_cc(ctxt->b, ctxt->eflags))
5724 ctxt->dst.val = ctxt->src.val;
5725 else if (ctxt->op_bytes != 4)
5726 ctxt->dst.type = OP_NONE; /* no writeback */
5728 case 0x80 ... 0x8f: /* jnz rel, etc*/
5729 if (test_cc(ctxt->b, ctxt->eflags))
5730 rc = jmp_rel(ctxt, ctxt->src.val);
5732 case 0x90 ... 0x9f: /* setcc r/m8 */
5733 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5735 case 0xb6 ... 0xb7: /* movzx */
5736 ctxt->dst.bytes = ctxt->op_bytes;
5737 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5738 : (u16) ctxt->src.val;
5740 case 0xbe ... 0xbf: /* movsx */
5741 ctxt->dst.bytes = ctxt->op_bytes;
5742 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5743 (s16) ctxt->src.val;
5746 goto cannot_emulate;
5751 if (rc != X86EMUL_CONTINUE)
5757 return EMULATION_FAILED;
5760 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5762 invalidate_registers(ctxt);
5765 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5767 writeback_registers(ctxt);
5770 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5772 if (ctxt->rep_prefix && (ctxt->d & String))
5775 if (ctxt->d & TwoMemOp)