1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
28 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
195 * fastop functions have a special calling convention:
200 * flags: rflags (in/out)
201 * ex: rsi (in:fastop pointer, out:zero if exception)
203 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
204 * different operand sizes can be reached by calculation, rather than a jump
205 * table (which would be bigger than the code).
207 * fastop functions are declared as taking a never-defined fastop parameter,
208 * so they can't be called from C directly.
217 int (*execute)(struct x86_emulate_ctxt *ctxt);
218 const struct opcode *group;
219 const struct group_dual *gdual;
220 const struct gprefix *gprefix;
221 const struct escape *esc;
222 const struct instr_dual *idual;
223 const struct mode_dual *mdual;
224 void (*fastop)(struct fastop *fake);
226 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
230 struct opcode mod012[8];
231 struct opcode mod3[8];
235 struct opcode pfx_no;
236 struct opcode pfx_66;
237 struct opcode pfx_f2;
238 struct opcode pfx_f3;
243 struct opcode high[64];
247 struct opcode mod012;
252 struct opcode mode32;
253 struct opcode mode64;
256 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
258 enum x86_transfer_type {
260 X86_TRANSFER_CALL_JMP,
262 X86_TRANSFER_TASK_SWITCH,
265 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 if (!(ctxt->regs_valid & (1 << nr))) {
268 ctxt->regs_valid |= 1 << nr;
269 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
271 return ctxt->_regs[nr];
274 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
276 ctxt->regs_valid |= 1 << nr;
277 ctxt->regs_dirty |= 1 << nr;
278 return &ctxt->_regs[nr];
281 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
284 return reg_write(ctxt, nr);
287 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
291 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
292 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
295 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
297 ctxt->regs_dirty = 0;
298 ctxt->regs_valid = 0;
302 * These EFLAGS bits are restored from saved value during emulation, and
303 * any changes are written back to the saved value after emulation.
305 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
306 X86_EFLAGS_PF|X86_EFLAGS_CF)
314 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
316 #define FOP_FUNC(name) \
317 ".align " __stringify(FASTOP_SIZE) " \n\t" \
318 ".type " name ", @function \n\t" \
321 #define FOP_RET "ret \n\t"
323 #define FOP_START(op) \
324 extern void em_##op(struct fastop *fake); \
325 asm(".pushsection .text, \"ax\" \n\t" \
326 ".global em_" #op " \n\t" \
333 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
336 #define FOP1E(op, dst) \
337 FOP_FUNC(#op "_" #dst) \
338 "10: " #op " %" #dst " \n\t" FOP_RET
340 #define FOP1EEX(op, dst) \
341 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
343 #define FASTOP1(op) \
348 ON64(FOP1E(op##q, rax)) \
351 /* 1-operand, using src2 (for MUL/DIV r/m) */
352 #define FASTOP1SRC2(op, name) \
357 ON64(FOP1E(op, rcx)) \
360 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
361 #define FASTOP1SRC2EX(op, name) \
366 ON64(FOP1EEX(op, rcx)) \
369 #define FOP2E(op, dst, src) \
370 FOP_FUNC(#op "_" #dst "_" #src) \
371 #op " %" #src ", %" #dst " \n\t" FOP_RET
373 #define FASTOP2(op) \
375 FOP2E(op##b, al, dl) \
376 FOP2E(op##w, ax, dx) \
377 FOP2E(op##l, eax, edx) \
378 ON64(FOP2E(op##q, rax, rdx)) \
381 /* 2 operand, word only */
382 #define FASTOP2W(op) \
385 FOP2E(op##w, ax, dx) \
386 FOP2E(op##l, eax, edx) \
387 ON64(FOP2E(op##q, rax, rdx)) \
390 /* 2 operand, src is CL */
391 #define FASTOP2CL(op) \
393 FOP2E(op##b, al, cl) \
394 FOP2E(op##w, ax, cl) \
395 FOP2E(op##l, eax, cl) \
396 ON64(FOP2E(op##q, rax, cl)) \
399 /* 2 operand, src and dest are reversed */
400 #define FASTOP2R(op, name) \
402 FOP2E(op##b, dl, al) \
403 FOP2E(op##w, dx, ax) \
404 FOP2E(op##l, edx, eax) \
405 ON64(FOP2E(op##q, rdx, rax)) \
408 #define FOP3E(op, dst, src, src2) \
409 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
410 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
412 /* 3-operand, word-only, src2=cl */
413 #define FASTOP3WCL(op) \
416 FOP3E(op##w, ax, dx, cl) \
417 FOP3E(op##l, eax, edx, cl) \
418 ON64(FOP3E(op##q, rax, rdx, cl)) \
421 /* Special case for SETcc - 1 instruction per cc */
422 #define FOP_SETCC(op) \
424 ".type " #op ", @function \n\t" \
429 asm(".pushsection .fixup, \"ax\"\n"
430 ".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret\n"
453 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
457 * XXX: inoutclob user must know where the argument is being expanded.
458 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
460 #define asm_safe(insn, inoutclob...) \
464 asm volatile("1:" insn "\n" \
466 ".pushsection .fixup, \"ax\"\n" \
467 "3: movl $1, %[_fault]\n" \
470 _ASM_EXTABLE(1b, 3b) \
471 : [_fault] "+qm"(_fault) inoutclob ); \
473 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
476 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
477 enum x86_intercept intercept,
478 enum x86_intercept_stage stage)
480 struct x86_instruction_info info = {
481 .intercept = intercept,
482 .rep_prefix = ctxt->rep_prefix,
483 .modrm_mod = ctxt->modrm_mod,
484 .modrm_reg = ctxt->modrm_reg,
485 .modrm_rm = ctxt->modrm_rm,
486 .src_val = ctxt->src.val64,
487 .dst_val = ctxt->dst.val64,
488 .src_bytes = ctxt->src.bytes,
489 .dst_bytes = ctxt->dst.bytes,
490 .ad_bytes = ctxt->ad_bytes,
491 .next_rip = ctxt->eip,
494 return ctxt->ops->intercept(ctxt, &info, stage);
497 static void assign_masked(ulong *dest, ulong src, ulong mask)
499 *dest = (*dest & ~mask) | (src & mask);
502 static void assign_register(unsigned long *reg, u64 val, int bytes)
504 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
507 *(u8 *)reg = (u8)val;
510 *(u16 *)reg = (u16)val;
514 break; /* 64b: zero-extend */
521 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
523 return (1UL << (ctxt->ad_bytes << 3)) - 1;
526 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
529 struct desc_struct ss;
531 if (ctxt->mode == X86EMUL_MODE_PROT64)
533 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
534 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
537 static int stack_size(struct x86_emulate_ctxt *ctxt)
539 return (__fls(stack_mask(ctxt)) + 1) >> 3;
542 /* Access/update address held in a register, based on addressing mode. */
543 static inline unsigned long
544 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
546 if (ctxt->ad_bytes == sizeof(unsigned long))
549 return reg & ad_mask(ctxt);
552 static inline unsigned long
553 register_address(struct x86_emulate_ctxt *ctxt, int reg)
555 return address_mask(ctxt, reg_read(ctxt, reg));
558 static void masked_increment(ulong *reg, ulong mask, int inc)
560 assign_masked(reg, *reg + inc, mask);
564 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
566 ulong *preg = reg_rmw(ctxt, reg);
568 assign_register(preg, *preg + inc, ctxt->ad_bytes);
571 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
573 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
576 static u32 desc_limit_scaled(struct desc_struct *desc)
578 u32 limit = get_desc_limit(desc);
580 return desc->g ? (limit << 12) | 0xfff : limit;
583 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
585 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
588 return ctxt->ops->get_cached_segment_base(ctxt, seg);
591 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
592 u32 error, bool valid)
595 ctxt->exception.vector = vec;
596 ctxt->exception.error_code = error;
597 ctxt->exception.error_code_valid = valid;
598 return X86EMUL_PROPAGATE_FAULT;
601 static int emulate_db(struct x86_emulate_ctxt *ctxt)
603 return emulate_exception(ctxt, DB_VECTOR, 0, false);
606 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
608 return emulate_exception(ctxt, GP_VECTOR, err, true);
611 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
613 return emulate_exception(ctxt, SS_VECTOR, err, true);
616 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
618 return emulate_exception(ctxt, UD_VECTOR, 0, false);
621 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
623 return emulate_exception(ctxt, TS_VECTOR, err, true);
626 static int emulate_de(struct x86_emulate_ctxt *ctxt)
628 return emulate_exception(ctxt, DE_VECTOR, 0, false);
631 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
633 return emulate_exception(ctxt, NM_VECTOR, 0, false);
636 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
639 struct desc_struct desc;
641 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
645 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
650 struct desc_struct desc;
652 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
653 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
657 * x86 defines three classes of vector instructions: explicitly
658 * aligned, explicitly unaligned, and the rest, which change behaviour
659 * depending on whether they're AVX encoded or not.
661 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
662 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
663 * 512 bytes of data must be aligned to a 16 byte boundary.
665 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
667 u64 alignment = ctxt->d & AlignMask;
669 if (likely(size < 16))
684 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
685 struct segmented_address addr,
686 unsigned *max_size, unsigned size,
687 bool write, bool fetch,
688 enum x86emul_mode mode, ulong *linear)
690 struct desc_struct desc;
697 la = seg_base(ctxt, addr.seg) + addr.ea;
700 case X86EMUL_MODE_PROT64:
702 va_bits = ctxt_virt_addr_bits(ctxt);
703 if (get_canonical(la, va_bits) != la)
706 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
707 if (size > *max_size)
711 *linear = la = (u32)la;
712 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
716 /* code segment in protected mode or read-only data segment */
717 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
718 || !(desc.type & 2)) && write)
720 /* unreadable code segment */
721 if (!fetch && (desc.type & 8) && !(desc.type & 2))
723 lim = desc_limit_scaled(&desc);
724 if (!(desc.type & 8) && (desc.type & 4)) {
725 /* expand-down segment */
728 lim = desc.d ? 0xffffffff : 0xffff;
732 if (lim == 0xffffffff)
735 *max_size = (u64)lim + 1 - addr.ea;
736 if (size > *max_size)
741 if (la & (insn_alignment(ctxt, size) - 1))
742 return emulate_gp(ctxt, 0);
743 return X86EMUL_CONTINUE;
745 if (addr.seg == VCPU_SREG_SS)
746 return emulate_ss(ctxt, 0);
748 return emulate_gp(ctxt, 0);
751 static int linearize(struct x86_emulate_ctxt *ctxt,
752 struct segmented_address addr,
753 unsigned size, bool write,
757 return __linearize(ctxt, addr, &max_size, size, write, false,
761 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
762 enum x86emul_mode mode)
767 struct segmented_address addr = { .seg = VCPU_SREG_CS,
770 if (ctxt->op_bytes != sizeof(unsigned long))
771 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
772 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
773 if (rc == X86EMUL_CONTINUE)
774 ctxt->_eip = addr.ea;
778 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
780 return assign_eip(ctxt, dst, ctxt->mode);
783 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
784 const struct desc_struct *cs_desc)
786 enum x86emul_mode mode = ctxt->mode;
790 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
794 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
796 mode = X86EMUL_MODE_PROT64;
798 mode = X86EMUL_MODE_PROT32; /* temporary value */
801 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
802 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
803 rc = assign_eip(ctxt, dst, mode);
804 if (rc == X86EMUL_CONTINUE)
809 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
811 return assign_eip_near(ctxt, ctxt->_eip + rel);
814 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
815 void *data, unsigned size)
817 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
820 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
821 ulong linear, void *data,
824 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
827 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
828 struct segmented_address addr,
835 rc = linearize(ctxt, addr, size, false, &linear);
836 if (rc != X86EMUL_CONTINUE)
838 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
841 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
842 struct segmented_address addr,
849 rc = linearize(ctxt, addr, size, true, &linear);
850 if (rc != X86EMUL_CONTINUE)
852 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
856 * Prefetch the remaining bytes of the instruction without crossing page
857 * boundary if they are not in fetch_cache yet.
859 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
862 unsigned size, max_size;
863 unsigned long linear;
864 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
865 struct segmented_address addr = { .seg = VCPU_SREG_CS,
866 .ea = ctxt->eip + cur_size };
869 * We do not know exactly how many bytes will be needed, and
870 * __linearize is expensive, so fetch as much as possible. We
871 * just have to avoid going beyond the 15 byte limit, the end
872 * of the segment, or the end of the page.
874 * __linearize is called with size 0 so that it does not do any
875 * boundary check itself. Instead, we use max_size to check
878 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
880 if (unlikely(rc != X86EMUL_CONTINUE))
883 size = min_t(unsigned, 15UL ^ cur_size, max_size);
884 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
887 * One instruction can only straddle two pages,
888 * and one has been loaded at the beginning of
889 * x86_decode_insn. So, if not enough bytes
890 * still, we must have hit the 15-byte boundary.
892 if (unlikely(size < op_size))
893 return emulate_gp(ctxt, 0);
895 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
896 size, &ctxt->exception);
897 if (unlikely(rc != X86EMUL_CONTINUE))
899 ctxt->fetch.end += size;
900 return X86EMUL_CONTINUE;
903 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
906 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
908 if (unlikely(done_size < size))
909 return __do_insn_fetch_bytes(ctxt, size - done_size);
911 return X86EMUL_CONTINUE;
914 /* Fetch next part of the instruction being emulated. */
915 #define insn_fetch(_type, _ctxt) \
918 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
919 if (rc != X86EMUL_CONTINUE) \
921 ctxt->_eip += sizeof(_type); \
922 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
923 ctxt->fetch.ptr += sizeof(_type); \
927 #define insn_fetch_arr(_arr, _size, _ctxt) \
929 rc = do_insn_fetch_bytes(_ctxt, _size); \
930 if (rc != X86EMUL_CONTINUE) \
932 ctxt->_eip += (_size); \
933 memcpy(_arr, ctxt->fetch.ptr, _size); \
934 ctxt->fetch.ptr += (_size); \
938 * Given the 'reg' portion of a ModRM byte, and a register block, return a
939 * pointer into the block that addresses the relevant register.
940 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
942 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
946 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
948 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
949 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
951 p = reg_rmw(ctxt, modrm_reg);
955 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
956 struct segmented_address addr,
957 u16 *size, unsigned long *address, int op_bytes)
964 rc = segmented_read_std(ctxt, addr, size, 2);
965 if (rc != X86EMUL_CONTINUE)
968 rc = segmented_read_std(ctxt, addr, address, op_bytes);
982 FASTOP1SRC2(mul, mul_ex);
983 FASTOP1SRC2(imul, imul_ex);
984 FASTOP1SRC2EX(div, div_ex);
985 FASTOP1SRC2EX(idiv, idiv_ex);
1014 FASTOP2R(cmp, cmp_r);
1016 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1018 /* If src is zero, do not writeback, but update flags */
1019 if (ctxt->src.val == 0)
1020 ctxt->dst.type = OP_NONE;
1021 return fastop(ctxt, em_bsf);
1024 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1026 /* If src is zero, do not writeback, but update flags */
1027 if (ctxt->src.val == 0)
1028 ctxt->dst.type = OP_NONE;
1029 return fastop(ctxt, em_bsr);
1032 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1035 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1037 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1038 asm("push %[flags]; popf; " CALL_NOSPEC
1039 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1043 static void fetch_register_operand(struct operand *op)
1045 switch (op->bytes) {
1047 op->val = *(u8 *)op->addr.reg;
1050 op->val = *(u16 *)op->addr.reg;
1053 op->val = *(u32 *)op->addr.reg;
1056 op->val = *(u64 *)op->addr.reg;
1061 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1063 ctxt->ops->get_fpu(ctxt);
1065 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1066 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1067 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1068 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1069 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1070 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1071 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1072 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1073 #ifdef CONFIG_X86_64
1074 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1075 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1076 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1077 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1078 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1079 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1080 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1081 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1085 ctxt->ops->put_fpu(ctxt);
1088 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1091 ctxt->ops->get_fpu(ctxt);
1093 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1094 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1095 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1096 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1097 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1098 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1099 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1100 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1101 #ifdef CONFIG_X86_64
1102 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1103 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1104 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1105 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1106 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1107 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1108 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1109 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1113 ctxt->ops->put_fpu(ctxt);
1116 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1118 ctxt->ops->get_fpu(ctxt);
1120 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1121 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1122 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1123 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1124 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1125 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1126 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1127 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1130 ctxt->ops->put_fpu(ctxt);
1133 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1135 ctxt->ops->get_fpu(ctxt);
1137 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1138 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1139 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1140 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1141 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1142 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1143 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1144 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1147 ctxt->ops->put_fpu(ctxt);
1150 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1152 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1153 return emulate_nm(ctxt);
1155 ctxt->ops->get_fpu(ctxt);
1156 asm volatile("fninit");
1157 ctxt->ops->put_fpu(ctxt);
1158 return X86EMUL_CONTINUE;
1161 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1165 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1166 return emulate_nm(ctxt);
1168 ctxt->ops->get_fpu(ctxt);
1169 asm volatile("fnstcw %0": "+m"(fcw));
1170 ctxt->ops->put_fpu(ctxt);
1172 ctxt->dst.val = fcw;
1174 return X86EMUL_CONTINUE;
1177 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1181 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1182 return emulate_nm(ctxt);
1184 ctxt->ops->get_fpu(ctxt);
1185 asm volatile("fnstsw %0": "+m"(fsw));
1186 ctxt->ops->put_fpu(ctxt);
1188 ctxt->dst.val = fsw;
1190 return X86EMUL_CONTINUE;
1193 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1196 unsigned reg = ctxt->modrm_reg;
1198 if (!(ctxt->d & ModRM))
1199 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1201 if (ctxt->d & Sse) {
1205 read_sse_reg(ctxt, &op->vec_val, reg);
1208 if (ctxt->d & Mmx) {
1217 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1218 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1220 fetch_register_operand(op);
1221 op->orig_val = op->val;
1224 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1226 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1227 ctxt->modrm_seg = VCPU_SREG_SS;
1230 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1234 int index_reg, base_reg, scale;
1235 int rc = X86EMUL_CONTINUE;
1238 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1239 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1240 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1242 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1243 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1244 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1245 ctxt->modrm_seg = VCPU_SREG_DS;
1247 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1249 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1250 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1252 if (ctxt->d & Sse) {
1255 op->addr.xmm = ctxt->modrm_rm;
1256 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1259 if (ctxt->d & Mmx) {
1262 op->addr.mm = ctxt->modrm_rm & 7;
1265 fetch_register_operand(op);
1271 if (ctxt->ad_bytes == 2) {
1272 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1273 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1274 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1275 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1277 /* 16-bit ModR/M decode. */
1278 switch (ctxt->modrm_mod) {
1280 if (ctxt->modrm_rm == 6)
1281 modrm_ea += insn_fetch(u16, ctxt);
1284 modrm_ea += insn_fetch(s8, ctxt);
1287 modrm_ea += insn_fetch(u16, ctxt);
1290 switch (ctxt->modrm_rm) {
1292 modrm_ea += bx + si;
1295 modrm_ea += bx + di;
1298 modrm_ea += bp + si;
1301 modrm_ea += bp + di;
1310 if (ctxt->modrm_mod != 0)
1317 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1318 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1319 ctxt->modrm_seg = VCPU_SREG_SS;
1320 modrm_ea = (u16)modrm_ea;
1322 /* 32/64-bit ModR/M decode. */
1323 if ((ctxt->modrm_rm & 7) == 4) {
1324 sib = insn_fetch(u8, ctxt);
1325 index_reg |= (sib >> 3) & 7;
1326 base_reg |= sib & 7;
1329 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1330 modrm_ea += insn_fetch(s32, ctxt);
1332 modrm_ea += reg_read(ctxt, base_reg);
1333 adjust_modrm_seg(ctxt, base_reg);
1334 /* Increment ESP on POP [ESP] */
1335 if ((ctxt->d & IncSP) &&
1336 base_reg == VCPU_REGS_RSP)
1337 modrm_ea += ctxt->op_bytes;
1340 modrm_ea += reg_read(ctxt, index_reg) << scale;
1341 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1342 modrm_ea += insn_fetch(s32, ctxt);
1343 if (ctxt->mode == X86EMUL_MODE_PROT64)
1344 ctxt->rip_relative = 1;
1346 base_reg = ctxt->modrm_rm;
1347 modrm_ea += reg_read(ctxt, base_reg);
1348 adjust_modrm_seg(ctxt, base_reg);
1350 switch (ctxt->modrm_mod) {
1352 modrm_ea += insn_fetch(s8, ctxt);
1355 modrm_ea += insn_fetch(s32, ctxt);
1359 op->addr.mem.ea = modrm_ea;
1360 if (ctxt->ad_bytes != 8)
1361 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1367 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1370 int rc = X86EMUL_CONTINUE;
1373 switch (ctxt->ad_bytes) {
1375 op->addr.mem.ea = insn_fetch(u16, ctxt);
1378 op->addr.mem.ea = insn_fetch(u32, ctxt);
1381 op->addr.mem.ea = insn_fetch(u64, ctxt);
1388 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1392 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1393 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1395 if (ctxt->src.bytes == 2)
1396 sv = (s16)ctxt->src.val & (s16)mask;
1397 else if (ctxt->src.bytes == 4)
1398 sv = (s32)ctxt->src.val & (s32)mask;
1400 sv = (s64)ctxt->src.val & (s64)mask;
1402 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1403 ctxt->dst.addr.mem.ea + (sv >> 3));
1406 /* only subword offset */
1407 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1410 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1411 unsigned long addr, void *dest, unsigned size)
1414 struct read_cache *mc = &ctxt->mem_read;
1416 if (mc->pos < mc->end)
1419 WARN_ON((mc->end + size) >= sizeof(mc->data));
1421 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1423 if (rc != X86EMUL_CONTINUE)
1429 memcpy(dest, mc->data + mc->pos, size);
1431 return X86EMUL_CONTINUE;
1434 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1435 struct segmented_address addr,
1442 rc = linearize(ctxt, addr, size, false, &linear);
1443 if (rc != X86EMUL_CONTINUE)
1445 return read_emulated(ctxt, linear, data, size);
1448 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1449 struct segmented_address addr,
1456 rc = linearize(ctxt, addr, size, true, &linear);
1457 if (rc != X86EMUL_CONTINUE)
1459 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1463 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1464 struct segmented_address addr,
1465 const void *orig_data, const void *data,
1471 rc = linearize(ctxt, addr, size, true, &linear);
1472 if (rc != X86EMUL_CONTINUE)
1474 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1475 size, &ctxt->exception);
1478 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1479 unsigned int size, unsigned short port,
1482 struct read_cache *rc = &ctxt->io_read;
1484 if (rc->pos == rc->end) { /* refill pio read ahead */
1485 unsigned int in_page, n;
1486 unsigned int count = ctxt->rep_prefix ?
1487 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1488 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1489 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1490 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1491 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1494 rc->pos = rc->end = 0;
1495 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1500 if (ctxt->rep_prefix && (ctxt->d & String) &&
1501 !(ctxt->eflags & X86_EFLAGS_DF)) {
1502 ctxt->dst.data = rc->data + rc->pos;
1503 ctxt->dst.type = OP_MEM_STR;
1504 ctxt->dst.count = (rc->end - rc->pos) / size;
1507 memcpy(dest, rc->data + rc->pos, size);
1513 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1514 u16 index, struct desc_struct *desc)
1519 ctxt->ops->get_idt(ctxt, &dt);
1521 if (dt.size < index * 8 + 7)
1522 return emulate_gp(ctxt, index << 3 | 0x2);
1524 addr = dt.address + index * 8;
1525 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1528 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1529 u16 selector, struct desc_ptr *dt)
1531 const struct x86_emulate_ops *ops = ctxt->ops;
1534 if (selector & 1 << 2) {
1535 struct desc_struct desc;
1538 memset (dt, 0, sizeof *dt);
1539 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1543 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1544 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1546 ops->get_gdt(ctxt, dt);
1549 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1550 u16 selector, ulong *desc_addr_p)
1553 u16 index = selector >> 3;
1556 get_descriptor_table_ptr(ctxt, selector, &dt);
1558 if (dt.size < index * 8 + 7)
1559 return emulate_gp(ctxt, selector & 0xfffc);
1561 addr = dt.address + index * 8;
1563 #ifdef CONFIG_X86_64
1564 if (addr >> 32 != 0) {
1567 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1568 if (!(efer & EFER_LMA))
1573 *desc_addr_p = addr;
1574 return X86EMUL_CONTINUE;
1577 /* allowed just for 8 bytes segments */
1578 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1579 u16 selector, struct desc_struct *desc,
1584 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1585 if (rc != X86EMUL_CONTINUE)
1588 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1591 /* allowed just for 8 bytes segments */
1592 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1593 u16 selector, struct desc_struct *desc)
1598 rc = get_descriptor_ptr(ctxt, selector, &addr);
1599 if (rc != X86EMUL_CONTINUE)
1602 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1605 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1606 u16 selector, int seg, u8 cpl,
1607 enum x86_transfer_type transfer,
1608 struct desc_struct *desc)
1610 struct desc_struct seg_desc, old_desc;
1612 unsigned err_vec = GP_VECTOR;
1614 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1620 memset(&seg_desc, 0, sizeof seg_desc);
1622 if (ctxt->mode == X86EMUL_MODE_REAL) {
1623 /* set real mode segment descriptor (keep limit etc. for
1625 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1626 set_desc_base(&seg_desc, selector << 4);
1628 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1629 /* VM86 needs a clean new segment descriptor */
1630 set_desc_base(&seg_desc, selector << 4);
1631 set_desc_limit(&seg_desc, 0xffff);
1641 /* TR should be in GDT only */
1642 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1645 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1646 if (null_selector) {
1647 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1650 if (seg == VCPU_SREG_SS) {
1651 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1655 * ctxt->ops->set_segment expects the CPL to be in
1656 * SS.DPL, so fake an expand-up 32-bit data segment.
1666 /* Skip all following checks */
1670 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1671 if (ret != X86EMUL_CONTINUE)
1674 err_code = selector & 0xfffc;
1675 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1678 /* can't load system descriptor into segment selector */
1679 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1680 if (transfer == X86_TRANSFER_CALL_JMP)
1681 return X86EMUL_UNHANDLEABLE;
1690 * segment is not a writable data segment or segment
1691 * selector's RPL != CPL or segment selector's RPL != CPL
1693 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1697 if (!(seg_desc.type & 8))
1700 if (seg_desc.type & 4) {
1706 if (rpl > cpl || dpl != cpl)
1709 /* in long-mode d/b must be clear if l is set */
1710 if (seg_desc.d && seg_desc.l) {
1713 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1714 if (efer & EFER_LMA)
1718 /* CS(RPL) <- CPL */
1719 selector = (selector & 0xfffc) | cpl;
1722 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1725 case VCPU_SREG_LDTR:
1726 if (seg_desc.s || seg_desc.type != 2)
1729 default: /* DS, ES, FS, or GS */
1731 * segment is not a data or readable code segment or
1732 * ((segment is a data or nonconforming code segment)
1733 * and (both RPL and CPL > DPL))
1735 if ((seg_desc.type & 0xa) == 0x8 ||
1736 (((seg_desc.type & 0xc) != 0xc) &&
1737 (rpl > dpl && cpl > dpl)))
1743 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1748 /* mark segment as accessed */
1749 if (!(seg_desc.type & 1)) {
1751 ret = write_segment_descriptor(ctxt, selector,
1753 if (ret != X86EMUL_CONTINUE)
1756 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1757 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1758 if (ret != X86EMUL_CONTINUE)
1760 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1761 ((u64)base3 << 32), ctxt))
1762 return emulate_gp(ctxt, err_code);
1765 if (seg == VCPU_SREG_TR) {
1766 old_desc = seg_desc;
1767 seg_desc.type |= 2; /* busy */
1768 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1769 sizeof(seg_desc), &ctxt->exception);
1770 if (ret != X86EMUL_CONTINUE)
1774 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1777 return X86EMUL_CONTINUE;
1779 return emulate_exception(ctxt, err_vec, err_code, true);
1782 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1783 u16 selector, int seg)
1785 u8 cpl = ctxt->ops->cpl(ctxt);
1788 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1789 * they can load it at CPL<3 (Intel's manual says only LSS can,
1792 * However, the Intel manual says that putting IST=1/DPL=3 in
1793 * an interrupt gate will result in SS=3 (the AMD manual instead
1794 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1795 * and only forbid it here.
1797 if (seg == VCPU_SREG_SS && selector == 3 &&
1798 ctxt->mode == X86EMUL_MODE_PROT64)
1799 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1801 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1802 X86_TRANSFER_NONE, NULL);
1805 static void write_register_operand(struct operand *op)
1807 return assign_register(op->addr.reg, op->val, op->bytes);
1810 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1814 write_register_operand(op);
1817 if (ctxt->lock_prefix)
1818 return segmented_cmpxchg(ctxt,
1824 return segmented_write(ctxt,
1830 return segmented_write(ctxt,
1833 op->bytes * op->count);
1836 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1839 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1847 return X86EMUL_CONTINUE;
1850 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1852 struct segmented_address addr;
1854 rsp_increment(ctxt, -bytes);
1855 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1856 addr.seg = VCPU_SREG_SS;
1858 return segmented_write(ctxt, addr, data, bytes);
1861 static int em_push(struct x86_emulate_ctxt *ctxt)
1863 /* Disable writeback. */
1864 ctxt->dst.type = OP_NONE;
1865 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1868 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1869 void *dest, int len)
1872 struct segmented_address addr;
1874 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1875 addr.seg = VCPU_SREG_SS;
1876 rc = segmented_read(ctxt, addr, dest, len);
1877 if (rc != X86EMUL_CONTINUE)
1880 rsp_increment(ctxt, len);
1884 static int em_pop(struct x86_emulate_ctxt *ctxt)
1886 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1889 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1890 void *dest, int len)
1893 unsigned long val, change_mask;
1894 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1895 int cpl = ctxt->ops->cpl(ctxt);
1897 rc = emulate_pop(ctxt, &val, len);
1898 if (rc != X86EMUL_CONTINUE)
1901 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1902 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1903 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1904 X86_EFLAGS_AC | X86_EFLAGS_ID;
1906 switch(ctxt->mode) {
1907 case X86EMUL_MODE_PROT64:
1908 case X86EMUL_MODE_PROT32:
1909 case X86EMUL_MODE_PROT16:
1911 change_mask |= X86_EFLAGS_IOPL;
1913 change_mask |= X86_EFLAGS_IF;
1915 case X86EMUL_MODE_VM86:
1917 return emulate_gp(ctxt, 0);
1918 change_mask |= X86_EFLAGS_IF;
1920 default: /* real mode */
1921 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1925 *(unsigned long *)dest =
1926 (ctxt->eflags & ~change_mask) | (val & change_mask);
1931 static int em_popf(struct x86_emulate_ctxt *ctxt)
1933 ctxt->dst.type = OP_REG;
1934 ctxt->dst.addr.reg = &ctxt->eflags;
1935 ctxt->dst.bytes = ctxt->op_bytes;
1936 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1939 static int em_enter(struct x86_emulate_ctxt *ctxt)
1942 unsigned frame_size = ctxt->src.val;
1943 unsigned nesting_level = ctxt->src2.val & 31;
1947 return X86EMUL_UNHANDLEABLE;
1949 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1950 rc = push(ctxt, &rbp, stack_size(ctxt));
1951 if (rc != X86EMUL_CONTINUE)
1953 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1955 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1956 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1958 return X86EMUL_CONTINUE;
1961 static int em_leave(struct x86_emulate_ctxt *ctxt)
1963 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1965 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1968 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1970 int seg = ctxt->src2.val;
1972 ctxt->src.val = get_segment_selector(ctxt, seg);
1973 if (ctxt->op_bytes == 4) {
1974 rsp_increment(ctxt, -2);
1978 return em_push(ctxt);
1981 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1983 int seg = ctxt->src2.val;
1984 unsigned long selector;
1987 rc = emulate_pop(ctxt, &selector, 2);
1988 if (rc != X86EMUL_CONTINUE)
1991 if (ctxt->modrm_reg == VCPU_SREG_SS)
1992 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1993 if (ctxt->op_bytes > 2)
1994 rsp_increment(ctxt, ctxt->op_bytes - 2);
1996 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2000 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2002 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2003 int rc = X86EMUL_CONTINUE;
2004 int reg = VCPU_REGS_RAX;
2006 while (reg <= VCPU_REGS_RDI) {
2007 (reg == VCPU_REGS_RSP) ?
2008 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2011 if (rc != X86EMUL_CONTINUE)
2020 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2022 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2023 return em_push(ctxt);
2026 static int em_popa(struct x86_emulate_ctxt *ctxt)
2028 int rc = X86EMUL_CONTINUE;
2029 int reg = VCPU_REGS_RDI;
2032 while (reg >= VCPU_REGS_RAX) {
2033 if (reg == VCPU_REGS_RSP) {
2034 rsp_increment(ctxt, ctxt->op_bytes);
2038 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2039 if (rc != X86EMUL_CONTINUE)
2041 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2047 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2049 const struct x86_emulate_ops *ops = ctxt->ops;
2056 /* TODO: Add limit checks */
2057 ctxt->src.val = ctxt->eflags;
2059 if (rc != X86EMUL_CONTINUE)
2062 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2064 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2066 if (rc != X86EMUL_CONTINUE)
2069 ctxt->src.val = ctxt->_eip;
2071 if (rc != X86EMUL_CONTINUE)
2074 ops->get_idt(ctxt, &dt);
2076 eip_addr = dt.address + (irq << 2);
2077 cs_addr = dt.address + (irq << 2) + 2;
2079 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2080 if (rc != X86EMUL_CONTINUE)
2083 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2084 if (rc != X86EMUL_CONTINUE)
2087 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2088 if (rc != X86EMUL_CONTINUE)
2096 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2100 invalidate_registers(ctxt);
2101 rc = __emulate_int_real(ctxt, irq);
2102 if (rc == X86EMUL_CONTINUE)
2103 writeback_registers(ctxt);
2107 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2109 switch(ctxt->mode) {
2110 case X86EMUL_MODE_REAL:
2111 return __emulate_int_real(ctxt, irq);
2112 case X86EMUL_MODE_VM86:
2113 case X86EMUL_MODE_PROT16:
2114 case X86EMUL_MODE_PROT32:
2115 case X86EMUL_MODE_PROT64:
2117 /* Protected mode interrupts unimplemented yet */
2118 return X86EMUL_UNHANDLEABLE;
2122 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2124 int rc = X86EMUL_CONTINUE;
2125 unsigned long temp_eip = 0;
2126 unsigned long temp_eflags = 0;
2127 unsigned long cs = 0;
2128 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2129 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2130 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2131 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2132 X86_EFLAGS_AC | X86_EFLAGS_ID |
2134 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2137 /* TODO: Add stack limit check */
2139 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2141 if (rc != X86EMUL_CONTINUE)
2144 if (temp_eip & ~0xffff)
2145 return emulate_gp(ctxt, 0);
2147 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2149 if (rc != X86EMUL_CONTINUE)
2152 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2154 if (rc != X86EMUL_CONTINUE)
2157 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2159 if (rc != X86EMUL_CONTINUE)
2162 ctxt->_eip = temp_eip;
2164 if (ctxt->op_bytes == 4)
2165 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2166 else if (ctxt->op_bytes == 2) {
2167 ctxt->eflags &= ~0xffff;
2168 ctxt->eflags |= temp_eflags;
2171 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2172 ctxt->eflags |= X86_EFLAGS_FIXED;
2173 ctxt->ops->set_nmi_mask(ctxt, false);
2178 static int em_iret(struct x86_emulate_ctxt *ctxt)
2180 switch(ctxt->mode) {
2181 case X86EMUL_MODE_REAL:
2182 return emulate_iret_real(ctxt);
2183 case X86EMUL_MODE_VM86:
2184 case X86EMUL_MODE_PROT16:
2185 case X86EMUL_MODE_PROT32:
2186 case X86EMUL_MODE_PROT64:
2188 /* iret from protected mode unimplemented yet */
2189 return X86EMUL_UNHANDLEABLE;
2193 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2197 struct desc_struct new_desc;
2198 u8 cpl = ctxt->ops->cpl(ctxt);
2200 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2202 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2203 X86_TRANSFER_CALL_JMP,
2205 if (rc != X86EMUL_CONTINUE)
2208 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2209 /* Error handling is not implemented. */
2210 if (rc != X86EMUL_CONTINUE)
2211 return X86EMUL_UNHANDLEABLE;
2216 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2218 return assign_eip_near(ctxt, ctxt->src.val);
2221 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2226 old_eip = ctxt->_eip;
2227 rc = assign_eip_near(ctxt, ctxt->src.val);
2228 if (rc != X86EMUL_CONTINUE)
2230 ctxt->src.val = old_eip;
2235 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2237 u64 old = ctxt->dst.orig_val64;
2239 if (ctxt->dst.bytes == 16)
2240 return X86EMUL_UNHANDLEABLE;
2242 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2243 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2244 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2245 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2246 ctxt->eflags &= ~X86_EFLAGS_ZF;
2248 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2249 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2251 ctxt->eflags |= X86_EFLAGS_ZF;
2253 return X86EMUL_CONTINUE;
2256 static int em_ret(struct x86_emulate_ctxt *ctxt)
2261 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2262 if (rc != X86EMUL_CONTINUE)
2265 return assign_eip_near(ctxt, eip);
2268 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2271 unsigned long eip, cs;
2272 int cpl = ctxt->ops->cpl(ctxt);
2273 struct desc_struct new_desc;
2275 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2276 if (rc != X86EMUL_CONTINUE)
2278 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2279 if (rc != X86EMUL_CONTINUE)
2281 /* Outer-privilege level return is not implemented */
2282 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2283 return X86EMUL_UNHANDLEABLE;
2284 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2287 if (rc != X86EMUL_CONTINUE)
2289 rc = assign_eip_far(ctxt, eip, &new_desc);
2290 /* Error handling is not implemented. */
2291 if (rc != X86EMUL_CONTINUE)
2292 return X86EMUL_UNHANDLEABLE;
2297 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2301 rc = em_ret_far(ctxt);
2302 if (rc != X86EMUL_CONTINUE)
2304 rsp_increment(ctxt, ctxt->src.val);
2305 return X86EMUL_CONTINUE;
2308 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2310 /* Save real source value, then compare EAX against destination. */
2311 ctxt->dst.orig_val = ctxt->dst.val;
2312 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2313 ctxt->src.orig_val = ctxt->src.val;
2314 ctxt->src.val = ctxt->dst.orig_val;
2315 fastop(ctxt, em_cmp);
2317 if (ctxt->eflags & X86_EFLAGS_ZF) {
2318 /* Success: write back to memory; no update of EAX */
2319 ctxt->src.type = OP_NONE;
2320 ctxt->dst.val = ctxt->src.orig_val;
2322 /* Failure: write the value we saw to EAX. */
2323 ctxt->src.type = OP_REG;
2324 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2325 ctxt->src.val = ctxt->dst.orig_val;
2326 /* Create write-cycle to dest by writing the same value */
2327 ctxt->dst.val = ctxt->dst.orig_val;
2329 return X86EMUL_CONTINUE;
2332 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2334 int seg = ctxt->src2.val;
2338 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2340 rc = load_segment_descriptor(ctxt, sel, seg);
2341 if (rc != X86EMUL_CONTINUE)
2344 ctxt->dst.val = ctxt->src.val;
2348 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2350 u32 eax, ebx, ecx, edx;
2354 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2355 return edx & bit(X86_FEATURE_LM);
2358 #define GET_SMSTATE(type, smbase, offset) \
2361 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2363 if (r != X86EMUL_CONTINUE) \
2364 return X86EMUL_UNHANDLEABLE; \
2368 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2370 desc->g = (flags >> 23) & 1;
2371 desc->d = (flags >> 22) & 1;
2372 desc->l = (flags >> 21) & 1;
2373 desc->avl = (flags >> 20) & 1;
2374 desc->p = (flags >> 15) & 1;
2375 desc->dpl = (flags >> 13) & 3;
2376 desc->s = (flags >> 12) & 1;
2377 desc->type = (flags >> 8) & 15;
2380 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2382 struct desc_struct desc;
2386 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2389 offset = 0x7f84 + n * 12;
2391 offset = 0x7f2c + (n - 3) * 12;
2393 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2394 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2395 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2396 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2397 return X86EMUL_CONTINUE;
2400 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2402 struct desc_struct desc;
2407 offset = 0x7e00 + n * 16;
2409 selector = GET_SMSTATE(u16, smbase, offset);
2410 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2411 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2412 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2413 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2415 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2416 return X86EMUL_CONTINUE;
2419 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2420 u64 cr0, u64 cr3, u64 cr4)
2425 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2427 if (cr4 & X86_CR4_PCIDE) {
2432 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2434 return X86EMUL_UNHANDLEABLE;
2437 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2438 * Then enable protected mode. However, PCID cannot be enabled
2439 * if EFER.LMA=0, so set it separately.
2441 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2443 return X86EMUL_UNHANDLEABLE;
2445 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2447 return X86EMUL_UNHANDLEABLE;
2449 if (cr4 & X86_CR4_PCIDE) {
2450 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2452 return X86EMUL_UNHANDLEABLE;
2454 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2456 return X86EMUL_UNHANDLEABLE;
2461 return X86EMUL_CONTINUE;
2464 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2466 struct desc_struct desc;
2469 u32 val, cr0, cr3, cr4;
2472 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2473 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2474 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2475 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2477 for (i = 0; i < 8; i++)
2478 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2480 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2481 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2482 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2483 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2485 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2486 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2487 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2488 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2489 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2491 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2492 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2493 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2494 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2495 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2497 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2498 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2499 ctxt->ops->set_gdt(ctxt, &dt);
2501 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2502 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2503 ctxt->ops->set_idt(ctxt, &dt);
2505 for (i = 0; i < 6; i++) {
2506 int r = rsm_load_seg_32(ctxt, smbase, i);
2507 if (r != X86EMUL_CONTINUE)
2511 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2513 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2515 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2518 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2520 struct desc_struct desc;
2522 u64 val, cr0, cr3, cr4;
2527 for (i = 0; i < 16; i++)
2528 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2530 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2531 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2533 val = GET_SMSTATE(u32, smbase, 0x7f68);
2534 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2535 val = GET_SMSTATE(u32, smbase, 0x7f60);
2536 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2538 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2539 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2540 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2541 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2542 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2543 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2545 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2546 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2547 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2548 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2549 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2550 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2552 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2553 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2554 ctxt->ops->set_idt(ctxt, &dt);
2556 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2557 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2558 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2559 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2560 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2561 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2563 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2564 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2565 ctxt->ops->set_gdt(ctxt, &dt);
2567 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2568 if (r != X86EMUL_CONTINUE)
2571 for (i = 0; i < 6; i++) {
2572 r = rsm_load_seg_64(ctxt, smbase, i);
2573 if (r != X86EMUL_CONTINUE)
2577 return X86EMUL_CONTINUE;
2580 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2582 unsigned long cr0, cr4, efer;
2586 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2587 return emulate_ud(ctxt);
2590 * Get back to real mode, to prepare a safe state in which to load
2591 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2592 * supports long mode.
2594 if (emulator_has_longmode(ctxt)) {
2595 struct desc_struct cs_desc;
2597 /* Zero CR4.PCIDE before CR0.PG. */
2598 cr4 = ctxt->ops->get_cr(ctxt, 4);
2599 if (cr4 & X86_CR4_PCIDE)
2600 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2602 /* A 32-bit code segment is required to clear EFER.LMA. */
2603 memset(&cs_desc, 0, sizeof(cs_desc));
2605 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2606 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2609 /* For the 64-bit case, this will clear EFER.LMA. */
2610 cr0 = ctxt->ops->get_cr(ctxt, 0);
2611 if (cr0 & X86_CR0_PE)
2612 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2614 if (emulator_has_longmode(ctxt)) {
2615 /* Clear CR4.PAE before clearing EFER.LME. */
2616 cr4 = ctxt->ops->get_cr(ctxt, 4);
2617 if (cr4 & X86_CR4_PAE)
2618 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2620 /* And finally go back to 32-bit mode. */
2622 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2625 smbase = ctxt->ops->get_smbase(ctxt);
2626 if (emulator_has_longmode(ctxt))
2627 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2629 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2631 if (ret != X86EMUL_CONTINUE) {
2632 /* FIXME: should triple fault */
2633 return X86EMUL_UNHANDLEABLE;
2636 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2637 ctxt->ops->set_nmi_mask(ctxt, false);
2639 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2640 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2641 return X86EMUL_CONTINUE;
2645 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2646 struct desc_struct *cs, struct desc_struct *ss)
2648 cs->l = 0; /* will be adjusted later */
2649 set_desc_base(cs, 0); /* flat segment */
2650 cs->g = 1; /* 4kb granularity */
2651 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2652 cs->type = 0x0b; /* Read, Execute, Accessed */
2654 cs->dpl = 0; /* will be adjusted later */
2659 set_desc_base(ss, 0); /* flat segment */
2660 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2661 ss->g = 1; /* 4kb granularity */
2663 ss->type = 0x03; /* Read/Write, Accessed */
2664 ss->d = 1; /* 32bit stack segment */
2671 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2673 u32 eax, ebx, ecx, edx;
2676 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2677 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2678 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2679 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2682 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2684 const struct x86_emulate_ops *ops = ctxt->ops;
2685 u32 eax, ebx, ecx, edx;
2688 * syscall should always be enabled in longmode - so only become
2689 * vendor specific (cpuid) if other modes are active...
2691 if (ctxt->mode == X86EMUL_MODE_PROT64)
2696 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2698 * Intel ("GenuineIntel")
2699 * remark: Intel CPUs only support "syscall" in 64bit
2700 * longmode. Also an 64bit guest with a
2701 * 32bit compat-app running will #UD !! While this
2702 * behaviour can be fixed (by emulating) into AMD
2703 * response - CPUs of AMD can't behave like Intel.
2705 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2706 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2707 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2710 /* AMD ("AuthenticAMD") */
2711 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2712 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2713 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2716 /* AMD ("AMDisbetter!") */
2717 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2718 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2719 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2722 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2726 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2728 const struct x86_emulate_ops *ops = ctxt->ops;
2729 struct desc_struct cs, ss;
2734 /* syscall is not available in real mode */
2735 if (ctxt->mode == X86EMUL_MODE_REAL ||
2736 ctxt->mode == X86EMUL_MODE_VM86)
2737 return emulate_ud(ctxt);
2739 if (!(em_syscall_is_enabled(ctxt)))
2740 return emulate_ud(ctxt);
2742 ops->get_msr(ctxt, MSR_EFER, &efer);
2743 setup_syscalls_segments(ctxt, &cs, &ss);
2745 if (!(efer & EFER_SCE))
2746 return emulate_ud(ctxt);
2748 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2750 cs_sel = (u16)(msr_data & 0xfffc);
2751 ss_sel = (u16)(msr_data + 8);
2753 if (efer & EFER_LMA) {
2757 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2758 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2760 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2761 if (efer & EFER_LMA) {
2762 #ifdef CONFIG_X86_64
2763 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2766 ctxt->mode == X86EMUL_MODE_PROT64 ?
2767 MSR_LSTAR : MSR_CSTAR, &msr_data);
2768 ctxt->_eip = msr_data;
2770 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2771 ctxt->eflags &= ~msr_data;
2772 ctxt->eflags |= X86_EFLAGS_FIXED;
2776 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2777 ctxt->_eip = (u32)msr_data;
2779 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2782 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2783 return X86EMUL_CONTINUE;
2786 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2788 const struct x86_emulate_ops *ops = ctxt->ops;
2789 struct desc_struct cs, ss;
2794 ops->get_msr(ctxt, MSR_EFER, &efer);
2795 /* inject #GP if in real mode */
2796 if (ctxt->mode == X86EMUL_MODE_REAL)
2797 return emulate_gp(ctxt, 0);
2800 * Not recognized on AMD in compat mode (but is recognized in legacy
2803 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2804 && !vendor_intel(ctxt))
2805 return emulate_ud(ctxt);
2807 /* sysenter/sysexit have not been tested in 64bit mode. */
2808 if (ctxt->mode == X86EMUL_MODE_PROT64)
2809 return X86EMUL_UNHANDLEABLE;
2811 setup_syscalls_segments(ctxt, &cs, &ss);
2813 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2814 if ((msr_data & 0xfffc) == 0x0)
2815 return emulate_gp(ctxt, 0);
2817 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2818 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2819 ss_sel = cs_sel + 8;
2820 if (efer & EFER_LMA) {
2825 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2826 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2828 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2829 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2831 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2832 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2835 return X86EMUL_CONTINUE;
2838 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2840 const struct x86_emulate_ops *ops = ctxt->ops;
2841 struct desc_struct cs, ss;
2842 u64 msr_data, rcx, rdx;
2844 u16 cs_sel = 0, ss_sel = 0;
2846 /* inject #GP if in real mode or Virtual 8086 mode */
2847 if (ctxt->mode == X86EMUL_MODE_REAL ||
2848 ctxt->mode == X86EMUL_MODE_VM86)
2849 return emulate_gp(ctxt, 0);
2851 setup_syscalls_segments(ctxt, &cs, &ss);
2853 if ((ctxt->rex_prefix & 0x8) != 0x0)
2854 usermode = X86EMUL_MODE_PROT64;
2856 usermode = X86EMUL_MODE_PROT32;
2858 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2859 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2863 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2865 case X86EMUL_MODE_PROT32:
2866 cs_sel = (u16)(msr_data + 16);
2867 if ((msr_data & 0xfffc) == 0x0)
2868 return emulate_gp(ctxt, 0);
2869 ss_sel = (u16)(msr_data + 24);
2873 case X86EMUL_MODE_PROT64:
2874 cs_sel = (u16)(msr_data + 32);
2875 if (msr_data == 0x0)
2876 return emulate_gp(ctxt, 0);
2877 ss_sel = cs_sel + 8;
2880 if (emul_is_noncanonical_address(rcx, ctxt) ||
2881 emul_is_noncanonical_address(rdx, ctxt))
2882 return emulate_gp(ctxt, 0);
2885 cs_sel |= SEGMENT_RPL_MASK;
2886 ss_sel |= SEGMENT_RPL_MASK;
2888 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2889 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2892 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2894 return X86EMUL_CONTINUE;
2897 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2900 if (ctxt->mode == X86EMUL_MODE_REAL)
2902 if (ctxt->mode == X86EMUL_MODE_VM86)
2904 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2905 return ctxt->ops->cpl(ctxt) > iopl;
2908 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2911 const struct x86_emulate_ops *ops = ctxt->ops;
2912 struct desc_struct tr_seg;
2915 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2916 unsigned mask = (1 << len) - 1;
2919 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2922 if (desc_limit_scaled(&tr_seg) < 103)
2924 base = get_desc_base(&tr_seg);
2925 #ifdef CONFIG_X86_64
2926 base |= ((u64)base3) << 32;
2928 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2929 if (r != X86EMUL_CONTINUE)
2931 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2933 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2934 if (r != X86EMUL_CONTINUE)
2936 if ((perm >> bit_idx) & mask)
2941 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2947 if (emulator_bad_iopl(ctxt))
2948 if (!emulator_io_port_access_allowed(ctxt, port, len))
2951 ctxt->perm_ok = true;
2956 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2959 * Intel CPUs mask the counter and pointers in quite strange
2960 * manner when ECX is zero due to REP-string optimizations.
2962 #ifdef CONFIG_X86_64
2963 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2966 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2969 case 0xa4: /* movsb */
2970 case 0xa5: /* movsd/w */
2971 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2973 case 0xaa: /* stosb */
2974 case 0xab: /* stosd/w */
2975 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2980 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2981 struct tss_segment_16 *tss)
2983 tss->ip = ctxt->_eip;
2984 tss->flag = ctxt->eflags;
2985 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2986 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2987 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2988 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2989 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2990 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2991 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2992 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2994 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2995 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2996 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2997 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2998 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3001 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3002 struct tss_segment_16 *tss)
3007 ctxt->_eip = tss->ip;
3008 ctxt->eflags = tss->flag | 2;
3009 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3010 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3011 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3012 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3013 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3014 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3015 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3016 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3019 * SDM says that segment selectors are loaded before segment
3022 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3023 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3024 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3025 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3026 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3031 * Now load segment descriptors. If fault happens at this stage
3032 * it is handled in a context of new task
3034 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3035 X86_TRANSFER_TASK_SWITCH, NULL);
3036 if (ret != X86EMUL_CONTINUE)
3038 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3039 X86_TRANSFER_TASK_SWITCH, NULL);
3040 if (ret != X86EMUL_CONTINUE)
3042 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3043 X86_TRANSFER_TASK_SWITCH, NULL);
3044 if (ret != X86EMUL_CONTINUE)
3046 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3047 X86_TRANSFER_TASK_SWITCH, NULL);
3048 if (ret != X86EMUL_CONTINUE)
3050 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3051 X86_TRANSFER_TASK_SWITCH, NULL);
3052 if (ret != X86EMUL_CONTINUE)
3055 return X86EMUL_CONTINUE;
3058 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3059 u16 tss_selector, u16 old_tss_sel,
3060 ulong old_tss_base, struct desc_struct *new_desc)
3062 struct tss_segment_16 tss_seg;
3064 u32 new_tss_base = get_desc_base(new_desc);
3066 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3067 if (ret != X86EMUL_CONTINUE)
3070 save_state_to_tss16(ctxt, &tss_seg);
3072 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3073 if (ret != X86EMUL_CONTINUE)
3076 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3077 if (ret != X86EMUL_CONTINUE)
3080 if (old_tss_sel != 0xffff) {
3081 tss_seg.prev_task_link = old_tss_sel;
3083 ret = linear_write_system(ctxt, new_tss_base,
3084 &tss_seg.prev_task_link,
3085 sizeof tss_seg.prev_task_link);
3086 if (ret != X86EMUL_CONTINUE)
3090 return load_state_from_tss16(ctxt, &tss_seg);
3093 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3094 struct tss_segment_32 *tss)
3096 /* CR3 and ldt selector are not saved intentionally */
3097 tss->eip = ctxt->_eip;
3098 tss->eflags = ctxt->eflags;
3099 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3100 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3101 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3102 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3103 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3104 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3105 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3106 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3108 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3109 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3110 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3111 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3112 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3113 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3116 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3117 struct tss_segment_32 *tss)
3122 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3123 return emulate_gp(ctxt, 0);
3124 ctxt->_eip = tss->eip;
3125 ctxt->eflags = tss->eflags | 2;
3127 /* General purpose registers */
3128 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3129 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3130 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3131 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3132 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3133 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3134 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3135 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3138 * SDM says that segment selectors are loaded before segment
3139 * descriptors. This is important because CPL checks will
3142 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3143 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3144 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3145 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3146 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3147 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3148 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3151 * If we're switching between Protected Mode and VM86, we need to make
3152 * sure to update the mode before loading the segment descriptors so
3153 * that the selectors are interpreted correctly.
3155 if (ctxt->eflags & X86_EFLAGS_VM) {
3156 ctxt->mode = X86EMUL_MODE_VM86;
3159 ctxt->mode = X86EMUL_MODE_PROT32;
3164 * Now load segment descriptors. If fault happenes at this stage
3165 * it is handled in a context of new task
3167 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3168 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3169 if (ret != X86EMUL_CONTINUE)
3171 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3172 X86_TRANSFER_TASK_SWITCH, NULL);
3173 if (ret != X86EMUL_CONTINUE)
3175 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3176 X86_TRANSFER_TASK_SWITCH, NULL);
3177 if (ret != X86EMUL_CONTINUE)
3179 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3180 X86_TRANSFER_TASK_SWITCH, NULL);
3181 if (ret != X86EMUL_CONTINUE)
3183 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3184 X86_TRANSFER_TASK_SWITCH, NULL);
3185 if (ret != X86EMUL_CONTINUE)
3187 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3188 X86_TRANSFER_TASK_SWITCH, NULL);
3189 if (ret != X86EMUL_CONTINUE)
3191 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3192 X86_TRANSFER_TASK_SWITCH, NULL);
3197 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3198 u16 tss_selector, u16 old_tss_sel,
3199 ulong old_tss_base, struct desc_struct *new_desc)
3201 struct tss_segment_32 tss_seg;
3203 u32 new_tss_base = get_desc_base(new_desc);
3204 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3205 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3207 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3208 if (ret != X86EMUL_CONTINUE)
3211 save_state_to_tss32(ctxt, &tss_seg);
3213 /* Only GP registers and segment selectors are saved */
3214 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3215 ldt_sel_offset - eip_offset);
3216 if (ret != X86EMUL_CONTINUE)
3219 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3220 if (ret != X86EMUL_CONTINUE)
3223 if (old_tss_sel != 0xffff) {
3224 tss_seg.prev_task_link = old_tss_sel;
3226 ret = linear_write_system(ctxt, new_tss_base,
3227 &tss_seg.prev_task_link,
3228 sizeof tss_seg.prev_task_link);
3229 if (ret != X86EMUL_CONTINUE)
3233 return load_state_from_tss32(ctxt, &tss_seg);
3236 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3237 u16 tss_selector, int idt_index, int reason,
3238 bool has_error_code, u32 error_code)
3240 const struct x86_emulate_ops *ops = ctxt->ops;
3241 struct desc_struct curr_tss_desc, next_tss_desc;
3243 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3244 ulong old_tss_base =
3245 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3247 ulong desc_addr, dr7;
3249 /* FIXME: old_tss_base == ~0 ? */
3251 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3252 if (ret != X86EMUL_CONTINUE)
3254 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3255 if (ret != X86EMUL_CONTINUE)
3258 /* FIXME: check that next_tss_desc is tss */
3261 * Check privileges. The three cases are task switch caused by...
3263 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3264 * 2. Exception/IRQ/iret: No check is performed
3265 * 3. jmp/call to TSS/task-gate: No check is performed since the
3266 * hardware checks it before exiting.
3268 if (reason == TASK_SWITCH_GATE) {
3269 if (idt_index != -1) {
3270 /* Software interrupts */
3271 struct desc_struct task_gate_desc;
3274 ret = read_interrupt_descriptor(ctxt, idt_index,
3276 if (ret != X86EMUL_CONTINUE)
3279 dpl = task_gate_desc.dpl;
3280 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3281 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3285 desc_limit = desc_limit_scaled(&next_tss_desc);
3286 if (!next_tss_desc.p ||
3287 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3288 desc_limit < 0x2b)) {
3289 return emulate_ts(ctxt, tss_selector & 0xfffc);
3292 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3293 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3294 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3297 if (reason == TASK_SWITCH_IRET)
3298 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3300 /* set back link to prev task only if NT bit is set in eflags
3301 note that old_tss_sel is not used after this point */
3302 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3303 old_tss_sel = 0xffff;
3305 if (next_tss_desc.type & 8)
3306 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3307 old_tss_base, &next_tss_desc);
3309 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3310 old_tss_base, &next_tss_desc);
3311 if (ret != X86EMUL_CONTINUE)
3314 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3315 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3317 if (reason != TASK_SWITCH_IRET) {
3318 next_tss_desc.type |= (1 << 1); /* set busy flag */
3319 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3322 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3323 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3325 if (has_error_code) {
3326 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3327 ctxt->lock_prefix = 0;
3328 ctxt->src.val = (unsigned long) error_code;
3329 ret = em_push(ctxt);
3332 ops->get_dr(ctxt, 7, &dr7);
3333 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3338 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3339 u16 tss_selector, int idt_index, int reason,
3340 bool has_error_code, u32 error_code)
3344 invalidate_registers(ctxt);
3345 ctxt->_eip = ctxt->eip;
3346 ctxt->dst.type = OP_NONE;
3348 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3349 has_error_code, error_code);
3351 if (rc == X86EMUL_CONTINUE) {
3352 ctxt->eip = ctxt->_eip;
3353 writeback_registers(ctxt);
3356 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3359 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3362 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3364 register_address_increment(ctxt, reg, df * op->bytes);
3365 op->addr.mem.ea = register_address(ctxt, reg);
3368 static int em_das(struct x86_emulate_ctxt *ctxt)
3371 bool af, cf, old_cf;
3373 cf = ctxt->eflags & X86_EFLAGS_CF;
3379 af = ctxt->eflags & X86_EFLAGS_AF;
3380 if ((al & 0x0f) > 9 || af) {
3382 cf = old_cf | (al >= 250);
3387 if (old_al > 0x99 || old_cf) {
3393 /* Set PF, ZF, SF */
3394 ctxt->src.type = OP_IMM;
3396 ctxt->src.bytes = 1;
3397 fastop(ctxt, em_or);
3398 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3400 ctxt->eflags |= X86_EFLAGS_CF;
3402 ctxt->eflags |= X86_EFLAGS_AF;
3403 return X86EMUL_CONTINUE;
3406 static int em_aam(struct x86_emulate_ctxt *ctxt)
3410 if (ctxt->src.val == 0)
3411 return emulate_de(ctxt);
3413 al = ctxt->dst.val & 0xff;
3414 ah = al / ctxt->src.val;
3415 al %= ctxt->src.val;
3417 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3419 /* Set PF, ZF, SF */
3420 ctxt->src.type = OP_IMM;
3422 ctxt->src.bytes = 1;
3423 fastop(ctxt, em_or);
3425 return X86EMUL_CONTINUE;
3428 static int em_aad(struct x86_emulate_ctxt *ctxt)
3430 u8 al = ctxt->dst.val & 0xff;
3431 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3433 al = (al + (ah * ctxt->src.val)) & 0xff;
3435 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3437 /* Set PF, ZF, SF */
3438 ctxt->src.type = OP_IMM;
3440 ctxt->src.bytes = 1;
3441 fastop(ctxt, em_or);
3443 return X86EMUL_CONTINUE;
3446 static int em_call(struct x86_emulate_ctxt *ctxt)
3449 long rel = ctxt->src.val;
3451 ctxt->src.val = (unsigned long)ctxt->_eip;
3452 rc = jmp_rel(ctxt, rel);
3453 if (rc != X86EMUL_CONTINUE)
3455 return em_push(ctxt);
3458 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3463 struct desc_struct old_desc, new_desc;
3464 const struct x86_emulate_ops *ops = ctxt->ops;
3465 int cpl = ctxt->ops->cpl(ctxt);
3466 enum x86emul_mode prev_mode = ctxt->mode;
3468 old_eip = ctxt->_eip;
3469 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3471 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3472 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3473 X86_TRANSFER_CALL_JMP, &new_desc);
3474 if (rc != X86EMUL_CONTINUE)
3477 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3478 if (rc != X86EMUL_CONTINUE)
3481 ctxt->src.val = old_cs;
3483 if (rc != X86EMUL_CONTINUE)
3486 ctxt->src.val = old_eip;
3488 /* If we failed, we tainted the memory, but the very least we should
3490 if (rc != X86EMUL_CONTINUE) {
3491 pr_warn_once("faulting far call emulation tainted memory\n");
3496 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3497 ctxt->mode = prev_mode;
3502 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3507 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3508 if (rc != X86EMUL_CONTINUE)
3510 rc = assign_eip_near(ctxt, eip);
3511 if (rc != X86EMUL_CONTINUE)
3513 rsp_increment(ctxt, ctxt->src.val);
3514 return X86EMUL_CONTINUE;
3517 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3519 /* Write back the register source. */
3520 ctxt->src.val = ctxt->dst.val;
3521 write_register_operand(&ctxt->src);
3523 /* Write back the memory destination with implicit LOCK prefix. */
3524 ctxt->dst.val = ctxt->src.orig_val;
3525 ctxt->lock_prefix = 1;
3526 return X86EMUL_CONTINUE;
3529 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3531 ctxt->dst.val = ctxt->src2.val;
3532 return fastop(ctxt, em_imul);
3535 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3537 ctxt->dst.type = OP_REG;
3538 ctxt->dst.bytes = ctxt->src.bytes;
3539 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3540 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3542 return X86EMUL_CONTINUE;
3545 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3549 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3550 return emulate_ud(ctxt);
3551 ctxt->dst.val = tsc_aux;
3552 return X86EMUL_CONTINUE;
3555 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3559 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3560 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3561 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3562 return X86EMUL_CONTINUE;
3565 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3569 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3570 return emulate_gp(ctxt, 0);
3571 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3572 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3573 return X86EMUL_CONTINUE;
3576 static int em_mov(struct x86_emulate_ctxt *ctxt)
3578 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3579 return X86EMUL_CONTINUE;
3582 #define FFL(x) bit(X86_FEATURE_##x)
3584 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3586 u32 ebx, ecx, edx, eax = 1;
3590 * Check MOVBE is set in the guest-visible CPUID leaf.
3592 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3593 if (!(ecx & FFL(MOVBE)))
3594 return emulate_ud(ctxt);
3596 switch (ctxt->op_bytes) {
3599 * From MOVBE definition: "...When the operand size is 16 bits,
3600 * the upper word of the destination register remains unchanged
3603 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3604 * rules so we have to do the operation almost per hand.
3606 tmp = (u16)ctxt->src.val;
3607 ctxt->dst.val &= ~0xffffUL;
3608 ctxt->dst.val |= (unsigned long)swab16(tmp);
3611 ctxt->dst.val = swab32((u32)ctxt->src.val);
3614 ctxt->dst.val = swab64(ctxt->src.val);
3619 return X86EMUL_CONTINUE;
3622 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3624 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3625 return emulate_gp(ctxt, 0);
3627 /* Disable writeback. */
3628 ctxt->dst.type = OP_NONE;
3629 return X86EMUL_CONTINUE;
3632 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3636 if (ctxt->mode == X86EMUL_MODE_PROT64)
3637 val = ctxt->src.val & ~0ULL;
3639 val = ctxt->src.val & ~0U;
3641 /* #UD condition is already handled. */
3642 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3643 return emulate_gp(ctxt, 0);
3645 /* Disable writeback. */
3646 ctxt->dst.type = OP_NONE;
3647 return X86EMUL_CONTINUE;
3650 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3654 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3655 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3656 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3657 return emulate_gp(ctxt, 0);
3659 return X86EMUL_CONTINUE;
3662 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3666 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3667 return emulate_gp(ctxt, 0);
3669 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3670 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3671 return X86EMUL_CONTINUE;
3674 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3676 if (ctxt->modrm_reg > VCPU_SREG_GS)
3677 return emulate_ud(ctxt);
3679 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3680 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3681 ctxt->dst.bytes = 2;
3682 return X86EMUL_CONTINUE;
3685 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3687 u16 sel = ctxt->src.val;
3689 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3690 return emulate_ud(ctxt);
3692 if (ctxt->modrm_reg == VCPU_SREG_SS)
3693 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3695 /* Disable writeback. */
3696 ctxt->dst.type = OP_NONE;
3697 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3700 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3702 u16 sel = ctxt->src.val;
3704 /* Disable writeback. */
3705 ctxt->dst.type = OP_NONE;
3706 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3709 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3711 u16 sel = ctxt->src.val;
3713 /* Disable writeback. */
3714 ctxt->dst.type = OP_NONE;
3715 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3718 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3723 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3724 if (rc == X86EMUL_CONTINUE)
3725 ctxt->ops->invlpg(ctxt, linear);
3726 /* Disable writeback. */
3727 ctxt->dst.type = OP_NONE;
3728 return X86EMUL_CONTINUE;
3731 static int em_clts(struct x86_emulate_ctxt *ctxt)
3735 cr0 = ctxt->ops->get_cr(ctxt, 0);
3737 ctxt->ops->set_cr(ctxt, 0, cr0);
3738 return X86EMUL_CONTINUE;
3741 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3743 int rc = ctxt->ops->fix_hypercall(ctxt);
3745 if (rc != X86EMUL_CONTINUE)
3748 /* Let the processor re-execute the fixed hypercall */
3749 ctxt->_eip = ctxt->eip;
3750 /* Disable writeback. */
3751 ctxt->dst.type = OP_NONE;
3752 return X86EMUL_CONTINUE;
3755 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3756 void (*get)(struct x86_emulate_ctxt *ctxt,
3757 struct desc_ptr *ptr))
3759 struct desc_ptr desc_ptr;
3761 if (ctxt->mode == X86EMUL_MODE_PROT64)
3763 get(ctxt, &desc_ptr);
3764 if (ctxt->op_bytes == 2) {
3766 desc_ptr.address &= 0x00ffffff;
3768 /* Disable writeback. */
3769 ctxt->dst.type = OP_NONE;
3770 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3771 &desc_ptr, 2 + ctxt->op_bytes);
3774 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3776 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3779 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3781 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3784 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3786 struct desc_ptr desc_ptr;
3789 if (ctxt->mode == X86EMUL_MODE_PROT64)
3791 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3792 &desc_ptr.size, &desc_ptr.address,
3794 if (rc != X86EMUL_CONTINUE)
3796 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3797 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3798 return emulate_gp(ctxt, 0);
3800 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3802 ctxt->ops->set_idt(ctxt, &desc_ptr);
3803 /* Disable writeback. */
3804 ctxt->dst.type = OP_NONE;
3805 return X86EMUL_CONTINUE;
3808 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3810 return em_lgdt_lidt(ctxt, true);
3813 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3815 return em_lgdt_lidt(ctxt, false);
3818 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3820 if (ctxt->dst.type == OP_MEM)
3821 ctxt->dst.bytes = 2;
3822 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3823 return X86EMUL_CONTINUE;
3826 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3828 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3829 | (ctxt->src.val & 0x0f));
3830 ctxt->dst.type = OP_NONE;
3831 return X86EMUL_CONTINUE;
3834 static int em_loop(struct x86_emulate_ctxt *ctxt)
3836 int rc = X86EMUL_CONTINUE;
3838 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3839 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3840 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3841 rc = jmp_rel(ctxt, ctxt->src.val);
3846 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3848 int rc = X86EMUL_CONTINUE;
3850 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3851 rc = jmp_rel(ctxt, ctxt->src.val);
3856 static int em_in(struct x86_emulate_ctxt *ctxt)
3858 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3860 return X86EMUL_IO_NEEDED;
3862 return X86EMUL_CONTINUE;
3865 static int em_out(struct x86_emulate_ctxt *ctxt)
3867 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3869 /* Disable writeback. */
3870 ctxt->dst.type = OP_NONE;
3871 return X86EMUL_CONTINUE;
3874 static int em_cli(struct x86_emulate_ctxt *ctxt)
3876 if (emulator_bad_iopl(ctxt))
3877 return emulate_gp(ctxt, 0);
3879 ctxt->eflags &= ~X86_EFLAGS_IF;
3880 return X86EMUL_CONTINUE;
3883 static int em_sti(struct x86_emulate_ctxt *ctxt)
3885 if (emulator_bad_iopl(ctxt))
3886 return emulate_gp(ctxt, 0);
3888 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3889 ctxt->eflags |= X86_EFLAGS_IF;
3890 return X86EMUL_CONTINUE;
3893 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3895 u32 eax, ebx, ecx, edx;
3898 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3899 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3900 ctxt->ops->cpl(ctxt)) {
3901 return emulate_gp(ctxt, 0);
3904 eax = reg_read(ctxt, VCPU_REGS_RAX);
3905 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3906 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3907 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3908 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3909 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3910 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3911 return X86EMUL_CONTINUE;
3914 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3918 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3920 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3922 ctxt->eflags &= ~0xffUL;
3923 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3924 return X86EMUL_CONTINUE;
3927 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3929 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3930 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3931 return X86EMUL_CONTINUE;
3934 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3936 switch (ctxt->op_bytes) {
3937 #ifdef CONFIG_X86_64
3939 asm("bswap %0" : "+r"(ctxt->dst.val));
3943 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3946 return X86EMUL_CONTINUE;
3949 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3951 /* emulating clflush regardless of cpuid */
3952 return X86EMUL_CONTINUE;
3955 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3957 /* emulating clflushopt regardless of cpuid */
3958 return X86EMUL_CONTINUE;
3961 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3963 ctxt->dst.val = (s32) ctxt->src.val;
3964 return X86EMUL_CONTINUE;
3967 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3969 u32 eax = 1, ebx, ecx = 0, edx;
3971 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3972 if (!(edx & FFL(FXSR)))
3973 return emulate_ud(ctxt);
3975 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3976 return emulate_nm(ctxt);
3979 * Don't emulate a case that should never be hit, instead of working
3980 * around a lack of fxsave64/fxrstor64 on old compilers.
3982 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3983 return X86EMUL_UNHANDLEABLE;
3985 return X86EMUL_CONTINUE;
3989 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3990 * and restore MXCSR.
3992 static size_t __fxstate_size(int nregs)
3994 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
3997 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4000 if (ctxt->mode == X86EMUL_MODE_PROT64)
4001 return __fxstate_size(16);
4003 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4004 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4008 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4011 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4012 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4014 * 3) 64-bit mode with REX.W prefix
4015 * - like (2), but XMM 8-15 are being saved and restored
4016 * 4) 64-bit mode without REX.W prefix
4017 * - like (3), but FIP and FDP are 64 bit
4019 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4020 * desired result. (4) is not emulated.
4022 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4023 * and FPU DS) should match.
4025 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4027 struct fxregs_state fx_state;
4030 rc = check_fxsr(ctxt);
4031 if (rc != X86EMUL_CONTINUE)
4034 ctxt->ops->get_fpu(ctxt);
4036 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4038 ctxt->ops->put_fpu(ctxt);
4040 if (rc != X86EMUL_CONTINUE)
4043 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4044 fxstate_size(ctxt));
4048 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4049 * in the host registers (via FXSAVE) instead, so they won't be modified.
4050 * (preemption has to stay disabled until FXRSTOR).
4052 * Use noinline to keep the stack for other functions called by callers small.
4054 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4055 const size_t used_size)
4057 struct fxregs_state fx_tmp;
4060 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4061 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4062 __fxstate_size(16) - used_size);
4067 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4069 struct fxregs_state fx_state;
4073 rc = check_fxsr(ctxt);
4074 if (rc != X86EMUL_CONTINUE)
4077 size = fxstate_size(ctxt);
4078 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4079 if (rc != X86EMUL_CONTINUE)
4082 ctxt->ops->get_fpu(ctxt);
4084 if (size < __fxstate_size(16)) {
4085 rc = fxregs_fixup(&fx_state, size);
4086 if (rc != X86EMUL_CONTINUE)
4090 if (fx_state.mxcsr >> 16) {
4091 rc = emulate_gp(ctxt, 0);
4095 if (rc == X86EMUL_CONTINUE)
4096 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4099 ctxt->ops->put_fpu(ctxt);
4104 static bool valid_cr(int nr)
4116 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4118 if (!valid_cr(ctxt->modrm_reg))
4119 return emulate_ud(ctxt);
4121 return X86EMUL_CONTINUE;
4124 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4126 u64 new_val = ctxt->src.val64;
4127 int cr = ctxt->modrm_reg;
4130 static u64 cr_reserved_bits[] = {
4131 0xffffffff00000000ULL,
4132 0, 0, 0, /* CR3 checked later */
4139 return emulate_ud(ctxt);
4141 if (new_val & cr_reserved_bits[cr])
4142 return emulate_gp(ctxt, 0);
4147 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4148 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4149 return emulate_gp(ctxt, 0);
4151 cr4 = ctxt->ops->get_cr(ctxt, 4);
4152 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4154 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4155 !(cr4 & X86_CR4_PAE))
4156 return emulate_gp(ctxt, 0);
4163 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4164 if (efer & EFER_LMA) {
4166 u32 eax, ebx, ecx, edx;
4170 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4172 maxphyaddr = eax & 0xff;
4175 rsvd = rsvd_bits(maxphyaddr, 63);
4176 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4177 rsvd &= ~CR3_PCID_INVD;
4181 return emulate_gp(ctxt, 0);
4186 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4188 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4189 return emulate_gp(ctxt, 0);
4195 return X86EMUL_CONTINUE;
4198 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4202 ctxt->ops->get_dr(ctxt, 7, &dr7);
4204 /* Check if DR7.Global_Enable is set */
4205 return dr7 & (1 << 13);
4208 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4210 int dr = ctxt->modrm_reg;
4214 return emulate_ud(ctxt);
4216 cr4 = ctxt->ops->get_cr(ctxt, 4);
4217 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4218 return emulate_ud(ctxt);
4220 if (check_dr7_gd(ctxt)) {
4223 ctxt->ops->get_dr(ctxt, 6, &dr6);
4225 dr6 |= DR6_BD | DR6_RTM;
4226 ctxt->ops->set_dr(ctxt, 6, dr6);
4227 return emulate_db(ctxt);
4230 return X86EMUL_CONTINUE;
4233 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4235 u64 new_val = ctxt->src.val64;
4236 int dr = ctxt->modrm_reg;
4238 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4239 return emulate_gp(ctxt, 0);
4241 return check_dr_read(ctxt);
4244 static int check_svme(struct x86_emulate_ctxt *ctxt)
4248 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4250 if (!(efer & EFER_SVME))
4251 return emulate_ud(ctxt);
4253 return X86EMUL_CONTINUE;
4256 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4258 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4260 /* Valid physical address? */
4261 if (rax & 0xffff000000000000ULL)
4262 return emulate_gp(ctxt, 0);
4264 return check_svme(ctxt);
4267 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4269 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4271 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4272 return emulate_ud(ctxt);
4274 return X86EMUL_CONTINUE;
4277 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4279 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4280 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4282 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4283 ctxt->ops->check_pmc(ctxt, rcx))
4284 return emulate_gp(ctxt, 0);
4286 return X86EMUL_CONTINUE;
4289 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4291 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4292 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4293 return emulate_gp(ctxt, 0);
4295 return X86EMUL_CONTINUE;
4298 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4300 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4301 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4302 return emulate_gp(ctxt, 0);
4304 return X86EMUL_CONTINUE;
4307 #define D(_y) { .flags = (_y) }
4308 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4309 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4310 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4311 #define N D(NotImpl)
4312 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4313 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4314 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4315 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4316 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4317 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4318 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4319 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4320 #define II(_f, _e, _i) \
4321 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4322 #define IIP(_f, _e, _i, _p) \
4323 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4324 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4325 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4327 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4328 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4329 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4330 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4331 #define I2bvIP(_f, _e, _i, _p) \
4332 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4334 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4335 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4336 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4338 static const struct opcode group7_rm0[] = {
4340 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4344 static const struct opcode group7_rm1[] = {
4345 DI(SrcNone | Priv, monitor),
4346 DI(SrcNone | Priv, mwait),
4350 static const struct opcode group7_rm3[] = {
4351 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4352 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4353 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4354 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4355 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4356 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4357 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4358 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4361 static const struct opcode group7_rm7[] = {
4363 DIP(SrcNone, rdtscp, check_rdtsc),
4367 static const struct opcode group1[] = {
4369 F(Lock | PageTable, em_or),
4372 F(Lock | PageTable, em_and),
4378 static const struct opcode group1A[] = {
4379 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4382 static const struct opcode group2[] = {
4383 F(DstMem | ModRM, em_rol),
4384 F(DstMem | ModRM, em_ror),
4385 F(DstMem | ModRM, em_rcl),
4386 F(DstMem | ModRM, em_rcr),
4387 F(DstMem | ModRM, em_shl),
4388 F(DstMem | ModRM, em_shr),
4389 F(DstMem | ModRM, em_shl),
4390 F(DstMem | ModRM, em_sar),
4393 static const struct opcode group3[] = {
4394 F(DstMem | SrcImm | NoWrite, em_test),
4395 F(DstMem | SrcImm | NoWrite, em_test),
4396 F(DstMem | SrcNone | Lock, em_not),
4397 F(DstMem | SrcNone | Lock, em_neg),
4398 F(DstXacc | Src2Mem, em_mul_ex),
4399 F(DstXacc | Src2Mem, em_imul_ex),
4400 F(DstXacc | Src2Mem, em_div_ex),
4401 F(DstXacc | Src2Mem, em_idiv_ex),
4404 static const struct opcode group4[] = {
4405 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4406 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4410 static const struct opcode group5[] = {
4411 F(DstMem | SrcNone | Lock, em_inc),
4412 F(DstMem | SrcNone | Lock, em_dec),
4413 I(SrcMem | NearBranch, em_call_near_abs),
4414 I(SrcMemFAddr | ImplicitOps, em_call_far),
4415 I(SrcMem | NearBranch, em_jmp_abs),
4416 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4417 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4420 static const struct opcode group6[] = {
4421 DI(Prot | DstMem, sldt),
4422 DI(Prot | DstMem, str),
4423 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4424 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4428 static const struct group_dual group7 = { {
4429 II(Mov | DstMem, em_sgdt, sgdt),
4430 II(Mov | DstMem, em_sidt, sidt),
4431 II(SrcMem | Priv, em_lgdt, lgdt),
4432 II(SrcMem | Priv, em_lidt, lidt),
4433 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4434 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4435 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4439 N, EXT(0, group7_rm3),
4440 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4441 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4445 static const struct opcode group8[] = {
4447 F(DstMem | SrcImmByte | NoWrite, em_bt),
4448 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4449 F(DstMem | SrcImmByte | Lock, em_btr),
4450 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4454 * The "memory" destination is actually always a register, since we come
4455 * from the register case of group9.
4457 static const struct gprefix pfx_0f_c7_7 = {
4458 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4462 static const struct group_dual group9 = { {
4463 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4465 N, N, N, N, N, N, N,
4466 GP(0, &pfx_0f_c7_7),
4469 static const struct opcode group11[] = {
4470 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4474 static const struct gprefix pfx_0f_ae_7 = {
4475 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4478 static const struct group_dual group15 = { {
4479 I(ModRM | Aligned16, em_fxsave),
4480 I(ModRM | Aligned16, em_fxrstor),
4481 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4483 N, N, N, N, N, N, N, N,
4486 static const struct gprefix pfx_0f_6f_0f_7f = {
4487 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4490 static const struct instr_dual instr_dual_0f_2b = {
4494 static const struct gprefix pfx_0f_2b = {
4495 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4498 static const struct gprefix pfx_0f_28_0f_29 = {
4499 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4502 static const struct gprefix pfx_0f_e7 = {
4503 N, I(Sse, em_mov), N, N,
4506 static const struct escape escape_d9 = { {
4507 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4510 N, N, N, N, N, N, N, N,
4512 N, N, N, N, N, N, N, N,
4514 N, N, N, N, N, N, N, N,
4516 N, N, N, N, N, N, N, N,
4518 N, N, N, N, N, N, N, N,
4520 N, N, N, N, N, N, N, N,
4522 N, N, N, N, N, N, N, N,
4524 N, N, N, N, N, N, N, N,
4527 static const struct escape escape_db = { {
4528 N, N, N, N, N, N, N, N,
4531 N, N, N, N, N, N, N, N,
4533 N, N, N, N, N, N, N, N,
4535 N, N, N, N, N, N, N, N,
4537 N, N, N, N, N, N, N, N,
4539 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4541 N, N, N, N, N, N, N, N,
4543 N, N, N, N, N, N, N, N,
4545 N, N, N, N, N, N, N, N,
4548 static const struct escape escape_dd = { {
4549 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4552 N, N, N, N, N, N, N, N,
4554 N, N, N, N, N, N, N, N,
4556 N, N, N, N, N, N, N, N,
4558 N, N, N, N, N, N, N, N,
4560 N, N, N, N, N, N, N, N,
4562 N, N, N, N, N, N, N, N,
4564 N, N, N, N, N, N, N, N,
4566 N, N, N, N, N, N, N, N,
4569 static const struct instr_dual instr_dual_0f_c3 = {
4570 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4573 static const struct mode_dual mode_dual_63 = {
4574 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4577 static const struct opcode opcode_table[256] = {
4579 F6ALU(Lock, em_add),
4580 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4581 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4583 F6ALU(Lock | PageTable, em_or),
4584 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4587 F6ALU(Lock, em_adc),
4588 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4589 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4591 F6ALU(Lock, em_sbb),
4592 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4593 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4595 F6ALU(Lock | PageTable, em_and), N, N,
4597 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4599 F6ALU(Lock, em_xor), N, N,
4601 F6ALU(NoWrite, em_cmp), N, N,
4603 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4605 X8(I(SrcReg | Stack, em_push)),
4607 X8(I(DstReg | Stack, em_pop)),
4609 I(ImplicitOps | Stack | No64, em_pusha),
4610 I(ImplicitOps | Stack | No64, em_popa),
4611 N, MD(ModRM, &mode_dual_63),
4614 I(SrcImm | Mov | Stack, em_push),
4615 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4616 I(SrcImmByte | Mov | Stack, em_push),
4617 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4618 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4619 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4621 X16(D(SrcImmByte | NearBranch)),
4623 G(ByteOp | DstMem | SrcImm, group1),
4624 G(DstMem | SrcImm, group1),
4625 G(ByteOp | DstMem | SrcImm | No64, group1),
4626 G(DstMem | SrcImmByte, group1),
4627 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4628 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4630 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4631 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4632 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4633 D(ModRM | SrcMem | NoAccess | DstReg),
4634 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4637 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4639 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4640 I(SrcImmFAddr | No64, em_call_far), N,
4641 II(ImplicitOps | Stack, em_pushf, pushf),
4642 II(ImplicitOps | Stack, em_popf, popf),
4643 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4645 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4646 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4647 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4648 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4650 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4651 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4652 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4653 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4655 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4657 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4659 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4660 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4661 I(ImplicitOps | NearBranch, em_ret),
4662 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4663 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4664 G(ByteOp, group11), G(0, group11),
4666 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4667 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4668 I(ImplicitOps, em_ret_far),
4669 D(ImplicitOps), DI(SrcImmByte, intn),
4670 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4672 G(Src2One | ByteOp, group2), G(Src2One, group2),
4673 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4674 I(DstAcc | SrcImmUByte | No64, em_aam),
4675 I(DstAcc | SrcImmUByte | No64, em_aad),
4676 F(DstAcc | ByteOp | No64, em_salc),
4677 I(DstAcc | SrcXLat | ByteOp, em_mov),
4679 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4681 X3(I(SrcImmByte | NearBranch, em_loop)),
4682 I(SrcImmByte | NearBranch, em_jcxz),
4683 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4684 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4686 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4687 I(SrcImmFAddr | No64, em_jmp_far),
4688 D(SrcImmByte | ImplicitOps | NearBranch),
4689 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4690 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4692 N, DI(ImplicitOps, icebp), N, N,
4693 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4694 G(ByteOp, group3), G(0, group3),
4696 D(ImplicitOps), D(ImplicitOps),
4697 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4698 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4701 static const struct opcode twobyte_table[256] = {
4703 G(0, group6), GD(0, &group7), N, N,
4704 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4705 II(ImplicitOps | Priv, em_clts, clts), N,
4706 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4707 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4709 N, N, N, N, N, N, N, N,
4710 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4711 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4713 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4714 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4715 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4717 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4720 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4721 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4722 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4725 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4726 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4727 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4728 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4729 I(ImplicitOps | EmulateOnUD, em_sysenter),
4730 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4732 N, N, N, N, N, N, N, N,
4734 X16(D(DstReg | SrcMem | ModRM)),
4736 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4741 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4746 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4748 X16(D(SrcImm | NearBranch)),
4750 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4752 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4753 II(ImplicitOps, em_cpuid, cpuid),
4754 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4755 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4756 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4758 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4759 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4760 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4761 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4762 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4763 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4765 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4766 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4767 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4768 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4769 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4770 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4774 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4775 I(DstReg | SrcMem | ModRM, em_bsf_c),
4776 I(DstReg | SrcMem | ModRM, em_bsr_c),
4777 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4779 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4780 N, ID(0, &instr_dual_0f_c3),
4781 N, N, N, GD(0, &group9),
4783 X8(I(DstReg, em_bswap)),
4785 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4787 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4788 N, N, N, N, N, N, N, N,
4790 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4793 static const struct instr_dual instr_dual_0f_38_f0 = {
4794 I(DstReg | SrcMem | Mov, em_movbe), N
4797 static const struct instr_dual instr_dual_0f_38_f1 = {
4798 I(DstMem | SrcReg | Mov, em_movbe), N
4801 static const struct gprefix three_byte_0f_38_f0 = {
4802 ID(0, &instr_dual_0f_38_f0), N, N, N
4805 static const struct gprefix three_byte_0f_38_f1 = {
4806 ID(0, &instr_dual_0f_38_f1), N, N, N
4810 * Insns below are selected by the prefix which indexed by the third opcode
4813 static const struct opcode opcode_map_0f_38[256] = {
4815 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4817 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4819 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4820 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4841 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4845 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4851 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4852 unsigned size, bool sign_extension)
4854 int rc = X86EMUL_CONTINUE;
4858 op->addr.mem.ea = ctxt->_eip;
4859 /* NB. Immediates are sign-extended as necessary. */
4860 switch (op->bytes) {
4862 op->val = insn_fetch(s8, ctxt);
4865 op->val = insn_fetch(s16, ctxt);
4868 op->val = insn_fetch(s32, ctxt);
4871 op->val = insn_fetch(s64, ctxt);
4874 if (!sign_extension) {
4875 switch (op->bytes) {
4883 op->val &= 0xffffffff;
4891 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4894 int rc = X86EMUL_CONTINUE;
4898 decode_register_operand(ctxt, op);
4901 rc = decode_imm(ctxt, op, 1, false);
4904 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4908 if (ctxt->d & BitOp)
4909 fetch_bit_operand(ctxt);
4910 op->orig_val = op->val;
4913 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4917 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4918 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4919 fetch_register_operand(op);
4920 op->orig_val = op->val;
4924 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4925 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4926 fetch_register_operand(op);
4927 op->orig_val = op->val;
4930 if (ctxt->d & ByteOp) {
4935 op->bytes = ctxt->op_bytes;
4936 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4937 fetch_register_operand(op);
4938 op->orig_val = op->val;
4942 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4944 register_address(ctxt, VCPU_REGS_RDI);
4945 op->addr.mem.seg = VCPU_SREG_ES;
4952 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4953 fetch_register_operand(op);
4958 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4961 rc = decode_imm(ctxt, op, 1, true);
4969 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4972 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4975 ctxt->memop.bytes = 1;
4976 if (ctxt->memop.type == OP_REG) {
4977 ctxt->memop.addr.reg = decode_register(ctxt,
4978 ctxt->modrm_rm, true);
4979 fetch_register_operand(&ctxt->memop);
4983 ctxt->memop.bytes = 2;
4986 ctxt->memop.bytes = 4;
4989 rc = decode_imm(ctxt, op, 2, false);
4992 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4996 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4998 register_address(ctxt, VCPU_REGS_RSI);
4999 op->addr.mem.seg = ctxt->seg_override;
5005 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5008 reg_read(ctxt, VCPU_REGS_RBX) +
5009 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5010 op->addr.mem.seg = ctxt->seg_override;
5015 op->addr.mem.ea = ctxt->_eip;
5016 op->bytes = ctxt->op_bytes + 2;
5017 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5020 ctxt->memop.bytes = ctxt->op_bytes + 2;
5024 op->val = VCPU_SREG_ES;
5028 op->val = VCPU_SREG_CS;
5032 op->val = VCPU_SREG_SS;
5036 op->val = VCPU_SREG_DS;
5040 op->val = VCPU_SREG_FS;
5044 op->val = VCPU_SREG_GS;
5047 /* Special instructions do their own operand decoding. */
5049 op->type = OP_NONE; /* Disable writeback. */
5057 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5059 int rc = X86EMUL_CONTINUE;
5060 int mode = ctxt->mode;
5061 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5062 bool op_prefix = false;
5063 bool has_seg_override = false;
5064 struct opcode opcode;
5066 struct desc_struct desc;
5068 ctxt->memop.type = OP_NONE;
5069 ctxt->memopp = NULL;
5070 ctxt->_eip = ctxt->eip;
5071 ctxt->fetch.ptr = ctxt->fetch.data;
5072 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5073 ctxt->opcode_len = 1;
5074 ctxt->intercept = x86_intercept_none;
5076 memcpy(ctxt->fetch.data, insn, insn_len);
5078 rc = __do_insn_fetch_bytes(ctxt, 1);
5079 if (rc != X86EMUL_CONTINUE)
5084 case X86EMUL_MODE_REAL:
5085 case X86EMUL_MODE_VM86:
5086 def_op_bytes = def_ad_bytes = 2;
5087 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5089 def_op_bytes = def_ad_bytes = 4;
5091 case X86EMUL_MODE_PROT16:
5092 def_op_bytes = def_ad_bytes = 2;
5094 case X86EMUL_MODE_PROT32:
5095 def_op_bytes = def_ad_bytes = 4;
5097 #ifdef CONFIG_X86_64
5098 case X86EMUL_MODE_PROT64:
5104 return EMULATION_FAILED;
5107 ctxt->op_bytes = def_op_bytes;
5108 ctxt->ad_bytes = def_ad_bytes;
5110 /* Legacy prefixes. */
5112 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5113 case 0x66: /* operand-size override */
5115 /* switch between 2/4 bytes */
5116 ctxt->op_bytes = def_op_bytes ^ 6;
5118 case 0x67: /* address-size override */
5119 if (mode == X86EMUL_MODE_PROT64)
5120 /* switch between 4/8 bytes */
5121 ctxt->ad_bytes = def_ad_bytes ^ 12;
5123 /* switch between 2/4 bytes */
5124 ctxt->ad_bytes = def_ad_bytes ^ 6;
5126 case 0x26: /* ES override */
5127 has_seg_override = true;
5128 ctxt->seg_override = VCPU_SREG_ES;
5130 case 0x2e: /* CS override */
5131 has_seg_override = true;
5132 ctxt->seg_override = VCPU_SREG_CS;
5134 case 0x36: /* SS override */
5135 has_seg_override = true;
5136 ctxt->seg_override = VCPU_SREG_SS;
5138 case 0x3e: /* DS override */
5139 has_seg_override = true;
5140 ctxt->seg_override = VCPU_SREG_DS;
5142 case 0x64: /* FS override */
5143 has_seg_override = true;
5144 ctxt->seg_override = VCPU_SREG_FS;
5146 case 0x65: /* GS override */
5147 has_seg_override = true;
5148 ctxt->seg_override = VCPU_SREG_GS;
5150 case 0x40 ... 0x4f: /* REX */
5151 if (mode != X86EMUL_MODE_PROT64)
5153 ctxt->rex_prefix = ctxt->b;
5155 case 0xf0: /* LOCK */
5156 ctxt->lock_prefix = 1;
5158 case 0xf2: /* REPNE/REPNZ */
5159 case 0xf3: /* REP/REPE/REPZ */
5160 ctxt->rep_prefix = ctxt->b;
5166 /* Any legacy prefix after a REX prefix nullifies its effect. */
5168 ctxt->rex_prefix = 0;
5174 if (ctxt->rex_prefix & 8)
5175 ctxt->op_bytes = 8; /* REX.W */
5177 /* Opcode byte(s). */
5178 opcode = opcode_table[ctxt->b];
5179 /* Two-byte opcode? */
5180 if (ctxt->b == 0x0f) {
5181 ctxt->opcode_len = 2;
5182 ctxt->b = insn_fetch(u8, ctxt);
5183 opcode = twobyte_table[ctxt->b];
5185 /* 0F_38 opcode map */
5186 if (ctxt->b == 0x38) {
5187 ctxt->opcode_len = 3;
5188 ctxt->b = insn_fetch(u8, ctxt);
5189 opcode = opcode_map_0f_38[ctxt->b];
5192 ctxt->d = opcode.flags;
5194 if (ctxt->d & ModRM)
5195 ctxt->modrm = insn_fetch(u8, ctxt);
5197 /* vex-prefix instructions are not implemented */
5198 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5199 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5203 while (ctxt->d & GroupMask) {
5204 switch (ctxt->d & GroupMask) {
5206 goffset = (ctxt->modrm >> 3) & 7;
5207 opcode = opcode.u.group[goffset];
5210 goffset = (ctxt->modrm >> 3) & 7;
5211 if ((ctxt->modrm >> 6) == 3)
5212 opcode = opcode.u.gdual->mod3[goffset];
5214 opcode = opcode.u.gdual->mod012[goffset];
5217 goffset = ctxt->modrm & 7;
5218 opcode = opcode.u.group[goffset];
5221 if (ctxt->rep_prefix && op_prefix)
5222 return EMULATION_FAILED;
5223 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5224 switch (simd_prefix) {
5225 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5226 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5227 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5228 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5232 if (ctxt->modrm > 0xbf) {
5233 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5234 u32 index = array_index_nospec(
5235 ctxt->modrm - 0xc0, size);
5237 opcode = opcode.u.esc->high[index];
5239 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5243 if ((ctxt->modrm >> 6) == 3)
5244 opcode = opcode.u.idual->mod3;
5246 opcode = opcode.u.idual->mod012;
5249 if (ctxt->mode == X86EMUL_MODE_PROT64)
5250 opcode = opcode.u.mdual->mode64;
5252 opcode = opcode.u.mdual->mode32;
5255 return EMULATION_FAILED;
5258 ctxt->d &= ~(u64)GroupMask;
5259 ctxt->d |= opcode.flags;
5264 return EMULATION_FAILED;
5266 ctxt->execute = opcode.u.execute;
5268 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5269 return EMULATION_FAILED;
5271 if (unlikely(ctxt->d &
5272 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5275 * These are copied unconditionally here, and checked unconditionally
5276 * in x86_emulate_insn.
5278 ctxt->check_perm = opcode.check_perm;
5279 ctxt->intercept = opcode.intercept;
5281 if (ctxt->d & NotImpl)
5282 return EMULATION_FAILED;
5284 if (mode == X86EMUL_MODE_PROT64) {
5285 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5287 else if (ctxt->d & NearBranch)
5291 if (ctxt->d & Op3264) {
5292 if (mode == X86EMUL_MODE_PROT64)
5298 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5302 ctxt->op_bytes = 16;
5303 else if (ctxt->d & Mmx)
5307 /* ModRM and SIB bytes. */
5308 if (ctxt->d & ModRM) {
5309 rc = decode_modrm(ctxt, &ctxt->memop);
5310 if (!has_seg_override) {
5311 has_seg_override = true;
5312 ctxt->seg_override = ctxt->modrm_seg;
5314 } else if (ctxt->d & MemAbs)
5315 rc = decode_abs(ctxt, &ctxt->memop);
5316 if (rc != X86EMUL_CONTINUE)
5319 if (!has_seg_override)
5320 ctxt->seg_override = VCPU_SREG_DS;
5322 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5325 * Decode and fetch the source operand: register, memory
5328 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5329 if (rc != X86EMUL_CONTINUE)
5333 * Decode and fetch the second source operand: register, memory
5336 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5337 if (rc != X86EMUL_CONTINUE)
5340 /* Decode and fetch the destination operand: register or memory. */
5341 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5343 if (ctxt->rip_relative && likely(ctxt->memopp))
5344 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5345 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5348 if (rc == X86EMUL_PROPAGATE_FAULT)
5349 ctxt->have_exception = true;
5350 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5353 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5355 return ctxt->d & PageTable;
5358 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5360 /* The second termination condition only applies for REPE
5361 * and REPNE. Test if the repeat string operation prefix is
5362 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5363 * corresponding termination condition according to:
5364 * - if REPE/REPZ and ZF = 0 then done
5365 * - if REPNE/REPNZ and ZF = 1 then done
5367 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5368 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5369 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5370 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5371 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5372 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5378 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5382 ctxt->ops->get_fpu(ctxt);
5383 rc = asm_safe("fwait");
5384 ctxt->ops->put_fpu(ctxt);
5386 if (unlikely(rc != X86EMUL_CONTINUE))
5387 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5389 return X86EMUL_CONTINUE;
5392 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5395 if (op->type == OP_MM)
5396 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5399 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5401 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5403 if (!(ctxt->d & ByteOp))
5404 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5406 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5407 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5408 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5409 : "c"(ctxt->src2.val));
5411 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5412 if (!fop) /* exception is returned in fop variable */
5413 return emulate_de(ctxt);
5414 return X86EMUL_CONTINUE;
5417 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5419 memset(&ctxt->rip_relative, 0,
5420 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5422 ctxt->io_read.pos = 0;
5423 ctxt->io_read.end = 0;
5424 ctxt->mem_read.end = 0;
5427 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5429 const struct x86_emulate_ops *ops = ctxt->ops;
5430 int rc = X86EMUL_CONTINUE;
5431 int saved_dst_type = ctxt->dst.type;
5432 unsigned emul_flags;
5434 ctxt->mem_read.pos = 0;
5436 /* LOCK prefix is allowed only with some instructions */
5437 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5438 rc = emulate_ud(ctxt);
5442 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5443 rc = emulate_ud(ctxt);
5447 emul_flags = ctxt->ops->get_hflags(ctxt);
5448 if (unlikely(ctxt->d &
5449 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5450 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5451 (ctxt->d & Undefined)) {
5452 rc = emulate_ud(ctxt);
5456 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5457 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5458 rc = emulate_ud(ctxt);
5462 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5463 rc = emulate_nm(ctxt);
5467 if (ctxt->d & Mmx) {
5468 rc = flush_pending_x87_faults(ctxt);
5469 if (rc != X86EMUL_CONTINUE)
5472 * Now that we know the fpu is exception safe, we can fetch
5475 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5476 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5477 if (!(ctxt->d & Mov))
5478 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5481 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5482 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5483 X86_ICPT_PRE_EXCEPT);
5484 if (rc != X86EMUL_CONTINUE)
5488 /* Instruction can only be executed in protected mode */
5489 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5490 rc = emulate_ud(ctxt);
5494 /* Privileged instruction can be executed only in CPL=0 */
5495 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5496 if (ctxt->d & PrivUD)
5497 rc = emulate_ud(ctxt);
5499 rc = emulate_gp(ctxt, 0);
5503 /* Do instruction specific permission checks */
5504 if (ctxt->d & CheckPerm) {
5505 rc = ctxt->check_perm(ctxt);
5506 if (rc != X86EMUL_CONTINUE)
5510 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5511 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5512 X86_ICPT_POST_EXCEPT);
5513 if (rc != X86EMUL_CONTINUE)
5517 if (ctxt->rep_prefix && (ctxt->d & String)) {
5518 /* All REP prefixes have the same first termination condition */
5519 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5520 string_registers_quirk(ctxt);
5521 ctxt->eip = ctxt->_eip;
5522 ctxt->eflags &= ~X86_EFLAGS_RF;
5528 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5529 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5530 ctxt->src.valptr, ctxt->src.bytes);
5531 if (rc != X86EMUL_CONTINUE)
5533 ctxt->src.orig_val64 = ctxt->src.val64;
5536 if (ctxt->src2.type == OP_MEM) {
5537 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5538 &ctxt->src2.val, ctxt->src2.bytes);
5539 if (rc != X86EMUL_CONTINUE)
5543 if ((ctxt->d & DstMask) == ImplicitOps)
5547 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5548 /* optimisation - avoid slow emulated read if Mov */
5549 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5550 &ctxt->dst.val, ctxt->dst.bytes);
5551 if (rc != X86EMUL_CONTINUE) {
5552 if (!(ctxt->d & NoWrite) &&
5553 rc == X86EMUL_PROPAGATE_FAULT &&
5554 ctxt->exception.vector == PF_VECTOR)
5555 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5559 /* Copy full 64-bit value for CMPXCHG8B. */
5560 ctxt->dst.orig_val64 = ctxt->dst.val64;
5564 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5565 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5566 X86_ICPT_POST_MEMACCESS);
5567 if (rc != X86EMUL_CONTINUE)
5571 if (ctxt->rep_prefix && (ctxt->d & String))
5572 ctxt->eflags |= X86_EFLAGS_RF;
5574 ctxt->eflags &= ~X86_EFLAGS_RF;
5576 if (ctxt->execute) {
5577 if (ctxt->d & Fastop) {
5578 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5579 rc = fastop(ctxt, fop);
5580 if (rc != X86EMUL_CONTINUE)
5584 rc = ctxt->execute(ctxt);
5585 if (rc != X86EMUL_CONTINUE)
5590 if (ctxt->opcode_len == 2)
5592 else if (ctxt->opcode_len == 3)
5593 goto threebyte_insn;
5596 case 0x70 ... 0x7f: /* jcc (short) */
5597 if (test_cc(ctxt->b, ctxt->eflags))
5598 rc = jmp_rel(ctxt, ctxt->src.val);
5600 case 0x8d: /* lea r16/r32, m */
5601 ctxt->dst.val = ctxt->src.addr.mem.ea;
5603 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5604 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5605 ctxt->dst.type = OP_NONE;
5609 case 0x98: /* cbw/cwde/cdqe */
5610 switch (ctxt->op_bytes) {
5611 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5612 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5613 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5616 case 0xcc: /* int3 */
5617 rc = emulate_int(ctxt, 3);
5619 case 0xcd: /* int n */
5620 rc = emulate_int(ctxt, ctxt->src.val);
5622 case 0xce: /* into */
5623 if (ctxt->eflags & X86_EFLAGS_OF)
5624 rc = emulate_int(ctxt, 4);
5626 case 0xe9: /* jmp rel */
5627 case 0xeb: /* jmp rel short */
5628 rc = jmp_rel(ctxt, ctxt->src.val);
5629 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5631 case 0xf4: /* hlt */
5632 ctxt->ops->halt(ctxt);
5634 case 0xf5: /* cmc */
5635 /* complement carry flag from eflags reg */
5636 ctxt->eflags ^= X86_EFLAGS_CF;
5638 case 0xf8: /* clc */
5639 ctxt->eflags &= ~X86_EFLAGS_CF;
5641 case 0xf9: /* stc */
5642 ctxt->eflags |= X86_EFLAGS_CF;
5644 case 0xfc: /* cld */
5645 ctxt->eflags &= ~X86_EFLAGS_DF;
5647 case 0xfd: /* std */
5648 ctxt->eflags |= X86_EFLAGS_DF;
5651 goto cannot_emulate;
5654 if (rc != X86EMUL_CONTINUE)
5658 if (ctxt->d & SrcWrite) {
5659 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5660 rc = writeback(ctxt, &ctxt->src);
5661 if (rc != X86EMUL_CONTINUE)
5664 if (!(ctxt->d & NoWrite)) {
5665 rc = writeback(ctxt, &ctxt->dst);
5666 if (rc != X86EMUL_CONTINUE)
5671 * restore dst type in case the decoding will be reused
5672 * (happens for string instruction )
5674 ctxt->dst.type = saved_dst_type;
5676 if ((ctxt->d & SrcMask) == SrcSI)
5677 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5679 if ((ctxt->d & DstMask) == DstDI)
5680 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5682 if (ctxt->rep_prefix && (ctxt->d & String)) {
5684 struct read_cache *r = &ctxt->io_read;
5685 if ((ctxt->d & SrcMask) == SrcSI)
5686 count = ctxt->src.count;
5688 count = ctxt->dst.count;
5689 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5691 if (!string_insn_completed(ctxt)) {
5693 * Re-enter guest when pio read ahead buffer is empty
5694 * or, if it is not used, after each 1024 iteration.
5696 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5697 (r->end == 0 || r->end != r->pos)) {
5699 * Reset read cache. Usually happens before
5700 * decode, but since instruction is restarted
5701 * we have to do it here.
5703 ctxt->mem_read.end = 0;
5704 writeback_registers(ctxt);
5705 return EMULATION_RESTART;
5707 goto done; /* skip rip writeback */
5709 ctxt->eflags &= ~X86_EFLAGS_RF;
5712 ctxt->eip = ctxt->_eip;
5715 if (rc == X86EMUL_PROPAGATE_FAULT) {
5716 WARN_ON(ctxt->exception.vector > 0x1f);
5717 ctxt->have_exception = true;
5719 if (rc == X86EMUL_INTERCEPTED)
5720 return EMULATION_INTERCEPTED;
5722 if (rc == X86EMUL_CONTINUE)
5723 writeback_registers(ctxt);
5725 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5729 case 0x09: /* wbinvd */
5730 (ctxt->ops->wbinvd)(ctxt);
5732 case 0x08: /* invd */
5733 case 0x0d: /* GrpP (prefetch) */
5734 case 0x18: /* Grp16 (prefetch/nop) */
5735 case 0x1f: /* nop */
5737 case 0x20: /* mov cr, reg */
5738 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5740 case 0x21: /* mov from dr to reg */
5741 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5743 case 0x40 ... 0x4f: /* cmov */
5744 if (test_cc(ctxt->b, ctxt->eflags))
5745 ctxt->dst.val = ctxt->src.val;
5746 else if (ctxt->op_bytes != 4)
5747 ctxt->dst.type = OP_NONE; /* no writeback */
5749 case 0x80 ... 0x8f: /* jnz rel, etc*/
5750 if (test_cc(ctxt->b, ctxt->eflags))
5751 rc = jmp_rel(ctxt, ctxt->src.val);
5753 case 0x90 ... 0x9f: /* setcc r/m8 */
5754 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5756 case 0xb6 ... 0xb7: /* movzx */
5757 ctxt->dst.bytes = ctxt->op_bytes;
5758 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5759 : (u16) ctxt->src.val;
5761 case 0xbe ... 0xbf: /* movsx */
5762 ctxt->dst.bytes = ctxt->op_bytes;
5763 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5764 (s16) ctxt->src.val;
5767 goto cannot_emulate;
5772 if (rc != X86EMUL_CONTINUE)
5778 return EMULATION_FAILED;
5781 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5783 invalidate_registers(ctxt);
5786 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5788 writeback_registers(ctxt);
5791 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5793 if (ctxt->rep_prefix && (ctxt->d & String))
5796 if (ctxt->d & TwoMemOp)