1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include <asm/kvm_emulate.h>
24 #include <linux/stringify.h>
25 #include <asm/fpu/api.h>
26 #include <asm/debugreg.h>
27 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
195 * fastop functions have a special calling convention:
200 * flags: rflags (in/out)
201 * ex: rsi (in:fastop pointer, out:zero if exception)
203 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
204 * different operand sizes can be reached by calculation, rather than a jump
205 * table (which would be bigger than the code).
207 * fastop functions are declared as taking a never-defined fastop parameter,
208 * so they can't be called from C directly.
217 int (*execute)(struct x86_emulate_ctxt *ctxt);
218 const struct opcode *group;
219 const struct group_dual *gdual;
220 const struct gprefix *gprefix;
221 const struct escape *esc;
222 const struct instr_dual *idual;
223 const struct mode_dual *mdual;
224 void (*fastop)(struct fastop *fake);
226 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
230 struct opcode mod012[8];
231 struct opcode mod3[8];
235 struct opcode pfx_no;
236 struct opcode pfx_66;
237 struct opcode pfx_f2;
238 struct opcode pfx_f3;
243 struct opcode high[64];
247 struct opcode mod012;
252 struct opcode mode32;
253 struct opcode mode64;
256 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
258 enum x86_transfer_type {
260 X86_TRANSFER_CALL_JMP,
262 X86_TRANSFER_TASK_SWITCH,
265 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 if (!(ctxt->regs_valid & (1 << nr))) {
268 ctxt->regs_valid |= 1 << nr;
269 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
271 return ctxt->_regs[nr];
274 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
276 ctxt->regs_valid |= 1 << nr;
277 ctxt->regs_dirty |= 1 << nr;
278 return &ctxt->_regs[nr];
281 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
284 return reg_write(ctxt, nr);
287 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
291 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
292 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
295 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
297 ctxt->regs_dirty = 0;
298 ctxt->regs_valid = 0;
302 * These EFLAGS bits are restored from saved value during emulation, and
303 * any changes are written back to the saved value after emulation.
305 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
306 X86_EFLAGS_PF|X86_EFLAGS_CF)
314 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
316 #define __FOP_FUNC(name) \
317 ".align " __stringify(FASTOP_SIZE) " \n\t" \
318 ".type " name ", @function \n\t" \
321 #define FOP_FUNC(name) \
324 #define __FOP_RET(name) \
326 ".size " name ", .-" name "\n\t"
328 #define FOP_RET(name) \
331 #define FOP_START(op) \
332 extern void em_##op(struct fastop *fake); \
333 asm(".pushsection .text, \"ax\" \n\t" \
334 ".global em_" #op " \n\t" \
335 ".align " __stringify(FASTOP_SIZE) " \n\t" \
341 #define __FOPNOP(name) \
346 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
348 #define FOP1E(op, dst) \
349 __FOP_FUNC(#op "_" #dst) \
350 "10: " #op " %" #dst " \n\t" \
351 __FOP_RET(#op "_" #dst)
353 #define FOP1EEX(op, dst) \
354 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
356 #define FASTOP1(op) \
361 ON64(FOP1E(op##q, rax)) \
364 /* 1-operand, using src2 (for MUL/DIV r/m) */
365 #define FASTOP1SRC2(op, name) \
370 ON64(FOP1E(op, rcx)) \
373 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
374 #define FASTOP1SRC2EX(op, name) \
379 ON64(FOP1EEX(op, rcx)) \
382 #define FOP2E(op, dst, src) \
383 __FOP_FUNC(#op "_" #dst "_" #src) \
384 #op " %" #src ", %" #dst " \n\t" \
385 __FOP_RET(#op "_" #dst "_" #src)
387 #define FASTOP2(op) \
389 FOP2E(op##b, al, dl) \
390 FOP2E(op##w, ax, dx) \
391 FOP2E(op##l, eax, edx) \
392 ON64(FOP2E(op##q, rax, rdx)) \
395 /* 2 operand, word only */
396 #define FASTOP2W(op) \
399 FOP2E(op##w, ax, dx) \
400 FOP2E(op##l, eax, edx) \
401 ON64(FOP2E(op##q, rax, rdx)) \
404 /* 2 operand, src is CL */
405 #define FASTOP2CL(op) \
407 FOP2E(op##b, al, cl) \
408 FOP2E(op##w, ax, cl) \
409 FOP2E(op##l, eax, cl) \
410 ON64(FOP2E(op##q, rax, cl)) \
413 /* 2 operand, src and dest are reversed */
414 #define FASTOP2R(op, name) \
416 FOP2E(op##b, dl, al) \
417 FOP2E(op##w, dx, ax) \
418 FOP2E(op##l, edx, eax) \
419 ON64(FOP2E(op##q, rdx, rax)) \
422 #define FOP3E(op, dst, src, src2) \
423 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
424 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
425 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
427 /* 3-operand, word-only, src2=cl */
428 #define FASTOP3WCL(op) \
431 FOP3E(op##w, ax, dx, cl) \
432 FOP3E(op##l, eax, edx, cl) \
433 ON64(FOP3E(op##q, rax, rdx, cl)) \
436 /* Special case for SETcc - 1 instruction per cc */
437 #define FOP_SETCC(op) \
439 ".type " #op ", @function \n\t" \
444 asm(".pushsection .fixup, \"ax\"\n"
445 ".global kvm_fastop_exception \n"
446 "kvm_fastop_exception: xor %esi, %esi; ret\n"
470 "pushf; sbb %al, %al; popf \n\t"
475 * XXX: inoutclob user must know where the argument is being expanded.
476 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
478 #define asm_safe(insn, inoutclob...) \
482 asm volatile("1:" insn "\n" \
484 ".pushsection .fixup, \"ax\"\n" \
485 "3: movl $1, %[_fault]\n" \
488 _ASM_EXTABLE(1b, 3b) \
489 : [_fault] "+qm"(_fault) inoutclob ); \
491 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
494 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
495 enum x86_intercept intercept,
496 enum x86_intercept_stage stage)
498 struct x86_instruction_info info = {
499 .intercept = intercept,
500 .rep_prefix = ctxt->rep_prefix,
501 .modrm_mod = ctxt->modrm_mod,
502 .modrm_reg = ctxt->modrm_reg,
503 .modrm_rm = ctxt->modrm_rm,
504 .src_val = ctxt->src.val64,
505 .dst_val = ctxt->dst.val64,
506 .src_bytes = ctxt->src.bytes,
507 .dst_bytes = ctxt->dst.bytes,
508 .ad_bytes = ctxt->ad_bytes,
509 .next_rip = ctxt->eip,
512 return ctxt->ops->intercept(ctxt, &info, stage);
515 static void assign_masked(ulong *dest, ulong src, ulong mask)
517 *dest = (*dest & ~mask) | (src & mask);
520 static void assign_register(unsigned long *reg, u64 val, int bytes)
522 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
525 *(u8 *)reg = (u8)val;
528 *(u16 *)reg = (u16)val;
532 break; /* 64b: zero-extend */
539 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
541 return (1UL << (ctxt->ad_bytes << 3)) - 1;
544 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
547 struct desc_struct ss;
549 if (ctxt->mode == X86EMUL_MODE_PROT64)
551 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
552 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
555 static int stack_size(struct x86_emulate_ctxt *ctxt)
557 return (__fls(stack_mask(ctxt)) + 1) >> 3;
560 /* Access/update address held in a register, based on addressing mode. */
561 static inline unsigned long
562 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
564 if (ctxt->ad_bytes == sizeof(unsigned long))
567 return reg & ad_mask(ctxt);
570 static inline unsigned long
571 register_address(struct x86_emulate_ctxt *ctxt, int reg)
573 return address_mask(ctxt, reg_read(ctxt, reg));
576 static void masked_increment(ulong *reg, ulong mask, int inc)
578 assign_masked(reg, *reg + inc, mask);
582 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
584 ulong *preg = reg_rmw(ctxt, reg);
586 assign_register(preg, *preg + inc, ctxt->ad_bytes);
589 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
591 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
594 static u32 desc_limit_scaled(struct desc_struct *desc)
596 u32 limit = get_desc_limit(desc);
598 return desc->g ? (limit << 12) | 0xfff : limit;
601 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
603 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
606 return ctxt->ops->get_cached_segment_base(ctxt, seg);
609 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
610 u32 error, bool valid)
613 ctxt->exception.vector = vec;
614 ctxt->exception.error_code = error;
615 ctxt->exception.error_code_valid = valid;
616 return X86EMUL_PROPAGATE_FAULT;
619 static int emulate_db(struct x86_emulate_ctxt *ctxt)
621 return emulate_exception(ctxt, DB_VECTOR, 0, false);
624 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
626 return emulate_exception(ctxt, GP_VECTOR, err, true);
629 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
631 return emulate_exception(ctxt, SS_VECTOR, err, true);
634 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
636 return emulate_exception(ctxt, UD_VECTOR, 0, false);
639 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
641 return emulate_exception(ctxt, TS_VECTOR, err, true);
644 static int emulate_de(struct x86_emulate_ctxt *ctxt)
646 return emulate_exception(ctxt, DE_VECTOR, 0, false);
649 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
651 return emulate_exception(ctxt, NM_VECTOR, 0, false);
654 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
657 struct desc_struct desc;
659 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
663 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
668 struct desc_struct desc;
670 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
671 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
675 * x86 defines three classes of vector instructions: explicitly
676 * aligned, explicitly unaligned, and the rest, which change behaviour
677 * depending on whether they're AVX encoded or not.
679 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
680 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
681 * 512 bytes of data must be aligned to a 16 byte boundary.
683 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
685 u64 alignment = ctxt->d & AlignMask;
687 if (likely(size < 16))
702 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
703 struct segmented_address addr,
704 unsigned *max_size, unsigned size,
705 bool write, bool fetch,
706 enum x86emul_mode mode, ulong *linear)
708 struct desc_struct desc;
715 la = seg_base(ctxt, addr.seg) + addr.ea;
718 case X86EMUL_MODE_PROT64:
720 va_bits = ctxt_virt_addr_bits(ctxt);
721 if (get_canonical(la, va_bits) != la)
724 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
725 if (size > *max_size)
729 *linear = la = (u32)la;
730 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
734 /* code segment in protected mode or read-only data segment */
735 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
736 || !(desc.type & 2)) && write)
738 /* unreadable code segment */
739 if (!fetch && (desc.type & 8) && !(desc.type & 2))
741 lim = desc_limit_scaled(&desc);
742 if (!(desc.type & 8) && (desc.type & 4)) {
743 /* expand-down segment */
746 lim = desc.d ? 0xffffffff : 0xffff;
750 if (lim == 0xffffffff)
753 *max_size = (u64)lim + 1 - addr.ea;
754 if (size > *max_size)
759 if (la & (insn_alignment(ctxt, size) - 1))
760 return emulate_gp(ctxt, 0);
761 return X86EMUL_CONTINUE;
763 if (addr.seg == VCPU_SREG_SS)
764 return emulate_ss(ctxt, 0);
766 return emulate_gp(ctxt, 0);
769 static int linearize(struct x86_emulate_ctxt *ctxt,
770 struct segmented_address addr,
771 unsigned size, bool write,
775 return __linearize(ctxt, addr, &max_size, size, write, false,
779 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)
784 struct segmented_address addr = { .seg = VCPU_SREG_CS,
787 if (ctxt->op_bytes != sizeof(unsigned long))
788 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
789 rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);
790 if (rc == X86EMUL_CONTINUE)
791 ctxt->_eip = addr.ea;
795 static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt)
798 struct desc_struct cs;
802 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
804 if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) {
805 /* Real mode. cpu must not have long mode active */
807 return X86EMUL_UNHANDLEABLE;
808 ctxt->mode = X86EMUL_MODE_REAL;
809 return X86EMUL_CONTINUE;
812 if (ctxt->eflags & X86_EFLAGS_VM) {
813 /* Protected/VM86 mode. cpu must not have long mode active */
815 return X86EMUL_UNHANDLEABLE;
816 ctxt->mode = X86EMUL_MODE_VM86;
817 return X86EMUL_CONTINUE;
820 if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS))
821 return X86EMUL_UNHANDLEABLE;
823 if (efer & EFER_LMA) {
825 /* Proper long mode */
826 ctxt->mode = X86EMUL_MODE_PROT64;
828 /* 32 bit compatibility mode*/
829 ctxt->mode = X86EMUL_MODE_PROT32;
831 ctxt->mode = X86EMUL_MODE_PROT16;
834 /* Legacy 32 bit / 16 bit mode */
835 ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
838 return X86EMUL_CONTINUE;
841 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
843 return assign_eip(ctxt, dst);
846 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst)
848 int rc = emulator_recalc_and_set_mode(ctxt);
850 if (rc != X86EMUL_CONTINUE)
853 return assign_eip(ctxt, dst);
856 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
858 return assign_eip_near(ctxt, ctxt->_eip + rel);
861 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
862 void *data, unsigned size)
864 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
867 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
868 ulong linear, void *data,
871 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
874 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
875 struct segmented_address addr,
882 rc = linearize(ctxt, addr, size, false, &linear);
883 if (rc != X86EMUL_CONTINUE)
885 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
888 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
889 struct segmented_address addr,
896 rc = linearize(ctxt, addr, size, true, &linear);
897 if (rc != X86EMUL_CONTINUE)
899 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
903 * Prefetch the remaining bytes of the instruction without crossing page
904 * boundary if they are not in fetch_cache yet.
906 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
909 unsigned size, max_size;
910 unsigned long linear;
911 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
912 struct segmented_address addr = { .seg = VCPU_SREG_CS,
913 .ea = ctxt->eip + cur_size };
916 * We do not know exactly how many bytes will be needed, and
917 * __linearize is expensive, so fetch as much as possible. We
918 * just have to avoid going beyond the 15 byte limit, the end
919 * of the segment, or the end of the page.
921 * __linearize is called with size 0 so that it does not do any
922 * boundary check itself. Instead, we use max_size to check
925 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
927 if (unlikely(rc != X86EMUL_CONTINUE))
930 size = min_t(unsigned, 15UL ^ cur_size, max_size);
931 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
934 * One instruction can only straddle two pages,
935 * and one has been loaded at the beginning of
936 * x86_decode_insn. So, if not enough bytes
937 * still, we must have hit the 15-byte boundary.
939 if (unlikely(size < op_size))
940 return emulate_gp(ctxt, 0);
942 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
943 size, &ctxt->exception);
944 if (unlikely(rc != X86EMUL_CONTINUE))
946 ctxt->fetch.end += size;
947 return X86EMUL_CONTINUE;
950 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
953 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
955 if (unlikely(done_size < size))
956 return __do_insn_fetch_bytes(ctxt, size - done_size);
958 return X86EMUL_CONTINUE;
961 /* Fetch next part of the instruction being emulated. */
962 #define insn_fetch(_type, _ctxt) \
965 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
966 if (rc != X86EMUL_CONTINUE) \
968 ctxt->_eip += sizeof(_type); \
969 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
970 ctxt->fetch.ptr += sizeof(_type); \
974 #define insn_fetch_arr(_arr, _size, _ctxt) \
976 rc = do_insn_fetch_bytes(_ctxt, _size); \
977 if (rc != X86EMUL_CONTINUE) \
979 ctxt->_eip += (_size); \
980 memcpy(_arr, ctxt->fetch.ptr, _size); \
981 ctxt->fetch.ptr += (_size); \
985 * Given the 'reg' portion of a ModRM byte, and a register block, return a
986 * pointer into the block that addresses the relevant register.
987 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
989 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
993 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
995 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
996 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
998 p = reg_rmw(ctxt, modrm_reg);
1002 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
1003 struct segmented_address addr,
1004 u16 *size, unsigned long *address, int op_bytes)
1011 rc = segmented_read_std(ctxt, addr, size, 2);
1012 if (rc != X86EMUL_CONTINUE)
1015 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1029 FASTOP1SRC2(mul, mul_ex);
1030 FASTOP1SRC2(imul, imul_ex);
1031 FASTOP1SRC2EX(div, div_ex);
1032 FASTOP1SRC2EX(idiv, idiv_ex);
1061 FASTOP2R(cmp, cmp_r);
1063 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1065 /* If src is zero, do not writeback, but update flags */
1066 if (ctxt->src.val == 0)
1067 ctxt->dst.type = OP_NONE;
1068 return fastop(ctxt, em_bsf);
1071 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1073 /* If src is zero, do not writeback, but update flags */
1074 if (ctxt->src.val == 0)
1075 ctxt->dst.type = OP_NONE;
1076 return fastop(ctxt, em_bsr);
1079 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1082 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1084 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1085 asm("push %[flags]; popf; " CALL_NOSPEC
1086 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1090 static void fetch_register_operand(struct operand *op)
1092 switch (op->bytes) {
1094 op->val = *(u8 *)op->addr.reg;
1097 op->val = *(u16 *)op->addr.reg;
1100 op->val = *(u32 *)op->addr.reg;
1103 op->val = *(u64 *)op->addr.reg;
1108 static void emulator_get_fpu(void)
1112 fpregs_assert_state_consistent();
1113 if (test_thread_flag(TIF_NEED_FPU_LOAD))
1114 switch_fpu_return();
1117 static void emulator_put_fpu(void)
1122 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1126 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1127 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1128 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1129 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1130 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1131 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1132 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1133 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1134 #ifdef CONFIG_X86_64
1135 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1136 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1137 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1138 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1139 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1140 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1141 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1142 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1149 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1154 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1155 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1156 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1157 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1158 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1159 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1160 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1161 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1162 #ifdef CONFIG_X86_64
1163 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1164 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1165 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1166 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1167 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1168 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1169 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1170 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1177 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1181 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1182 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1183 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1184 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1185 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1186 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1187 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1188 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1194 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1198 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1199 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1200 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1201 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1202 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1203 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1204 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1205 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1211 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1213 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1214 return emulate_nm(ctxt);
1217 asm volatile("fninit");
1219 return X86EMUL_CONTINUE;
1222 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1226 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1227 return emulate_nm(ctxt);
1230 asm volatile("fnstcw %0": "+m"(fcw));
1233 ctxt->dst.val = fcw;
1235 return X86EMUL_CONTINUE;
1238 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1242 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1243 return emulate_nm(ctxt);
1246 asm volatile("fnstsw %0": "+m"(fsw));
1249 ctxt->dst.val = fsw;
1251 return X86EMUL_CONTINUE;
1254 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1257 unsigned reg = ctxt->modrm_reg;
1259 if (!(ctxt->d & ModRM))
1260 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1262 if (ctxt->d & Sse) {
1266 read_sse_reg(ctxt, &op->vec_val, reg);
1269 if (ctxt->d & Mmx) {
1278 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1279 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1281 fetch_register_operand(op);
1282 op->orig_val = op->val;
1285 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1287 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1288 ctxt->modrm_seg = VCPU_SREG_SS;
1291 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1295 int index_reg, base_reg, scale;
1296 int rc = X86EMUL_CONTINUE;
1299 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1300 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1301 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1303 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1304 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1305 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1306 ctxt->modrm_seg = VCPU_SREG_DS;
1308 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1310 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1311 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1313 if (ctxt->d & Sse) {
1316 op->addr.xmm = ctxt->modrm_rm;
1317 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1320 if (ctxt->d & Mmx) {
1323 op->addr.mm = ctxt->modrm_rm & 7;
1326 fetch_register_operand(op);
1332 if (ctxt->ad_bytes == 2) {
1333 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1334 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1335 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1336 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1338 /* 16-bit ModR/M decode. */
1339 switch (ctxt->modrm_mod) {
1341 if (ctxt->modrm_rm == 6)
1342 modrm_ea += insn_fetch(u16, ctxt);
1345 modrm_ea += insn_fetch(s8, ctxt);
1348 modrm_ea += insn_fetch(u16, ctxt);
1351 switch (ctxt->modrm_rm) {
1353 modrm_ea += bx + si;
1356 modrm_ea += bx + di;
1359 modrm_ea += bp + si;
1362 modrm_ea += bp + di;
1371 if (ctxt->modrm_mod != 0)
1378 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1379 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1380 ctxt->modrm_seg = VCPU_SREG_SS;
1381 modrm_ea = (u16)modrm_ea;
1383 /* 32/64-bit ModR/M decode. */
1384 if ((ctxt->modrm_rm & 7) == 4) {
1385 sib = insn_fetch(u8, ctxt);
1386 index_reg |= (sib >> 3) & 7;
1387 base_reg |= sib & 7;
1390 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1391 modrm_ea += insn_fetch(s32, ctxt);
1393 modrm_ea += reg_read(ctxt, base_reg);
1394 adjust_modrm_seg(ctxt, base_reg);
1395 /* Increment ESP on POP [ESP] */
1396 if ((ctxt->d & IncSP) &&
1397 base_reg == VCPU_REGS_RSP)
1398 modrm_ea += ctxt->op_bytes;
1401 modrm_ea += reg_read(ctxt, index_reg) << scale;
1402 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1403 modrm_ea += insn_fetch(s32, ctxt);
1404 if (ctxt->mode == X86EMUL_MODE_PROT64)
1405 ctxt->rip_relative = 1;
1407 base_reg = ctxt->modrm_rm;
1408 modrm_ea += reg_read(ctxt, base_reg);
1409 adjust_modrm_seg(ctxt, base_reg);
1411 switch (ctxt->modrm_mod) {
1413 modrm_ea += insn_fetch(s8, ctxt);
1416 modrm_ea += insn_fetch(s32, ctxt);
1420 op->addr.mem.ea = modrm_ea;
1421 if (ctxt->ad_bytes != 8)
1422 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1428 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1431 int rc = X86EMUL_CONTINUE;
1434 switch (ctxt->ad_bytes) {
1436 op->addr.mem.ea = insn_fetch(u16, ctxt);
1439 op->addr.mem.ea = insn_fetch(u32, ctxt);
1442 op->addr.mem.ea = insn_fetch(u64, ctxt);
1449 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1453 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1454 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1456 if (ctxt->src.bytes == 2)
1457 sv = (s16)ctxt->src.val & (s16)mask;
1458 else if (ctxt->src.bytes == 4)
1459 sv = (s32)ctxt->src.val & (s32)mask;
1461 sv = (s64)ctxt->src.val & (s64)mask;
1463 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1464 ctxt->dst.addr.mem.ea + (sv >> 3));
1467 /* only subword offset */
1468 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1471 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1472 unsigned long addr, void *dest, unsigned size)
1475 struct read_cache *mc = &ctxt->mem_read;
1477 if (mc->pos < mc->end)
1480 WARN_ON((mc->end + size) >= sizeof(mc->data));
1482 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1484 if (rc != X86EMUL_CONTINUE)
1490 memcpy(dest, mc->data + mc->pos, size);
1492 return X86EMUL_CONTINUE;
1495 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1496 struct segmented_address addr,
1503 rc = linearize(ctxt, addr, size, false, &linear);
1504 if (rc != X86EMUL_CONTINUE)
1506 return read_emulated(ctxt, linear, data, size);
1509 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1510 struct segmented_address addr,
1517 rc = linearize(ctxt, addr, size, true, &linear);
1518 if (rc != X86EMUL_CONTINUE)
1520 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1524 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1525 struct segmented_address addr,
1526 const void *orig_data, const void *data,
1532 rc = linearize(ctxt, addr, size, true, &linear);
1533 if (rc != X86EMUL_CONTINUE)
1535 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1536 size, &ctxt->exception);
1539 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1540 unsigned int size, unsigned short port,
1543 struct read_cache *rc = &ctxt->io_read;
1545 if (rc->pos == rc->end) { /* refill pio read ahead */
1546 unsigned int in_page, n;
1547 unsigned int count = ctxt->rep_prefix ?
1548 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1549 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1550 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1551 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1552 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1555 rc->pos = rc->end = 0;
1556 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1561 if (ctxt->rep_prefix && (ctxt->d & String) &&
1562 !(ctxt->eflags & X86_EFLAGS_DF)) {
1563 ctxt->dst.data = rc->data + rc->pos;
1564 ctxt->dst.type = OP_MEM_STR;
1565 ctxt->dst.count = (rc->end - rc->pos) / size;
1568 memcpy(dest, rc->data + rc->pos, size);
1574 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1575 u16 index, struct desc_struct *desc)
1580 ctxt->ops->get_idt(ctxt, &dt);
1582 if (dt.size < index * 8 + 7)
1583 return emulate_gp(ctxt, index << 3 | 0x2);
1585 addr = dt.address + index * 8;
1586 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1589 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1590 u16 selector, struct desc_ptr *dt)
1592 const struct x86_emulate_ops *ops = ctxt->ops;
1595 if (selector & 1 << 2) {
1596 struct desc_struct desc;
1599 memset(dt, 0, sizeof(*dt));
1600 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1604 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1605 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1607 ops->get_gdt(ctxt, dt);
1610 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1611 u16 selector, ulong *desc_addr_p)
1614 u16 index = selector >> 3;
1617 get_descriptor_table_ptr(ctxt, selector, &dt);
1619 if (dt.size < index * 8 + 7)
1620 return emulate_gp(ctxt, selector & 0xfffc);
1622 addr = dt.address + index * 8;
1624 #ifdef CONFIG_X86_64
1625 if (addr >> 32 != 0) {
1628 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1629 if (!(efer & EFER_LMA))
1634 *desc_addr_p = addr;
1635 return X86EMUL_CONTINUE;
1638 /* allowed just for 8 bytes segments */
1639 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1640 u16 selector, struct desc_struct *desc,
1645 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1646 if (rc != X86EMUL_CONTINUE)
1649 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1652 /* allowed just for 8 bytes segments */
1653 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1654 u16 selector, struct desc_struct *desc)
1659 rc = get_descriptor_ptr(ctxt, selector, &addr);
1660 if (rc != X86EMUL_CONTINUE)
1663 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1666 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1667 u16 selector, int seg, u8 cpl,
1668 enum x86_transfer_type transfer,
1669 struct desc_struct *desc)
1671 struct desc_struct seg_desc, old_desc;
1673 unsigned err_vec = GP_VECTOR;
1675 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1681 memset(&seg_desc, 0, sizeof(seg_desc));
1683 if (ctxt->mode == X86EMUL_MODE_REAL) {
1684 /* set real mode segment descriptor (keep limit etc. for
1686 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1687 set_desc_base(&seg_desc, selector << 4);
1689 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1690 /* VM86 needs a clean new segment descriptor */
1691 set_desc_base(&seg_desc, selector << 4);
1692 set_desc_limit(&seg_desc, 0xffff);
1702 /* TR should be in GDT only */
1703 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1706 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1707 if (null_selector) {
1708 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1711 if (seg == VCPU_SREG_SS) {
1712 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1716 * ctxt->ops->set_segment expects the CPL to be in
1717 * SS.DPL, so fake an expand-up 32-bit data segment.
1727 /* Skip all following checks */
1731 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1732 if (ret != X86EMUL_CONTINUE)
1735 err_code = selector & 0xfffc;
1736 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1739 /* can't load system descriptor into segment selector */
1740 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1741 if (transfer == X86_TRANSFER_CALL_JMP)
1742 return X86EMUL_UNHANDLEABLE;
1751 * segment is not a writable data segment or segment
1752 * selector's RPL != CPL or segment selector's RPL != CPL
1754 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1758 if (!(seg_desc.type & 8))
1761 if (seg_desc.type & 4) {
1767 if (rpl > cpl || dpl != cpl)
1770 /* in long-mode d/b must be clear if l is set */
1771 if (seg_desc.d && seg_desc.l) {
1774 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1775 if (efer & EFER_LMA)
1779 /* CS(RPL) <- CPL */
1780 selector = (selector & 0xfffc) | cpl;
1783 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1786 case VCPU_SREG_LDTR:
1787 if (seg_desc.s || seg_desc.type != 2)
1790 default: /* DS, ES, FS, or GS */
1792 * segment is not a data or readable code segment or
1793 * ((segment is a data or nonconforming code segment)
1794 * and (both RPL and CPL > DPL))
1796 if ((seg_desc.type & 0xa) == 0x8 ||
1797 (((seg_desc.type & 0xc) != 0xc) &&
1798 (rpl > dpl && cpl > dpl)))
1804 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1809 /* mark segment as accessed */
1810 if (!(seg_desc.type & 1)) {
1812 ret = write_segment_descriptor(ctxt, selector,
1814 if (ret != X86EMUL_CONTINUE)
1817 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1818 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1819 if (ret != X86EMUL_CONTINUE)
1821 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1822 ((u64)base3 << 32), ctxt))
1823 return emulate_gp(ctxt, err_code);
1826 if (seg == VCPU_SREG_TR) {
1827 old_desc = seg_desc;
1828 seg_desc.type |= 2; /* busy */
1829 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1830 sizeof(seg_desc), &ctxt->exception);
1831 if (ret != X86EMUL_CONTINUE)
1835 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1838 return X86EMUL_CONTINUE;
1840 return emulate_exception(ctxt, err_vec, err_code, true);
1843 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1844 u16 selector, int seg)
1846 u8 cpl = ctxt->ops->cpl(ctxt);
1849 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1850 * they can load it at CPL<3 (Intel's manual says only LSS can,
1853 * However, the Intel manual says that putting IST=1/DPL=3 in
1854 * an interrupt gate will result in SS=3 (the AMD manual instead
1855 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1856 * and only forbid it here.
1858 if (seg == VCPU_SREG_SS && selector == 3 &&
1859 ctxt->mode == X86EMUL_MODE_PROT64)
1860 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1862 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1863 X86_TRANSFER_NONE, NULL);
1866 static void write_register_operand(struct operand *op)
1868 return assign_register(op->addr.reg, op->val, op->bytes);
1871 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1875 write_register_operand(op);
1878 if (ctxt->lock_prefix)
1879 return segmented_cmpxchg(ctxt,
1885 return segmented_write(ctxt,
1891 return segmented_write(ctxt,
1894 op->bytes * op->count);
1897 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1900 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1908 return X86EMUL_CONTINUE;
1911 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1913 struct segmented_address addr;
1915 rsp_increment(ctxt, -bytes);
1916 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1917 addr.seg = VCPU_SREG_SS;
1919 return segmented_write(ctxt, addr, data, bytes);
1922 static int em_push(struct x86_emulate_ctxt *ctxt)
1924 /* Disable writeback. */
1925 ctxt->dst.type = OP_NONE;
1926 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1929 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1930 void *dest, int len)
1933 struct segmented_address addr;
1935 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1936 addr.seg = VCPU_SREG_SS;
1937 rc = segmented_read(ctxt, addr, dest, len);
1938 if (rc != X86EMUL_CONTINUE)
1941 rsp_increment(ctxt, len);
1945 static int em_pop(struct x86_emulate_ctxt *ctxt)
1947 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1950 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1951 void *dest, int len)
1954 unsigned long val, change_mask;
1955 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1956 int cpl = ctxt->ops->cpl(ctxt);
1958 rc = emulate_pop(ctxt, &val, len);
1959 if (rc != X86EMUL_CONTINUE)
1962 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1963 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1964 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1965 X86_EFLAGS_AC | X86_EFLAGS_ID;
1967 switch(ctxt->mode) {
1968 case X86EMUL_MODE_PROT64:
1969 case X86EMUL_MODE_PROT32:
1970 case X86EMUL_MODE_PROT16:
1972 change_mask |= X86_EFLAGS_IOPL;
1974 change_mask |= X86_EFLAGS_IF;
1976 case X86EMUL_MODE_VM86:
1978 return emulate_gp(ctxt, 0);
1979 change_mask |= X86_EFLAGS_IF;
1981 default: /* real mode */
1982 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1986 *(unsigned long *)dest =
1987 (ctxt->eflags & ~change_mask) | (val & change_mask);
1992 static int em_popf(struct x86_emulate_ctxt *ctxt)
1994 ctxt->dst.type = OP_REG;
1995 ctxt->dst.addr.reg = &ctxt->eflags;
1996 ctxt->dst.bytes = ctxt->op_bytes;
1997 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2000 static int em_enter(struct x86_emulate_ctxt *ctxt)
2003 unsigned frame_size = ctxt->src.val;
2004 unsigned nesting_level = ctxt->src2.val & 31;
2008 return X86EMUL_UNHANDLEABLE;
2010 rbp = reg_read(ctxt, VCPU_REGS_RBP);
2011 rc = push(ctxt, &rbp, stack_size(ctxt));
2012 if (rc != X86EMUL_CONTINUE)
2014 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
2016 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
2017 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
2019 return X86EMUL_CONTINUE;
2022 static int em_leave(struct x86_emulate_ctxt *ctxt)
2024 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
2026 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
2029 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
2031 int seg = ctxt->src2.val;
2033 ctxt->src.val = get_segment_selector(ctxt, seg);
2034 if (ctxt->op_bytes == 4) {
2035 rsp_increment(ctxt, -2);
2039 return em_push(ctxt);
2042 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
2044 int seg = ctxt->src2.val;
2045 unsigned long selector;
2048 rc = emulate_pop(ctxt, &selector, 2);
2049 if (rc != X86EMUL_CONTINUE)
2052 if (seg == VCPU_SREG_SS)
2053 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2054 if (ctxt->op_bytes > 2)
2055 rsp_increment(ctxt, ctxt->op_bytes - 2);
2057 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2061 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2063 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2064 int rc = X86EMUL_CONTINUE;
2065 int reg = VCPU_REGS_RAX;
2067 while (reg <= VCPU_REGS_RDI) {
2068 (reg == VCPU_REGS_RSP) ?
2069 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2072 if (rc != X86EMUL_CONTINUE)
2081 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2083 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2084 return em_push(ctxt);
2087 static int em_popa(struct x86_emulate_ctxt *ctxt)
2089 int rc = X86EMUL_CONTINUE;
2090 int reg = VCPU_REGS_RDI;
2093 while (reg >= VCPU_REGS_RAX) {
2094 if (reg == VCPU_REGS_RSP) {
2095 rsp_increment(ctxt, ctxt->op_bytes);
2099 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2100 if (rc != X86EMUL_CONTINUE)
2102 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2108 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2110 const struct x86_emulate_ops *ops = ctxt->ops;
2117 /* TODO: Add limit checks */
2118 ctxt->src.val = ctxt->eflags;
2120 if (rc != X86EMUL_CONTINUE)
2123 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2125 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2127 if (rc != X86EMUL_CONTINUE)
2130 ctxt->src.val = ctxt->_eip;
2132 if (rc != X86EMUL_CONTINUE)
2135 ops->get_idt(ctxt, &dt);
2137 eip_addr = dt.address + (irq << 2);
2138 cs_addr = dt.address + (irq << 2) + 2;
2140 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2141 if (rc != X86EMUL_CONTINUE)
2144 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2145 if (rc != X86EMUL_CONTINUE)
2148 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2149 if (rc != X86EMUL_CONTINUE)
2157 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2161 invalidate_registers(ctxt);
2162 rc = __emulate_int_real(ctxt, irq);
2163 if (rc == X86EMUL_CONTINUE)
2164 writeback_registers(ctxt);
2168 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2170 switch(ctxt->mode) {
2171 case X86EMUL_MODE_REAL:
2172 return __emulate_int_real(ctxt, irq);
2173 case X86EMUL_MODE_VM86:
2174 case X86EMUL_MODE_PROT16:
2175 case X86EMUL_MODE_PROT32:
2176 case X86EMUL_MODE_PROT64:
2178 /* Protected mode interrupts unimplemented yet */
2179 return X86EMUL_UNHANDLEABLE;
2183 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2185 int rc = X86EMUL_CONTINUE;
2186 unsigned long temp_eip = 0;
2187 unsigned long temp_eflags = 0;
2188 unsigned long cs = 0;
2189 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2190 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2191 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2192 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2193 X86_EFLAGS_AC | X86_EFLAGS_ID |
2195 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2198 /* TODO: Add stack limit check */
2200 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2202 if (rc != X86EMUL_CONTINUE)
2205 if (temp_eip & ~0xffff)
2206 return emulate_gp(ctxt, 0);
2208 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2210 if (rc != X86EMUL_CONTINUE)
2213 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2215 if (rc != X86EMUL_CONTINUE)
2218 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2220 if (rc != X86EMUL_CONTINUE)
2223 ctxt->_eip = temp_eip;
2225 if (ctxt->op_bytes == 4)
2226 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2227 else if (ctxt->op_bytes == 2) {
2228 ctxt->eflags &= ~0xffff;
2229 ctxt->eflags |= temp_eflags;
2232 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2233 ctxt->eflags |= X86_EFLAGS_FIXED;
2234 ctxt->ops->set_nmi_mask(ctxt, false);
2239 static int em_iret(struct x86_emulate_ctxt *ctxt)
2241 switch(ctxt->mode) {
2242 case X86EMUL_MODE_REAL:
2243 return emulate_iret_real(ctxt);
2244 case X86EMUL_MODE_VM86:
2245 case X86EMUL_MODE_PROT16:
2246 case X86EMUL_MODE_PROT32:
2247 case X86EMUL_MODE_PROT64:
2249 /* iret from protected mode unimplemented yet */
2250 return X86EMUL_UNHANDLEABLE;
2254 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2258 struct desc_struct new_desc;
2259 u8 cpl = ctxt->ops->cpl(ctxt);
2261 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2263 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2264 X86_TRANSFER_CALL_JMP,
2266 if (rc != X86EMUL_CONTINUE)
2269 rc = assign_eip_far(ctxt, ctxt->src.val);
2270 /* Error handling is not implemented. */
2271 if (rc != X86EMUL_CONTINUE)
2272 return X86EMUL_UNHANDLEABLE;
2277 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2279 return assign_eip_near(ctxt, ctxt->src.val);
2282 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2287 old_eip = ctxt->_eip;
2288 rc = assign_eip_near(ctxt, ctxt->src.val);
2289 if (rc != X86EMUL_CONTINUE)
2291 ctxt->src.val = old_eip;
2296 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2298 u64 old = ctxt->dst.orig_val64;
2300 if (ctxt->dst.bytes == 16)
2301 return X86EMUL_UNHANDLEABLE;
2303 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2304 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2305 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2306 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2307 ctxt->eflags &= ~X86_EFLAGS_ZF;
2309 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2310 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2312 ctxt->eflags |= X86_EFLAGS_ZF;
2314 return X86EMUL_CONTINUE;
2317 static int em_ret(struct x86_emulate_ctxt *ctxt)
2322 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2323 if (rc != X86EMUL_CONTINUE)
2326 return assign_eip_near(ctxt, eip);
2329 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2332 unsigned long eip, cs;
2333 int cpl = ctxt->ops->cpl(ctxt);
2334 struct desc_struct new_desc;
2336 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2337 if (rc != X86EMUL_CONTINUE)
2339 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2340 if (rc != X86EMUL_CONTINUE)
2342 /* Outer-privilege level return is not implemented */
2343 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2344 return X86EMUL_UNHANDLEABLE;
2345 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2348 if (rc != X86EMUL_CONTINUE)
2350 rc = assign_eip_far(ctxt, eip);
2351 /* Error handling is not implemented. */
2352 if (rc != X86EMUL_CONTINUE)
2353 return X86EMUL_UNHANDLEABLE;
2358 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2362 rc = em_ret_far(ctxt);
2363 if (rc != X86EMUL_CONTINUE)
2365 rsp_increment(ctxt, ctxt->src.val);
2366 return X86EMUL_CONTINUE;
2369 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2371 /* Save real source value, then compare EAX against destination. */
2372 ctxt->dst.orig_val = ctxt->dst.val;
2373 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2374 ctxt->src.orig_val = ctxt->src.val;
2375 ctxt->src.val = ctxt->dst.orig_val;
2376 fastop(ctxt, em_cmp);
2378 if (ctxt->eflags & X86_EFLAGS_ZF) {
2379 /* Success: write back to memory; no update of EAX */
2380 ctxt->src.type = OP_NONE;
2381 ctxt->dst.val = ctxt->src.orig_val;
2383 /* Failure: write the value we saw to EAX. */
2384 ctxt->src.type = OP_REG;
2385 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2386 ctxt->src.val = ctxt->dst.orig_val;
2387 /* Create write-cycle to dest by writing the same value */
2388 ctxt->dst.val = ctxt->dst.orig_val;
2390 return X86EMUL_CONTINUE;
2393 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2395 int seg = ctxt->src2.val;
2399 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2401 rc = load_segment_descriptor(ctxt, sel, seg);
2402 if (rc != X86EMUL_CONTINUE)
2405 ctxt->dst.val = ctxt->src.val;
2409 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2411 #ifdef CONFIG_X86_64
2412 u32 eax, ebx, ecx, edx;
2416 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2417 return edx & bit(X86_FEATURE_LM);
2423 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2425 desc->g = (flags >> 23) & 1;
2426 desc->d = (flags >> 22) & 1;
2427 desc->l = (flags >> 21) & 1;
2428 desc->avl = (flags >> 20) & 1;
2429 desc->p = (flags >> 15) & 1;
2430 desc->dpl = (flags >> 13) & 3;
2431 desc->s = (flags >> 12) & 1;
2432 desc->type = (flags >> 8) & 15;
2435 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2438 struct desc_struct desc;
2442 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2445 offset = 0x7f84 + n * 12;
2447 offset = 0x7f2c + (n - 3) * 12;
2449 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2450 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2451 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2452 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2453 return X86EMUL_CONTINUE;
2456 #ifdef CONFIG_X86_64
2457 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2460 struct desc_struct desc;
2465 offset = 0x7e00 + n * 16;
2467 selector = GET_SMSTATE(u16, smstate, offset);
2468 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2469 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2470 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2471 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2473 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2474 return X86EMUL_CONTINUE;
2478 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2479 u64 cr0, u64 cr3, u64 cr4)
2484 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2486 if (cr4 & X86_CR4_PCIDE) {
2491 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2493 return X86EMUL_UNHANDLEABLE;
2496 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2497 * Then enable protected mode. However, PCID cannot be enabled
2498 * if EFER.LMA=0, so set it separately.
2500 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2502 return X86EMUL_UNHANDLEABLE;
2504 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2506 return X86EMUL_UNHANDLEABLE;
2508 if (cr4 & X86_CR4_PCIDE) {
2509 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2511 return X86EMUL_UNHANDLEABLE;
2513 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2515 return X86EMUL_UNHANDLEABLE;
2520 return X86EMUL_CONTINUE;
2523 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2524 const char *smstate)
2526 struct desc_struct desc;
2529 u32 val, cr0, cr3, cr4;
2532 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2533 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2534 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2535 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2537 for (i = 0; i < 8; i++)
2538 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2540 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2541 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2542 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2543 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2545 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2546 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2547 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2548 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2549 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2551 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2552 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2553 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2554 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2555 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2557 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2558 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2559 ctxt->ops->set_gdt(ctxt, &dt);
2561 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2562 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2563 ctxt->ops->set_idt(ctxt, &dt);
2565 for (i = 0; i < 6; i++) {
2566 int r = rsm_load_seg_32(ctxt, smstate, i);
2567 if (r != X86EMUL_CONTINUE)
2571 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2573 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2575 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2578 #ifdef CONFIG_X86_64
2579 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2580 const char *smstate)
2582 struct desc_struct desc;
2584 u64 val, cr0, cr3, cr4;
2589 for (i = 0; i < 16; i++)
2590 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2592 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2593 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2595 val = GET_SMSTATE(u32, smstate, 0x7f68);
2596 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2597 val = GET_SMSTATE(u32, smstate, 0x7f60);
2598 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2600 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2601 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2602 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2603 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2604 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2605 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2607 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2608 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2609 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2610 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2611 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2612 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2614 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2615 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2616 ctxt->ops->set_idt(ctxt, &dt);
2618 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2619 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2620 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2621 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2622 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2623 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2625 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2626 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2627 ctxt->ops->set_gdt(ctxt, &dt);
2629 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2630 if (r != X86EMUL_CONTINUE)
2633 for (i = 0; i < 6; i++) {
2634 r = rsm_load_seg_64(ctxt, smstate, i);
2635 if (r != X86EMUL_CONTINUE)
2639 return X86EMUL_CONTINUE;
2643 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2645 unsigned long cr0, cr4, efer;
2650 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2651 return emulate_ud(ctxt);
2653 smbase = ctxt->ops->get_smbase(ctxt);
2655 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2656 if (ret != X86EMUL_CONTINUE)
2657 return X86EMUL_UNHANDLEABLE;
2659 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2660 ctxt->ops->set_nmi_mask(ctxt, false);
2662 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2663 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2666 * Get back to real mode, to prepare a safe state in which to load
2667 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2668 * supports long mode.
2670 if (emulator_has_longmode(ctxt)) {
2671 struct desc_struct cs_desc;
2673 /* Zero CR4.PCIDE before CR0.PG. */
2674 cr4 = ctxt->ops->get_cr(ctxt, 4);
2675 if (cr4 & X86_CR4_PCIDE)
2676 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2678 /* A 32-bit code segment is required to clear EFER.LMA. */
2679 memset(&cs_desc, 0, sizeof(cs_desc));
2681 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2682 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2685 /* For the 64-bit case, this will clear EFER.LMA. */
2686 cr0 = ctxt->ops->get_cr(ctxt, 0);
2687 if (cr0 & X86_CR0_PE)
2688 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2690 if (emulator_has_longmode(ctxt)) {
2691 /* Clear CR4.PAE before clearing EFER.LME. */
2692 cr4 = ctxt->ops->get_cr(ctxt, 4);
2693 if (cr4 & X86_CR4_PAE)
2694 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2696 /* And finally go back to 32-bit mode. */
2698 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2702 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2703 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2706 if (ctxt->ops->pre_leave_smm(ctxt, buf))
2707 return X86EMUL_UNHANDLEABLE;
2709 #ifdef CONFIG_X86_64
2710 if (emulator_has_longmode(ctxt))
2711 ret = rsm_load_state_64(ctxt, buf);
2714 ret = rsm_load_state_32(ctxt, buf);
2716 if (ret != X86EMUL_CONTINUE) {
2717 /* FIXME: should triple fault */
2718 return X86EMUL_UNHANDLEABLE;
2721 ctxt->ops->post_leave_smm(ctxt);
2723 return X86EMUL_CONTINUE;
2727 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2728 struct desc_struct *cs, struct desc_struct *ss)
2730 cs->l = 0; /* will be adjusted later */
2731 set_desc_base(cs, 0); /* flat segment */
2732 cs->g = 1; /* 4kb granularity */
2733 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2734 cs->type = 0x0b; /* Read, Execute, Accessed */
2736 cs->dpl = 0; /* will be adjusted later */
2741 set_desc_base(ss, 0); /* flat segment */
2742 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2743 ss->g = 1; /* 4kb granularity */
2745 ss->type = 0x03; /* Read/Write, Accessed */
2746 ss->d = 1; /* 32bit stack segment */
2753 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2755 u32 eax, ebx, ecx, edx;
2758 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2759 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2760 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2761 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2764 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2766 const struct x86_emulate_ops *ops = ctxt->ops;
2767 u32 eax, ebx, ecx, edx;
2770 * syscall should always be enabled in longmode - so only become
2771 * vendor specific (cpuid) if other modes are active...
2773 if (ctxt->mode == X86EMUL_MODE_PROT64)
2778 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2780 * Intel ("GenuineIntel")
2781 * remark: Intel CPUs only support "syscall" in 64bit
2782 * longmode. Also an 64bit guest with a
2783 * 32bit compat-app running will #UD !! While this
2784 * behaviour can be fixed (by emulating) into AMD
2785 * response - CPUs of AMD can't behave like Intel.
2787 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2788 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2789 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2792 /* AMD ("AuthenticAMD") */
2793 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2794 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2795 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2798 /* AMD ("AMDisbetter!") */
2799 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2800 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2801 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2804 /* Hygon ("HygonGenuine") */
2805 if (ebx == X86EMUL_CPUID_VENDOR_HygonGenuine_ebx &&
2806 ecx == X86EMUL_CPUID_VENDOR_HygonGenuine_ecx &&
2807 edx == X86EMUL_CPUID_VENDOR_HygonGenuine_edx)
2811 * default: (not Intel, not AMD, not Hygon), apply Intel's
2817 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2819 const struct x86_emulate_ops *ops = ctxt->ops;
2820 struct desc_struct cs, ss;
2825 /* syscall is not available in real mode */
2826 if (ctxt->mode == X86EMUL_MODE_REAL ||
2827 ctxt->mode == X86EMUL_MODE_VM86)
2828 return emulate_ud(ctxt);
2830 if (!(em_syscall_is_enabled(ctxt)))
2831 return emulate_ud(ctxt);
2833 ops->get_msr(ctxt, MSR_EFER, &efer);
2834 setup_syscalls_segments(ctxt, &cs, &ss);
2836 if (!(efer & EFER_SCE))
2837 return emulate_ud(ctxt);
2839 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2841 cs_sel = (u16)(msr_data & 0xfffc);
2842 ss_sel = (u16)(msr_data + 8);
2844 if (efer & EFER_LMA) {
2848 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2849 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2851 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2852 if (efer & EFER_LMA) {
2853 #ifdef CONFIG_X86_64
2854 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2857 ctxt->mode == X86EMUL_MODE_PROT64 ?
2858 MSR_LSTAR : MSR_CSTAR, &msr_data);
2859 ctxt->_eip = msr_data;
2861 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2862 ctxt->eflags &= ~msr_data;
2863 ctxt->eflags |= X86_EFLAGS_FIXED;
2867 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2868 ctxt->_eip = (u32)msr_data;
2870 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2873 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2874 return X86EMUL_CONTINUE;
2877 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2879 const struct x86_emulate_ops *ops = ctxt->ops;
2880 struct desc_struct cs, ss;
2885 ops->get_msr(ctxt, MSR_EFER, &efer);
2886 /* inject #GP if in real mode */
2887 if (ctxt->mode == X86EMUL_MODE_REAL)
2888 return emulate_gp(ctxt, 0);
2891 * Not recognized on AMD in compat mode (but is recognized in legacy
2894 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2895 && !vendor_intel(ctxt))
2896 return emulate_ud(ctxt);
2898 /* sysenter/sysexit have not been tested in 64bit mode. */
2899 if (ctxt->mode == X86EMUL_MODE_PROT64)
2900 return X86EMUL_UNHANDLEABLE;
2902 setup_syscalls_segments(ctxt, &cs, &ss);
2904 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2905 if ((msr_data & 0xfffc) == 0x0)
2906 return emulate_gp(ctxt, 0);
2908 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2909 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2910 ss_sel = cs_sel + 8;
2911 if (efer & EFER_LMA) {
2916 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2917 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2919 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2920 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2922 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2923 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2925 if (efer & EFER_LMA)
2926 ctxt->mode = X86EMUL_MODE_PROT64;
2928 return X86EMUL_CONTINUE;
2931 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2933 const struct x86_emulate_ops *ops = ctxt->ops;
2934 struct desc_struct cs, ss;
2935 u64 msr_data, rcx, rdx;
2937 u16 cs_sel = 0, ss_sel = 0;
2939 /* inject #GP if in real mode or Virtual 8086 mode */
2940 if (ctxt->mode == X86EMUL_MODE_REAL ||
2941 ctxt->mode == X86EMUL_MODE_VM86)
2942 return emulate_gp(ctxt, 0);
2944 setup_syscalls_segments(ctxt, &cs, &ss);
2946 if ((ctxt->rex_prefix & 0x8) != 0x0)
2947 usermode = X86EMUL_MODE_PROT64;
2949 usermode = X86EMUL_MODE_PROT32;
2951 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2952 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2956 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2958 case X86EMUL_MODE_PROT32:
2959 cs_sel = (u16)(msr_data + 16);
2960 if ((msr_data & 0xfffc) == 0x0)
2961 return emulate_gp(ctxt, 0);
2962 ss_sel = (u16)(msr_data + 24);
2966 case X86EMUL_MODE_PROT64:
2967 cs_sel = (u16)(msr_data + 32);
2968 if (msr_data == 0x0)
2969 return emulate_gp(ctxt, 0);
2970 ss_sel = cs_sel + 8;
2973 if (emul_is_noncanonical_address(rcx, ctxt) ||
2974 emul_is_noncanonical_address(rdx, ctxt))
2975 return emulate_gp(ctxt, 0);
2978 cs_sel |= SEGMENT_RPL_MASK;
2979 ss_sel |= SEGMENT_RPL_MASK;
2981 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2982 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2985 ctxt->mode = usermode;
2986 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2988 return X86EMUL_CONTINUE;
2991 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2994 if (ctxt->mode == X86EMUL_MODE_REAL)
2996 if (ctxt->mode == X86EMUL_MODE_VM86)
2998 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2999 return ctxt->ops->cpl(ctxt) > iopl;
3002 #define VMWARE_PORT_VMPORT (0x5658)
3003 #define VMWARE_PORT_VMRPC (0x5659)
3005 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
3008 const struct x86_emulate_ops *ops = ctxt->ops;
3009 struct desc_struct tr_seg;
3012 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
3013 unsigned mask = (1 << len) - 1;
3017 * VMware allows access to these ports even if denied
3018 * by TSS I/O permission bitmap. Mimic behavior.
3020 if (enable_vmware_backdoor &&
3021 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
3024 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
3027 if (desc_limit_scaled(&tr_seg) < 103)
3029 base = get_desc_base(&tr_seg);
3030 #ifdef CONFIG_X86_64
3031 base |= ((u64)base3) << 32;
3033 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
3034 if (r != X86EMUL_CONTINUE)
3036 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
3038 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
3039 if (r != X86EMUL_CONTINUE)
3041 if ((perm >> bit_idx) & mask)
3046 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
3052 if (emulator_bad_iopl(ctxt))
3053 if (!emulator_io_port_access_allowed(ctxt, port, len))
3056 ctxt->perm_ok = true;
3061 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
3064 * Intel CPUs mask the counter and pointers in quite strange
3065 * manner when ECX is zero due to REP-string optimizations.
3067 #ifdef CONFIG_X86_64
3068 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
3071 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
3074 case 0xa4: /* movsb */
3075 case 0xa5: /* movsd/w */
3076 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3078 case 0xaa: /* stosb */
3079 case 0xab: /* stosd/w */
3080 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3085 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3086 struct tss_segment_16 *tss)
3088 tss->ip = ctxt->_eip;
3089 tss->flag = ctxt->eflags;
3090 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3091 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3092 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3093 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3094 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3095 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3096 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3097 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3099 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3100 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3101 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3102 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3103 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3106 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3107 struct tss_segment_16 *tss)
3112 ctxt->_eip = tss->ip;
3113 ctxt->eflags = tss->flag | 2;
3114 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3115 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3116 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3117 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3118 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3119 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3120 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3121 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3124 * SDM says that segment selectors are loaded before segment
3127 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3128 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3129 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3130 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3131 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3136 * Now load segment descriptors. If fault happens at this stage
3137 * it is handled in a context of new task
3139 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3140 X86_TRANSFER_TASK_SWITCH, NULL);
3141 if (ret != X86EMUL_CONTINUE)
3143 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3144 X86_TRANSFER_TASK_SWITCH, NULL);
3145 if (ret != X86EMUL_CONTINUE)
3147 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3148 X86_TRANSFER_TASK_SWITCH, NULL);
3149 if (ret != X86EMUL_CONTINUE)
3151 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3152 X86_TRANSFER_TASK_SWITCH, NULL);
3153 if (ret != X86EMUL_CONTINUE)
3155 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3156 X86_TRANSFER_TASK_SWITCH, NULL);
3157 if (ret != X86EMUL_CONTINUE)
3160 return X86EMUL_CONTINUE;
3163 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3164 u16 tss_selector, u16 old_tss_sel,
3165 ulong old_tss_base, struct desc_struct *new_desc)
3167 struct tss_segment_16 tss_seg;
3169 u32 new_tss_base = get_desc_base(new_desc);
3171 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3172 if (ret != X86EMUL_CONTINUE)
3175 save_state_to_tss16(ctxt, &tss_seg);
3177 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3178 if (ret != X86EMUL_CONTINUE)
3181 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3182 if (ret != X86EMUL_CONTINUE)
3185 if (old_tss_sel != 0xffff) {
3186 tss_seg.prev_task_link = old_tss_sel;
3188 ret = linear_write_system(ctxt, new_tss_base,
3189 &tss_seg.prev_task_link,
3190 sizeof(tss_seg.prev_task_link));
3191 if (ret != X86EMUL_CONTINUE)
3195 return load_state_from_tss16(ctxt, &tss_seg);
3198 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3199 struct tss_segment_32 *tss)
3201 /* CR3 and ldt selector are not saved intentionally */
3202 tss->eip = ctxt->_eip;
3203 tss->eflags = ctxt->eflags;
3204 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3205 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3206 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3207 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3208 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3209 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3210 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3211 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3213 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3214 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3215 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3216 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3217 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3218 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3221 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3222 struct tss_segment_32 *tss)
3227 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3228 return emulate_gp(ctxt, 0);
3229 ctxt->_eip = tss->eip;
3230 ctxt->eflags = tss->eflags | 2;
3232 /* General purpose registers */
3233 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3234 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3235 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3236 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3237 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3238 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3239 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3240 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3243 * SDM says that segment selectors are loaded before segment
3244 * descriptors. This is important because CPL checks will
3247 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3248 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3249 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3250 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3251 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3252 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3253 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3256 * If we're switching between Protected Mode and VM86, we need to make
3257 * sure to update the mode before loading the segment descriptors so
3258 * that the selectors are interpreted correctly.
3260 if (ctxt->eflags & X86_EFLAGS_VM) {
3261 ctxt->mode = X86EMUL_MODE_VM86;
3264 ctxt->mode = X86EMUL_MODE_PROT32;
3269 * Now load segment descriptors. If fault happenes at this stage
3270 * it is handled in a context of new task
3272 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3273 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3274 if (ret != X86EMUL_CONTINUE)
3276 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3277 X86_TRANSFER_TASK_SWITCH, NULL);
3278 if (ret != X86EMUL_CONTINUE)
3280 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3281 X86_TRANSFER_TASK_SWITCH, NULL);
3282 if (ret != X86EMUL_CONTINUE)
3284 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3285 X86_TRANSFER_TASK_SWITCH, NULL);
3286 if (ret != X86EMUL_CONTINUE)
3288 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3289 X86_TRANSFER_TASK_SWITCH, NULL);
3290 if (ret != X86EMUL_CONTINUE)
3292 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3293 X86_TRANSFER_TASK_SWITCH, NULL);
3294 if (ret != X86EMUL_CONTINUE)
3296 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3297 X86_TRANSFER_TASK_SWITCH, NULL);
3302 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3303 u16 tss_selector, u16 old_tss_sel,
3304 ulong old_tss_base, struct desc_struct *new_desc)
3306 struct tss_segment_32 tss_seg;
3308 u32 new_tss_base = get_desc_base(new_desc);
3309 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3310 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3312 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3313 if (ret != X86EMUL_CONTINUE)
3316 save_state_to_tss32(ctxt, &tss_seg);
3318 /* Only GP registers and segment selectors are saved */
3319 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3320 ldt_sel_offset - eip_offset);
3321 if (ret != X86EMUL_CONTINUE)
3324 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3325 if (ret != X86EMUL_CONTINUE)
3328 if (old_tss_sel != 0xffff) {
3329 tss_seg.prev_task_link = old_tss_sel;
3331 ret = linear_write_system(ctxt, new_tss_base,
3332 &tss_seg.prev_task_link,
3333 sizeof(tss_seg.prev_task_link));
3334 if (ret != X86EMUL_CONTINUE)
3338 return load_state_from_tss32(ctxt, &tss_seg);
3341 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3342 u16 tss_selector, int idt_index, int reason,
3343 bool has_error_code, u32 error_code)
3345 const struct x86_emulate_ops *ops = ctxt->ops;
3346 struct desc_struct curr_tss_desc, next_tss_desc;
3348 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3349 ulong old_tss_base =
3350 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3352 ulong desc_addr, dr7;
3354 /* FIXME: old_tss_base == ~0 ? */
3356 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3357 if (ret != X86EMUL_CONTINUE)
3359 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3360 if (ret != X86EMUL_CONTINUE)
3363 /* FIXME: check that next_tss_desc is tss */
3366 * Check privileges. The three cases are task switch caused by...
3368 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3369 * 2. Exception/IRQ/iret: No check is performed
3370 * 3. jmp/call to TSS/task-gate: No check is performed since the
3371 * hardware checks it before exiting.
3373 if (reason == TASK_SWITCH_GATE) {
3374 if (idt_index != -1) {
3375 /* Software interrupts */
3376 struct desc_struct task_gate_desc;
3379 ret = read_interrupt_descriptor(ctxt, idt_index,
3381 if (ret != X86EMUL_CONTINUE)
3384 dpl = task_gate_desc.dpl;
3385 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3386 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3390 desc_limit = desc_limit_scaled(&next_tss_desc);
3391 if (!next_tss_desc.p ||
3392 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3393 desc_limit < 0x2b)) {
3394 return emulate_ts(ctxt, tss_selector & 0xfffc);
3397 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3398 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3399 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3402 if (reason == TASK_SWITCH_IRET)
3403 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3405 /* set back link to prev task only if NT bit is set in eflags
3406 note that old_tss_sel is not used after this point */
3407 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3408 old_tss_sel = 0xffff;
3410 if (next_tss_desc.type & 8)
3411 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3412 old_tss_base, &next_tss_desc);
3414 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3415 old_tss_base, &next_tss_desc);
3416 if (ret != X86EMUL_CONTINUE)
3419 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3420 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3422 if (reason != TASK_SWITCH_IRET) {
3423 next_tss_desc.type |= (1 << 1); /* set busy flag */
3424 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3427 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3428 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3430 if (has_error_code) {
3431 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3432 ctxt->lock_prefix = 0;
3433 ctxt->src.val = (unsigned long) error_code;
3434 ret = em_push(ctxt);
3437 ops->get_dr(ctxt, 7, &dr7);
3438 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3443 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3444 u16 tss_selector, int idt_index, int reason,
3445 bool has_error_code, u32 error_code)
3449 invalidate_registers(ctxt);
3450 ctxt->_eip = ctxt->eip;
3451 ctxt->dst.type = OP_NONE;
3453 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3454 has_error_code, error_code);
3456 if (rc == X86EMUL_CONTINUE) {
3457 ctxt->eip = ctxt->_eip;
3458 writeback_registers(ctxt);
3461 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3464 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3467 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3469 register_address_increment(ctxt, reg, df * op->bytes);
3470 op->addr.mem.ea = register_address(ctxt, reg);
3473 static int em_das(struct x86_emulate_ctxt *ctxt)
3476 bool af, cf, old_cf;
3478 cf = ctxt->eflags & X86_EFLAGS_CF;
3484 af = ctxt->eflags & X86_EFLAGS_AF;
3485 if ((al & 0x0f) > 9 || af) {
3487 cf = old_cf | (al >= 250);
3492 if (old_al > 0x99 || old_cf) {
3498 /* Set PF, ZF, SF */
3499 ctxt->src.type = OP_IMM;
3501 ctxt->src.bytes = 1;
3502 fastop(ctxt, em_or);
3503 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3505 ctxt->eflags |= X86_EFLAGS_CF;
3507 ctxt->eflags |= X86_EFLAGS_AF;
3508 return X86EMUL_CONTINUE;
3511 static int em_aam(struct x86_emulate_ctxt *ctxt)
3515 if (ctxt->src.val == 0)
3516 return emulate_de(ctxt);
3518 al = ctxt->dst.val & 0xff;
3519 ah = al / ctxt->src.val;
3520 al %= ctxt->src.val;
3522 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3524 /* Set PF, ZF, SF */
3525 ctxt->src.type = OP_IMM;
3527 ctxt->src.bytes = 1;
3528 fastop(ctxt, em_or);
3530 return X86EMUL_CONTINUE;
3533 static int em_aad(struct x86_emulate_ctxt *ctxt)
3535 u8 al = ctxt->dst.val & 0xff;
3536 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3538 al = (al + (ah * ctxt->src.val)) & 0xff;
3540 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3542 /* Set PF, ZF, SF */
3543 ctxt->src.type = OP_IMM;
3545 ctxt->src.bytes = 1;
3546 fastop(ctxt, em_or);
3548 return X86EMUL_CONTINUE;
3551 static int em_call(struct x86_emulate_ctxt *ctxt)
3554 long rel = ctxt->src.val;
3556 ctxt->src.val = (unsigned long)ctxt->_eip;
3557 rc = jmp_rel(ctxt, rel);
3558 if (rc != X86EMUL_CONTINUE)
3560 return em_push(ctxt);
3563 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3568 struct desc_struct old_desc, new_desc;
3569 const struct x86_emulate_ops *ops = ctxt->ops;
3570 int cpl = ctxt->ops->cpl(ctxt);
3571 enum x86emul_mode prev_mode = ctxt->mode;
3573 old_eip = ctxt->_eip;
3574 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3576 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3577 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3578 X86_TRANSFER_CALL_JMP, &new_desc);
3579 if (rc != X86EMUL_CONTINUE)
3582 rc = assign_eip_far(ctxt, ctxt->src.val);
3583 if (rc != X86EMUL_CONTINUE)
3586 ctxt->src.val = old_cs;
3588 if (rc != X86EMUL_CONTINUE)
3591 ctxt->src.val = old_eip;
3593 /* If we failed, we tainted the memory, but the very least we should
3595 if (rc != X86EMUL_CONTINUE) {
3596 pr_warn_once("faulting far call emulation tainted memory\n");
3601 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3602 ctxt->mode = prev_mode;
3607 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3612 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3613 if (rc != X86EMUL_CONTINUE)
3615 rc = assign_eip_near(ctxt, eip);
3616 if (rc != X86EMUL_CONTINUE)
3618 rsp_increment(ctxt, ctxt->src.val);
3619 return X86EMUL_CONTINUE;
3622 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3624 /* Write back the register source. */
3625 ctxt->src.val = ctxt->dst.val;
3626 write_register_operand(&ctxt->src);
3628 /* Write back the memory destination with implicit LOCK prefix. */
3629 ctxt->dst.val = ctxt->src.orig_val;
3630 ctxt->lock_prefix = 1;
3631 return X86EMUL_CONTINUE;
3634 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3636 ctxt->dst.val = ctxt->src2.val;
3637 return fastop(ctxt, em_imul);
3640 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3642 ctxt->dst.type = OP_REG;
3643 ctxt->dst.bytes = ctxt->src.bytes;
3644 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3645 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3647 return X86EMUL_CONTINUE;
3650 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3654 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3655 return emulate_ud(ctxt);
3656 ctxt->dst.val = tsc_aux;
3657 return X86EMUL_CONTINUE;
3660 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3664 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3665 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3666 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3667 return X86EMUL_CONTINUE;
3670 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3674 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3675 return emulate_gp(ctxt, 0);
3676 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3677 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3678 return X86EMUL_CONTINUE;
3681 static int em_mov(struct x86_emulate_ctxt *ctxt)
3683 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3684 return X86EMUL_CONTINUE;
3687 #define FFL(x) bit(X86_FEATURE_##x)
3689 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3691 u32 ebx, ecx, edx, eax = 1;
3695 * Check MOVBE is set in the guest-visible CPUID leaf.
3697 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3698 if (!(ecx & FFL(MOVBE)))
3699 return emulate_ud(ctxt);
3701 switch (ctxt->op_bytes) {
3704 * From MOVBE definition: "...When the operand size is 16 bits,
3705 * the upper word of the destination register remains unchanged
3708 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3709 * rules so we have to do the operation almost per hand.
3711 tmp = (u16)ctxt->src.val;
3712 ctxt->dst.val &= ~0xffffUL;
3713 ctxt->dst.val |= (unsigned long)swab16(tmp);
3716 ctxt->dst.val = swab32((u32)ctxt->src.val);
3719 ctxt->dst.val = swab64(ctxt->src.val);
3724 return X86EMUL_CONTINUE;
3727 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3729 int cr_num = ctxt->modrm_reg;
3732 if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val))
3733 return emulate_gp(ctxt, 0);
3735 /* Disable writeback. */
3736 ctxt->dst.type = OP_NONE;
3740 * CR0 write might have updated CR0.PE and/or CR0.PG
3741 * which can affect the cpu's execution mode.
3743 r = emulator_recalc_and_set_mode(ctxt);
3744 if (r != X86EMUL_CONTINUE)
3748 return X86EMUL_CONTINUE;
3751 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3755 if (ctxt->mode == X86EMUL_MODE_PROT64)
3756 val = ctxt->src.val & ~0ULL;
3758 val = ctxt->src.val & ~0U;
3760 /* #UD condition is already handled. */
3761 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3762 return emulate_gp(ctxt, 0);
3764 /* Disable writeback. */
3765 ctxt->dst.type = OP_NONE;
3766 return X86EMUL_CONTINUE;
3769 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3773 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3774 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3775 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3776 return emulate_gp(ctxt, 0);
3778 return X86EMUL_CONTINUE;
3781 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3785 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3786 return emulate_gp(ctxt, 0);
3788 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3789 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3790 return X86EMUL_CONTINUE;
3793 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3795 if (segment > VCPU_SREG_GS &&
3796 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3797 ctxt->ops->cpl(ctxt) > 0)
3798 return emulate_gp(ctxt, 0);
3800 ctxt->dst.val = get_segment_selector(ctxt, segment);
3801 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3802 ctxt->dst.bytes = 2;
3803 return X86EMUL_CONTINUE;
3806 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3808 if (ctxt->modrm_reg > VCPU_SREG_GS)
3809 return emulate_ud(ctxt);
3811 return em_store_sreg(ctxt, ctxt->modrm_reg);
3814 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3816 u16 sel = ctxt->src.val;
3818 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3819 return emulate_ud(ctxt);
3821 if (ctxt->modrm_reg == VCPU_SREG_SS)
3822 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3824 /* Disable writeback. */
3825 ctxt->dst.type = OP_NONE;
3826 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3829 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3831 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3834 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3836 u16 sel = ctxt->src.val;
3838 /* Disable writeback. */
3839 ctxt->dst.type = OP_NONE;
3840 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3843 static int em_str(struct x86_emulate_ctxt *ctxt)
3845 return em_store_sreg(ctxt, VCPU_SREG_TR);
3848 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3850 u16 sel = ctxt->src.val;
3852 /* Disable writeback. */
3853 ctxt->dst.type = OP_NONE;
3854 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3857 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3862 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3863 if (rc == X86EMUL_CONTINUE)
3864 ctxt->ops->invlpg(ctxt, linear);
3865 /* Disable writeback. */
3866 ctxt->dst.type = OP_NONE;
3867 return X86EMUL_CONTINUE;
3870 static int em_clts(struct x86_emulate_ctxt *ctxt)
3874 cr0 = ctxt->ops->get_cr(ctxt, 0);
3876 ctxt->ops->set_cr(ctxt, 0, cr0);
3877 return X86EMUL_CONTINUE;
3880 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3882 int rc = ctxt->ops->fix_hypercall(ctxt);
3884 if (rc != X86EMUL_CONTINUE)
3887 /* Let the processor re-execute the fixed hypercall */
3888 ctxt->_eip = ctxt->eip;
3889 /* Disable writeback. */
3890 ctxt->dst.type = OP_NONE;
3891 return X86EMUL_CONTINUE;
3894 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3895 void (*get)(struct x86_emulate_ctxt *ctxt,
3896 struct desc_ptr *ptr))
3898 struct desc_ptr desc_ptr;
3900 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3901 ctxt->ops->cpl(ctxt) > 0)
3902 return emulate_gp(ctxt, 0);
3904 if (ctxt->mode == X86EMUL_MODE_PROT64)
3906 get(ctxt, &desc_ptr);
3907 if (ctxt->op_bytes == 2) {
3909 desc_ptr.address &= 0x00ffffff;
3911 /* Disable writeback. */
3912 ctxt->dst.type = OP_NONE;
3913 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3914 &desc_ptr, 2 + ctxt->op_bytes);
3917 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3919 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3922 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3924 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3927 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3929 struct desc_ptr desc_ptr;
3932 if (ctxt->mode == X86EMUL_MODE_PROT64)
3934 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3935 &desc_ptr.size, &desc_ptr.address,
3937 if (rc != X86EMUL_CONTINUE)
3939 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3940 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3941 return emulate_gp(ctxt, 0);
3943 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3945 ctxt->ops->set_idt(ctxt, &desc_ptr);
3946 /* Disable writeback. */
3947 ctxt->dst.type = OP_NONE;
3948 return X86EMUL_CONTINUE;
3951 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3953 return em_lgdt_lidt(ctxt, true);
3956 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3958 return em_lgdt_lidt(ctxt, false);
3961 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3963 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3964 ctxt->ops->cpl(ctxt) > 0)
3965 return emulate_gp(ctxt, 0);
3967 if (ctxt->dst.type == OP_MEM)
3968 ctxt->dst.bytes = 2;
3969 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3970 return X86EMUL_CONTINUE;
3973 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3975 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3976 | (ctxt->src.val & 0x0f));
3977 ctxt->dst.type = OP_NONE;
3978 return X86EMUL_CONTINUE;
3981 static int em_loop(struct x86_emulate_ctxt *ctxt)
3983 int rc = X86EMUL_CONTINUE;
3985 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3986 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3987 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3988 rc = jmp_rel(ctxt, ctxt->src.val);
3993 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3995 int rc = X86EMUL_CONTINUE;
3997 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3998 rc = jmp_rel(ctxt, ctxt->src.val);
4003 static int em_in(struct x86_emulate_ctxt *ctxt)
4005 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
4007 return X86EMUL_IO_NEEDED;
4009 return X86EMUL_CONTINUE;
4012 static int em_out(struct x86_emulate_ctxt *ctxt)
4014 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
4016 /* Disable writeback. */
4017 ctxt->dst.type = OP_NONE;
4018 return X86EMUL_CONTINUE;
4021 static int em_cli(struct x86_emulate_ctxt *ctxt)
4023 if (emulator_bad_iopl(ctxt))
4024 return emulate_gp(ctxt, 0);
4026 ctxt->eflags &= ~X86_EFLAGS_IF;
4027 return X86EMUL_CONTINUE;
4030 static int em_sti(struct x86_emulate_ctxt *ctxt)
4032 if (emulator_bad_iopl(ctxt))
4033 return emulate_gp(ctxt, 0);
4035 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4036 ctxt->eflags |= X86_EFLAGS_IF;
4037 return X86EMUL_CONTINUE;
4040 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
4042 u32 eax, ebx, ecx, edx;
4045 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
4046 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
4047 ctxt->ops->cpl(ctxt)) {
4048 return emulate_gp(ctxt, 0);
4051 eax = reg_read(ctxt, VCPU_REGS_RAX);
4052 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4053 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
4054 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
4055 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
4056 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
4057 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
4058 return X86EMUL_CONTINUE;
4061 static int em_sahf(struct x86_emulate_ctxt *ctxt)
4065 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
4067 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
4069 ctxt->eflags &= ~0xffUL;
4070 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
4071 return X86EMUL_CONTINUE;
4074 static int em_lahf(struct x86_emulate_ctxt *ctxt)
4076 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
4077 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
4078 return X86EMUL_CONTINUE;
4081 static int em_bswap(struct x86_emulate_ctxt *ctxt)
4083 switch (ctxt->op_bytes) {
4084 #ifdef CONFIG_X86_64
4086 asm("bswap %0" : "+r"(ctxt->dst.val));
4090 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
4093 return X86EMUL_CONTINUE;
4096 static int em_clflush(struct x86_emulate_ctxt *ctxt)
4098 /* emulating clflush regardless of cpuid */
4099 return X86EMUL_CONTINUE;
4102 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4104 /* emulating clflushopt regardless of cpuid */
4105 return X86EMUL_CONTINUE;
4108 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4110 ctxt->dst.val = (s32) ctxt->src.val;
4111 return X86EMUL_CONTINUE;
4114 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4116 u32 eax = 1, ebx, ecx = 0, edx;
4118 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4119 if (!(edx & FFL(FXSR)))
4120 return emulate_ud(ctxt);
4122 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4123 return emulate_nm(ctxt);
4126 * Don't emulate a case that should never be hit, instead of working
4127 * around a lack of fxsave64/fxrstor64 on old compilers.
4129 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4130 return X86EMUL_UNHANDLEABLE;
4132 return X86EMUL_CONTINUE;
4136 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4137 * and restore MXCSR.
4139 static size_t __fxstate_size(int nregs)
4141 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4144 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4147 if (ctxt->mode == X86EMUL_MODE_PROT64)
4148 return __fxstate_size(16);
4150 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4151 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4155 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4158 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4159 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4161 * 3) 64-bit mode with REX.W prefix
4162 * - like (2), but XMM 8-15 are being saved and restored
4163 * 4) 64-bit mode without REX.W prefix
4164 * - like (3), but FIP and FDP are 64 bit
4166 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4167 * desired result. (4) is not emulated.
4169 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4170 * and FPU DS) should match.
4172 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4174 struct fxregs_state fx_state;
4177 rc = check_fxsr(ctxt);
4178 if (rc != X86EMUL_CONTINUE)
4183 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4187 if (rc != X86EMUL_CONTINUE)
4190 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4191 fxstate_size(ctxt));
4195 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4196 * in the host registers (via FXSAVE) instead, so they won't be modified.
4197 * (preemption has to stay disabled until FXRSTOR).
4199 * Use noinline to keep the stack for other functions called by callers small.
4201 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4202 const size_t used_size)
4204 struct fxregs_state fx_tmp;
4207 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4208 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4209 __fxstate_size(16) - used_size);
4214 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4216 struct fxregs_state fx_state;
4220 rc = check_fxsr(ctxt);
4221 if (rc != X86EMUL_CONTINUE)
4224 size = fxstate_size(ctxt);
4225 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4226 if (rc != X86EMUL_CONTINUE)
4231 if (size < __fxstate_size(16)) {
4232 rc = fxregs_fixup(&fx_state, size);
4233 if (rc != X86EMUL_CONTINUE)
4237 if (fx_state.mxcsr >> 16) {
4238 rc = emulate_gp(ctxt, 0);
4242 if (rc == X86EMUL_CONTINUE)
4243 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4251 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4255 eax = reg_read(ctxt, VCPU_REGS_RAX);
4256 edx = reg_read(ctxt, VCPU_REGS_RDX);
4257 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4259 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4260 return emulate_gp(ctxt, 0);
4262 return X86EMUL_CONTINUE;
4265 static bool valid_cr(int nr)
4277 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4279 if (!valid_cr(ctxt->modrm_reg))
4280 return emulate_ud(ctxt);
4282 return X86EMUL_CONTINUE;
4285 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4287 u64 new_val = ctxt->src.val64;
4288 int cr = ctxt->modrm_reg;
4291 static u64 cr_reserved_bits[] = {
4292 0xffffffff00000000ULL,
4293 0, 0, 0, /* CR3 checked later */
4300 return emulate_ud(ctxt);
4302 if (new_val & cr_reserved_bits[cr])
4303 return emulate_gp(ctxt, 0);
4308 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4309 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4310 return emulate_gp(ctxt, 0);
4312 cr4 = ctxt->ops->get_cr(ctxt, 4);
4313 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4315 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4316 !(cr4 & X86_CR4_PAE))
4317 return emulate_gp(ctxt, 0);
4324 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4325 if (efer & EFER_LMA) {
4327 u32 eax, ebx, ecx, edx;
4331 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4333 maxphyaddr = eax & 0xff;
4336 rsvd = rsvd_bits(maxphyaddr, 63);
4337 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4338 rsvd &= ~X86_CR3_PCID_NOFLUSH;
4342 return emulate_gp(ctxt, 0);
4347 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4349 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4350 return emulate_gp(ctxt, 0);
4356 return X86EMUL_CONTINUE;
4359 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4363 ctxt->ops->get_dr(ctxt, 7, &dr7);
4365 /* Check if DR7.Global_Enable is set */
4366 return dr7 & (1 << 13);
4369 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4371 int dr = ctxt->modrm_reg;
4375 return emulate_ud(ctxt);
4377 cr4 = ctxt->ops->get_cr(ctxt, 4);
4378 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4379 return emulate_ud(ctxt);
4381 if (check_dr7_gd(ctxt)) {
4384 ctxt->ops->get_dr(ctxt, 6, &dr6);
4385 dr6 &= ~DR_TRAP_BITS;
4386 dr6 |= DR6_BD | DR6_RTM;
4387 ctxt->ops->set_dr(ctxt, 6, dr6);
4388 return emulate_db(ctxt);
4391 return X86EMUL_CONTINUE;
4394 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4396 u64 new_val = ctxt->src.val64;
4397 int dr = ctxt->modrm_reg;
4399 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4400 return emulate_gp(ctxt, 0);
4402 return check_dr_read(ctxt);
4405 static int check_svme(struct x86_emulate_ctxt *ctxt)
4409 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4411 if (!(efer & EFER_SVME))
4412 return emulate_ud(ctxt);
4414 return X86EMUL_CONTINUE;
4417 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4419 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4421 /* Valid physical address? */
4422 if (rax & 0xffff000000000000ULL)
4423 return emulate_gp(ctxt, 0);
4425 return check_svme(ctxt);
4428 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4430 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4432 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4433 return emulate_ud(ctxt);
4435 return X86EMUL_CONTINUE;
4438 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4440 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4441 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4444 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4445 * in Ring3 when CR4.PCE=0.
4447 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4448 return X86EMUL_CONTINUE;
4450 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4451 ctxt->ops->check_pmc(ctxt, rcx))
4452 return emulate_gp(ctxt, 0);
4454 return X86EMUL_CONTINUE;
4457 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4459 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4460 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4461 return emulate_gp(ctxt, 0);
4463 return X86EMUL_CONTINUE;
4466 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4468 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4469 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4470 return emulate_gp(ctxt, 0);
4472 return X86EMUL_CONTINUE;
4475 #define D(_y) { .flags = (_y) }
4476 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4477 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4478 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4479 #define N D(NotImpl)
4480 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4481 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4482 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4483 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4484 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4485 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4486 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4487 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4488 #define II(_f, _e, _i) \
4489 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4490 #define IIP(_f, _e, _i, _p) \
4491 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4492 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4493 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4495 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4496 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4497 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4498 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4499 #define I2bvIP(_f, _e, _i, _p) \
4500 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4502 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4503 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4504 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4506 static const struct opcode group7_rm0[] = {
4508 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4512 static const struct opcode group7_rm1[] = {
4513 DI(SrcNone | Priv, monitor),
4514 DI(SrcNone | Priv, mwait),
4518 static const struct opcode group7_rm2[] = {
4520 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4524 static const struct opcode group7_rm3[] = {
4525 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4526 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4527 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4528 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4529 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4530 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4531 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4532 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4535 static const struct opcode group7_rm7[] = {
4537 DIP(SrcNone, rdtscp, check_rdtsc),
4541 static const struct opcode group1[] = {
4543 F(Lock | PageTable, em_or),
4546 F(Lock | PageTable, em_and),
4552 static const struct opcode group1A[] = {
4553 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4556 static const struct opcode group2[] = {
4557 F(DstMem | ModRM, em_rol),
4558 F(DstMem | ModRM, em_ror),
4559 F(DstMem | ModRM, em_rcl),
4560 F(DstMem | ModRM, em_rcr),
4561 F(DstMem | ModRM, em_shl),
4562 F(DstMem | ModRM, em_shr),
4563 F(DstMem | ModRM, em_shl),
4564 F(DstMem | ModRM, em_sar),
4567 static const struct opcode group3[] = {
4568 F(DstMem | SrcImm | NoWrite, em_test),
4569 F(DstMem | SrcImm | NoWrite, em_test),
4570 F(DstMem | SrcNone | Lock, em_not),
4571 F(DstMem | SrcNone | Lock, em_neg),
4572 F(DstXacc | Src2Mem, em_mul_ex),
4573 F(DstXacc | Src2Mem, em_imul_ex),
4574 F(DstXacc | Src2Mem, em_div_ex),
4575 F(DstXacc | Src2Mem, em_idiv_ex),
4578 static const struct opcode group4[] = {
4579 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4580 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4584 static const struct opcode group5[] = {
4585 F(DstMem | SrcNone | Lock, em_inc),
4586 F(DstMem | SrcNone | Lock, em_dec),
4587 I(SrcMem | NearBranch, em_call_near_abs),
4588 I(SrcMemFAddr | ImplicitOps, em_call_far),
4589 I(SrcMem | NearBranch, em_jmp_abs),
4590 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4591 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4594 static const struct opcode group6[] = {
4595 II(Prot | DstMem, em_sldt, sldt),
4596 II(Prot | DstMem, em_str, str),
4597 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4598 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4602 static const struct group_dual group7 = { {
4603 II(Mov | DstMem, em_sgdt, sgdt),
4604 II(Mov | DstMem, em_sidt, sidt),
4605 II(SrcMem | Priv, em_lgdt, lgdt),
4606 II(SrcMem | Priv, em_lidt, lidt),
4607 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4608 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4609 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4615 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4616 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4620 static const struct opcode group8[] = {
4622 F(DstMem | SrcImmByte | NoWrite, em_bt),
4623 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4624 F(DstMem | SrcImmByte | Lock, em_btr),
4625 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4629 * The "memory" destination is actually always a register, since we come
4630 * from the register case of group9.
4632 static const struct gprefix pfx_0f_c7_7 = {
4633 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4637 static const struct group_dual group9 = { {
4638 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4640 N, N, N, N, N, N, N,
4641 GP(0, &pfx_0f_c7_7),
4644 static const struct opcode group11[] = {
4645 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4649 static const struct gprefix pfx_0f_ae_7 = {
4650 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4653 static const struct group_dual group15 = { {
4654 I(ModRM | Aligned16, em_fxsave),
4655 I(ModRM | Aligned16, em_fxrstor),
4656 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4658 N, N, N, N, N, N, N, N,
4661 static const struct gprefix pfx_0f_6f_0f_7f = {
4662 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4665 static const struct instr_dual instr_dual_0f_2b = {
4669 static const struct gprefix pfx_0f_2b = {
4670 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4673 static const struct gprefix pfx_0f_10_0f_11 = {
4674 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4677 static const struct gprefix pfx_0f_28_0f_29 = {
4678 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4681 static const struct gprefix pfx_0f_e7 = {
4682 N, I(Sse, em_mov), N, N,
4685 static const struct escape escape_d9 = { {
4686 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4689 N, N, N, N, N, N, N, N,
4691 N, N, N, N, N, N, N, N,
4693 N, N, N, N, N, N, N, N,
4695 N, N, N, N, N, N, N, N,
4697 N, N, N, N, N, N, N, N,
4699 N, N, N, N, N, N, N, N,
4701 N, N, N, N, N, N, N, N,
4703 N, N, N, N, N, N, N, N,
4706 static const struct escape escape_db = { {
4707 N, N, N, N, N, N, N, N,
4710 N, N, N, N, N, N, N, N,
4712 N, N, N, N, N, N, N, N,
4714 N, N, N, N, N, N, N, N,
4716 N, N, N, N, N, N, N, N,
4718 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4720 N, N, N, N, N, N, N, N,
4722 N, N, N, N, N, N, N, N,
4724 N, N, N, N, N, N, N, N,
4727 static const struct escape escape_dd = { {
4728 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4731 N, N, N, N, N, N, N, N,
4733 N, N, N, N, N, N, N, N,
4735 N, N, N, N, N, N, N, N,
4737 N, N, N, N, N, N, N, N,
4739 N, N, N, N, N, N, N, N,
4741 N, N, N, N, N, N, N, N,
4743 N, N, N, N, N, N, N, N,
4745 N, N, N, N, N, N, N, N,
4748 static const struct instr_dual instr_dual_0f_c3 = {
4749 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4752 static const struct mode_dual mode_dual_63 = {
4753 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4756 static const struct opcode opcode_table[256] = {
4758 F6ALU(Lock, em_add),
4759 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4760 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4762 F6ALU(Lock | PageTable, em_or),
4763 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4766 F6ALU(Lock, em_adc),
4767 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4768 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4770 F6ALU(Lock, em_sbb),
4771 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4772 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4774 F6ALU(Lock | PageTable, em_and), N, N,
4776 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4778 F6ALU(Lock, em_xor), N, N,
4780 F6ALU(NoWrite, em_cmp), N, N,
4782 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4784 X8(I(SrcReg | Stack, em_push)),
4786 X8(I(DstReg | Stack, em_pop)),
4788 I(ImplicitOps | Stack | No64, em_pusha),
4789 I(ImplicitOps | Stack | No64, em_popa),
4790 N, MD(ModRM, &mode_dual_63),
4793 I(SrcImm | Mov | Stack, em_push),
4794 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4795 I(SrcImmByte | Mov | Stack, em_push),
4796 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4797 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4798 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4800 X16(D(SrcImmByte | NearBranch)),
4802 G(ByteOp | DstMem | SrcImm, group1),
4803 G(DstMem | SrcImm, group1),
4804 G(ByteOp | DstMem | SrcImm | No64, group1),
4805 G(DstMem | SrcImmByte, group1),
4806 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4807 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4809 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4810 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4811 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4812 D(ModRM | SrcMem | NoAccess | DstReg),
4813 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4816 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4818 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4819 I(SrcImmFAddr | No64, em_call_far), N,
4820 II(ImplicitOps | Stack, em_pushf, pushf),
4821 II(ImplicitOps | Stack, em_popf, popf),
4822 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4824 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4825 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4826 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4827 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4829 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4830 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4831 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4832 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4834 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4836 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4838 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4839 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4840 I(ImplicitOps | NearBranch, em_ret),
4841 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4842 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4843 G(ByteOp, group11), G(0, group11),
4845 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4846 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4847 I(ImplicitOps, em_ret_far),
4848 D(ImplicitOps), DI(SrcImmByte, intn),
4849 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4851 G(Src2One | ByteOp, group2), G(Src2One, group2),
4852 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4853 I(DstAcc | SrcImmUByte | No64, em_aam),
4854 I(DstAcc | SrcImmUByte | No64, em_aad),
4855 F(DstAcc | ByteOp | No64, em_salc),
4856 I(DstAcc | SrcXLat | ByteOp, em_mov),
4858 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4860 X3(I(SrcImmByte | NearBranch, em_loop)),
4861 I(SrcImmByte | NearBranch, em_jcxz),
4862 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4863 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4865 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4866 I(SrcImmFAddr | No64, em_jmp_far),
4867 D(SrcImmByte | ImplicitOps | NearBranch),
4868 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4869 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4871 N, DI(ImplicitOps, icebp), N, N,
4872 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4873 G(ByteOp, group3), G(0, group3),
4875 D(ImplicitOps), D(ImplicitOps),
4876 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4877 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4880 static const struct opcode twobyte_table[256] = {
4882 G(0, group6), GD(0, &group7), N, N,
4883 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4884 II(ImplicitOps | Priv, em_clts, clts), N,
4885 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4886 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4888 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4889 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4891 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4892 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4894 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4895 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4896 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4898 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4901 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4902 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4903 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4906 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4907 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4908 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4909 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4910 I(ImplicitOps | EmulateOnUD, em_sysenter),
4911 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4913 N, N, N, N, N, N, N, N,
4915 X16(D(DstReg | SrcMem | ModRM)),
4917 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4922 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4927 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4929 X16(D(SrcImm | NearBranch)),
4931 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4933 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4934 II(ImplicitOps, em_cpuid, cpuid),
4935 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4936 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4937 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4939 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4940 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4941 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4942 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4943 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4944 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4946 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4947 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4948 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4949 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4950 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4951 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4955 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4956 I(DstReg | SrcMem | ModRM, em_bsf_c),
4957 I(DstReg | SrcMem | ModRM, em_bsr_c),
4958 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4960 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4961 N, ID(0, &instr_dual_0f_c3),
4962 N, N, N, GD(0, &group9),
4964 X8(I(DstReg, em_bswap)),
4966 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4968 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4969 N, N, N, N, N, N, N, N,
4971 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4974 static const struct instr_dual instr_dual_0f_38_f0 = {
4975 I(DstReg | SrcMem | Mov, em_movbe), N
4978 static const struct instr_dual instr_dual_0f_38_f1 = {
4979 I(DstMem | SrcReg | Mov, em_movbe), N
4982 static const struct gprefix three_byte_0f_38_f0 = {
4983 ID(0, &instr_dual_0f_38_f0), N, N, N
4986 static const struct gprefix three_byte_0f_38_f1 = {
4987 ID(0, &instr_dual_0f_38_f1), N, N, N
4991 * Insns below are selected by the prefix which indexed by the third opcode
4994 static const struct opcode opcode_map_0f_38[256] = {
4996 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4998 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
5000 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
5001 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
5022 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
5026 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5032 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
5033 unsigned size, bool sign_extension)
5035 int rc = X86EMUL_CONTINUE;
5039 op->addr.mem.ea = ctxt->_eip;
5040 /* NB. Immediates are sign-extended as necessary. */
5041 switch (op->bytes) {
5043 op->val = insn_fetch(s8, ctxt);
5046 op->val = insn_fetch(s16, ctxt);
5049 op->val = insn_fetch(s32, ctxt);
5052 op->val = insn_fetch(s64, ctxt);
5055 if (!sign_extension) {
5056 switch (op->bytes) {
5064 op->val &= 0xffffffff;
5072 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
5075 int rc = X86EMUL_CONTINUE;
5079 decode_register_operand(ctxt, op);
5082 rc = decode_imm(ctxt, op, 1, false);
5085 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5089 if (ctxt->d & BitOp)
5090 fetch_bit_operand(ctxt);
5091 op->orig_val = op->val;
5094 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
5098 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5099 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5100 fetch_register_operand(op);
5101 op->orig_val = op->val;
5105 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
5106 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5107 fetch_register_operand(op);
5108 op->orig_val = op->val;
5111 if (ctxt->d & ByteOp) {
5116 op->bytes = ctxt->op_bytes;
5117 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5118 fetch_register_operand(op);
5119 op->orig_val = op->val;
5123 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5125 register_address(ctxt, VCPU_REGS_RDI);
5126 op->addr.mem.seg = VCPU_SREG_ES;
5133 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5134 fetch_register_operand(op);
5139 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5142 rc = decode_imm(ctxt, op, 1, true);
5150 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5153 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5156 ctxt->memop.bytes = 1;
5157 if (ctxt->memop.type == OP_REG) {
5158 ctxt->memop.addr.reg = decode_register(ctxt,
5159 ctxt->modrm_rm, true);
5160 fetch_register_operand(&ctxt->memop);
5164 ctxt->memop.bytes = 2;
5167 ctxt->memop.bytes = 4;
5170 rc = decode_imm(ctxt, op, 2, false);
5173 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5177 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5179 register_address(ctxt, VCPU_REGS_RSI);
5180 op->addr.mem.seg = ctxt->seg_override;
5186 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5189 reg_read(ctxt, VCPU_REGS_RBX) +
5190 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5191 op->addr.mem.seg = ctxt->seg_override;
5196 op->addr.mem.ea = ctxt->_eip;
5197 op->bytes = ctxt->op_bytes + 2;
5198 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5201 ctxt->memop.bytes = ctxt->op_bytes + 2;
5205 op->val = VCPU_SREG_ES;
5209 op->val = VCPU_SREG_CS;
5213 op->val = VCPU_SREG_SS;
5217 op->val = VCPU_SREG_DS;
5221 op->val = VCPU_SREG_FS;
5225 op->val = VCPU_SREG_GS;
5228 /* Special instructions do their own operand decoding. */
5230 op->type = OP_NONE; /* Disable writeback. */
5238 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5240 int rc = X86EMUL_CONTINUE;
5241 int mode = ctxt->mode;
5242 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5243 bool op_prefix = false;
5244 bool has_seg_override = false;
5245 struct opcode opcode;
5247 struct desc_struct desc;
5249 ctxt->memop.type = OP_NONE;
5250 ctxt->memopp = NULL;
5251 ctxt->_eip = ctxt->eip;
5252 ctxt->fetch.ptr = ctxt->fetch.data;
5253 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5254 ctxt->opcode_len = 1;
5255 ctxt->intercept = x86_intercept_none;
5257 memcpy(ctxt->fetch.data, insn, insn_len);
5259 rc = __do_insn_fetch_bytes(ctxt, 1);
5260 if (rc != X86EMUL_CONTINUE)
5265 case X86EMUL_MODE_REAL:
5266 case X86EMUL_MODE_VM86:
5267 def_op_bytes = def_ad_bytes = 2;
5268 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5270 def_op_bytes = def_ad_bytes = 4;
5272 case X86EMUL_MODE_PROT16:
5273 def_op_bytes = def_ad_bytes = 2;
5275 case X86EMUL_MODE_PROT32:
5276 def_op_bytes = def_ad_bytes = 4;
5278 #ifdef CONFIG_X86_64
5279 case X86EMUL_MODE_PROT64:
5285 return EMULATION_FAILED;
5288 ctxt->op_bytes = def_op_bytes;
5289 ctxt->ad_bytes = def_ad_bytes;
5291 /* Legacy prefixes. */
5293 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5294 case 0x66: /* operand-size override */
5296 /* switch between 2/4 bytes */
5297 ctxt->op_bytes = def_op_bytes ^ 6;
5299 case 0x67: /* address-size override */
5300 if (mode == X86EMUL_MODE_PROT64)
5301 /* switch between 4/8 bytes */
5302 ctxt->ad_bytes = def_ad_bytes ^ 12;
5304 /* switch between 2/4 bytes */
5305 ctxt->ad_bytes = def_ad_bytes ^ 6;
5307 case 0x26: /* ES override */
5308 has_seg_override = true;
5309 ctxt->seg_override = VCPU_SREG_ES;
5311 case 0x2e: /* CS override */
5312 has_seg_override = true;
5313 ctxt->seg_override = VCPU_SREG_CS;
5315 case 0x36: /* SS override */
5316 has_seg_override = true;
5317 ctxt->seg_override = VCPU_SREG_SS;
5319 case 0x3e: /* DS override */
5320 has_seg_override = true;
5321 ctxt->seg_override = VCPU_SREG_DS;
5323 case 0x64: /* FS override */
5324 has_seg_override = true;
5325 ctxt->seg_override = VCPU_SREG_FS;
5327 case 0x65: /* GS override */
5328 has_seg_override = true;
5329 ctxt->seg_override = VCPU_SREG_GS;
5331 case 0x40 ... 0x4f: /* REX */
5332 if (mode != X86EMUL_MODE_PROT64)
5334 ctxt->rex_prefix = ctxt->b;
5336 case 0xf0: /* LOCK */
5337 ctxt->lock_prefix = 1;
5339 case 0xf2: /* REPNE/REPNZ */
5340 case 0xf3: /* REP/REPE/REPZ */
5341 ctxt->rep_prefix = ctxt->b;
5347 /* Any legacy prefix after a REX prefix nullifies its effect. */
5349 ctxt->rex_prefix = 0;
5355 if (ctxt->rex_prefix & 8)
5356 ctxt->op_bytes = 8; /* REX.W */
5358 /* Opcode byte(s). */
5359 opcode = opcode_table[ctxt->b];
5360 /* Two-byte opcode? */
5361 if (ctxt->b == 0x0f) {
5362 ctxt->opcode_len = 2;
5363 ctxt->b = insn_fetch(u8, ctxt);
5364 opcode = twobyte_table[ctxt->b];
5366 /* 0F_38 opcode map */
5367 if (ctxt->b == 0x38) {
5368 ctxt->opcode_len = 3;
5369 ctxt->b = insn_fetch(u8, ctxt);
5370 opcode = opcode_map_0f_38[ctxt->b];
5373 ctxt->d = opcode.flags;
5375 if (ctxt->d & ModRM)
5376 ctxt->modrm = insn_fetch(u8, ctxt);
5378 /* vex-prefix instructions are not implemented */
5379 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5380 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5384 while (ctxt->d & GroupMask) {
5385 switch (ctxt->d & GroupMask) {
5387 goffset = (ctxt->modrm >> 3) & 7;
5388 opcode = opcode.u.group[goffset];
5391 goffset = (ctxt->modrm >> 3) & 7;
5392 if ((ctxt->modrm >> 6) == 3)
5393 opcode = opcode.u.gdual->mod3[goffset];
5395 opcode = opcode.u.gdual->mod012[goffset];
5398 goffset = ctxt->modrm & 7;
5399 opcode = opcode.u.group[goffset];
5402 if (ctxt->rep_prefix && op_prefix)
5403 return EMULATION_FAILED;
5404 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5405 switch (simd_prefix) {
5406 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5407 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5408 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5409 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5413 if (ctxt->modrm > 0xbf) {
5414 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5415 u32 index = array_index_nospec(
5416 ctxt->modrm - 0xc0, size);
5418 opcode = opcode.u.esc->high[index];
5420 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5424 if ((ctxt->modrm >> 6) == 3)
5425 opcode = opcode.u.idual->mod3;
5427 opcode = opcode.u.idual->mod012;
5430 if (ctxt->mode == X86EMUL_MODE_PROT64)
5431 opcode = opcode.u.mdual->mode64;
5433 opcode = opcode.u.mdual->mode32;
5436 return EMULATION_FAILED;
5439 ctxt->d &= ~(u64)GroupMask;
5440 ctxt->d |= opcode.flags;
5445 return EMULATION_FAILED;
5447 ctxt->execute = opcode.u.execute;
5449 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5450 return EMULATION_FAILED;
5452 if (unlikely(ctxt->d &
5453 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5456 * These are copied unconditionally here, and checked unconditionally
5457 * in x86_emulate_insn.
5459 ctxt->check_perm = opcode.check_perm;
5460 ctxt->intercept = opcode.intercept;
5462 if (ctxt->d & NotImpl)
5463 return EMULATION_FAILED;
5465 if (mode == X86EMUL_MODE_PROT64) {
5466 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5468 else if (ctxt->d & NearBranch)
5472 if (ctxt->d & Op3264) {
5473 if (mode == X86EMUL_MODE_PROT64)
5479 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5483 ctxt->op_bytes = 16;
5484 else if (ctxt->d & Mmx)
5488 /* ModRM and SIB bytes. */
5489 if (ctxt->d & ModRM) {
5490 rc = decode_modrm(ctxt, &ctxt->memop);
5491 if (!has_seg_override) {
5492 has_seg_override = true;
5493 ctxt->seg_override = ctxt->modrm_seg;
5495 } else if (ctxt->d & MemAbs)
5496 rc = decode_abs(ctxt, &ctxt->memop);
5497 if (rc != X86EMUL_CONTINUE)
5500 if (!has_seg_override)
5501 ctxt->seg_override = VCPU_SREG_DS;
5503 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5506 * Decode and fetch the source operand: register, memory
5509 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5510 if (rc != X86EMUL_CONTINUE)
5514 * Decode and fetch the second source operand: register, memory
5517 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5518 if (rc != X86EMUL_CONTINUE)
5521 /* Decode and fetch the destination operand: register or memory. */
5522 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5524 if (ctxt->rip_relative && likely(ctxt->memopp))
5525 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5526 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5529 if (rc == X86EMUL_PROPAGATE_FAULT)
5530 ctxt->have_exception = true;
5531 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5534 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5536 return ctxt->d & PageTable;
5539 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5541 /* The second termination condition only applies for REPE
5542 * and REPNE. Test if the repeat string operation prefix is
5543 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5544 * corresponding termination condition according to:
5545 * - if REPE/REPZ and ZF = 0 then done
5546 * - if REPNE/REPNZ and ZF = 1 then done
5548 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5549 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5550 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5551 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5552 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5553 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5559 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5564 rc = asm_safe("fwait");
5567 if (unlikely(rc != X86EMUL_CONTINUE))
5568 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5570 return X86EMUL_CONTINUE;
5573 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5576 if (op->type == OP_MM)
5577 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5580 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5582 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5584 if (!(ctxt->d & ByteOp))
5585 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5587 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5588 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5589 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5590 : "c"(ctxt->src2.val));
5592 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5593 if (!fop) /* exception is returned in fop variable */
5594 return emulate_de(ctxt);
5595 return X86EMUL_CONTINUE;
5598 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5600 memset(&ctxt->rip_relative, 0,
5601 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5603 ctxt->io_read.pos = 0;
5604 ctxt->io_read.end = 0;
5605 ctxt->mem_read.end = 0;
5608 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5610 const struct x86_emulate_ops *ops = ctxt->ops;
5611 int rc = X86EMUL_CONTINUE;
5612 int saved_dst_type = ctxt->dst.type;
5613 unsigned emul_flags;
5615 ctxt->mem_read.pos = 0;
5617 /* LOCK prefix is allowed only with some instructions */
5618 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5619 rc = emulate_ud(ctxt);
5623 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5624 rc = emulate_ud(ctxt);
5628 emul_flags = ctxt->ops->get_hflags(ctxt);
5629 if (unlikely(ctxt->d &
5630 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5631 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5632 (ctxt->d & Undefined)) {
5633 rc = emulate_ud(ctxt);
5637 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5638 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5639 rc = emulate_ud(ctxt);
5643 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5644 rc = emulate_nm(ctxt);
5648 if (ctxt->d & Mmx) {
5649 rc = flush_pending_x87_faults(ctxt);
5650 if (rc != X86EMUL_CONTINUE)
5653 * Now that we know the fpu is exception safe, we can fetch
5656 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5657 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5658 if (!(ctxt->d & Mov))
5659 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5662 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5663 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5664 X86_ICPT_PRE_EXCEPT);
5665 if (rc != X86EMUL_CONTINUE)
5669 /* Instruction can only be executed in protected mode */
5670 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5671 rc = emulate_ud(ctxt);
5675 /* Privileged instruction can be executed only in CPL=0 */
5676 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5677 if (ctxt->d & PrivUD)
5678 rc = emulate_ud(ctxt);
5680 rc = emulate_gp(ctxt, 0);
5684 /* Do instruction specific permission checks */
5685 if (ctxt->d & CheckPerm) {
5686 rc = ctxt->check_perm(ctxt);
5687 if (rc != X86EMUL_CONTINUE)
5691 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5692 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5693 X86_ICPT_POST_EXCEPT);
5694 if (rc != X86EMUL_CONTINUE)
5698 if (ctxt->rep_prefix && (ctxt->d & String)) {
5699 /* All REP prefixes have the same first termination condition */
5700 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5701 string_registers_quirk(ctxt);
5702 ctxt->eip = ctxt->_eip;
5703 ctxt->eflags &= ~X86_EFLAGS_RF;
5709 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5710 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5711 ctxt->src.valptr, ctxt->src.bytes);
5712 if (rc != X86EMUL_CONTINUE)
5714 ctxt->src.orig_val64 = ctxt->src.val64;
5717 if (ctxt->src2.type == OP_MEM) {
5718 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5719 &ctxt->src2.val, ctxt->src2.bytes);
5720 if (rc != X86EMUL_CONTINUE)
5724 if ((ctxt->d & DstMask) == ImplicitOps)
5728 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5729 /* optimisation - avoid slow emulated read if Mov */
5730 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5731 &ctxt->dst.val, ctxt->dst.bytes);
5732 if (rc != X86EMUL_CONTINUE) {
5733 if (!(ctxt->d & NoWrite) &&
5734 rc == X86EMUL_PROPAGATE_FAULT &&
5735 ctxt->exception.vector == PF_VECTOR)
5736 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5740 /* Copy full 64-bit value for CMPXCHG8B. */
5741 ctxt->dst.orig_val64 = ctxt->dst.val64;
5745 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5746 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5747 X86_ICPT_POST_MEMACCESS);
5748 if (rc != X86EMUL_CONTINUE)
5752 if (ctxt->rep_prefix && (ctxt->d & String))
5753 ctxt->eflags |= X86_EFLAGS_RF;
5755 ctxt->eflags &= ~X86_EFLAGS_RF;
5757 if (ctxt->execute) {
5758 if (ctxt->d & Fastop) {
5759 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5760 rc = fastop(ctxt, fop);
5761 if (rc != X86EMUL_CONTINUE)
5765 rc = ctxt->execute(ctxt);
5766 if (rc != X86EMUL_CONTINUE)
5771 if (ctxt->opcode_len == 2)
5773 else if (ctxt->opcode_len == 3)
5774 goto threebyte_insn;
5777 case 0x70 ... 0x7f: /* jcc (short) */
5778 if (test_cc(ctxt->b, ctxt->eflags))
5779 rc = jmp_rel(ctxt, ctxt->src.val);
5781 case 0x8d: /* lea r16/r32, m */
5782 ctxt->dst.val = ctxt->src.addr.mem.ea;
5784 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5785 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5786 ctxt->dst.type = OP_NONE;
5790 case 0x98: /* cbw/cwde/cdqe */
5791 switch (ctxt->op_bytes) {
5792 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5793 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5794 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5797 case 0xcc: /* int3 */
5798 rc = emulate_int(ctxt, 3);
5800 case 0xcd: /* int n */
5801 rc = emulate_int(ctxt, ctxt->src.val);
5803 case 0xce: /* into */
5804 if (ctxt->eflags & X86_EFLAGS_OF)
5805 rc = emulate_int(ctxt, 4);
5807 case 0xe9: /* jmp rel */
5808 case 0xeb: /* jmp rel short */
5809 rc = jmp_rel(ctxt, ctxt->src.val);
5810 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5812 case 0xf4: /* hlt */
5813 ctxt->ops->halt(ctxt);
5815 case 0xf5: /* cmc */
5816 /* complement carry flag from eflags reg */
5817 ctxt->eflags ^= X86_EFLAGS_CF;
5819 case 0xf8: /* clc */
5820 ctxt->eflags &= ~X86_EFLAGS_CF;
5822 case 0xf9: /* stc */
5823 ctxt->eflags |= X86_EFLAGS_CF;
5825 case 0xfc: /* cld */
5826 ctxt->eflags &= ~X86_EFLAGS_DF;
5828 case 0xfd: /* std */
5829 ctxt->eflags |= X86_EFLAGS_DF;
5832 goto cannot_emulate;
5835 if (rc != X86EMUL_CONTINUE)
5839 if (ctxt->d & SrcWrite) {
5840 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5841 rc = writeback(ctxt, &ctxt->src);
5842 if (rc != X86EMUL_CONTINUE)
5845 if (!(ctxt->d & NoWrite)) {
5846 rc = writeback(ctxt, &ctxt->dst);
5847 if (rc != X86EMUL_CONTINUE)
5852 * restore dst type in case the decoding will be reused
5853 * (happens for string instruction )
5855 ctxt->dst.type = saved_dst_type;
5857 if ((ctxt->d & SrcMask) == SrcSI)
5858 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5860 if ((ctxt->d & DstMask) == DstDI)
5861 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5863 if (ctxt->rep_prefix && (ctxt->d & String)) {
5865 struct read_cache *r = &ctxt->io_read;
5866 if ((ctxt->d & SrcMask) == SrcSI)
5867 count = ctxt->src.count;
5869 count = ctxt->dst.count;
5870 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5872 if (!string_insn_completed(ctxt)) {
5874 * Re-enter guest when pio read ahead buffer is empty
5875 * or, if it is not used, after each 1024 iteration.
5877 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5878 (r->end == 0 || r->end != r->pos)) {
5880 * Reset read cache. Usually happens before
5881 * decode, but since instruction is restarted
5882 * we have to do it here.
5884 ctxt->mem_read.end = 0;
5885 writeback_registers(ctxt);
5886 return EMULATION_RESTART;
5888 goto done; /* skip rip writeback */
5890 ctxt->eflags &= ~X86_EFLAGS_RF;
5893 ctxt->eip = ctxt->_eip;
5894 if (ctxt->mode != X86EMUL_MODE_PROT64)
5895 ctxt->eip = (u32)ctxt->_eip;
5898 if (rc == X86EMUL_PROPAGATE_FAULT) {
5899 WARN_ON(ctxt->exception.vector > 0x1f);
5900 ctxt->have_exception = true;
5902 if (rc == X86EMUL_INTERCEPTED)
5903 return EMULATION_INTERCEPTED;
5905 if (rc == X86EMUL_CONTINUE)
5906 writeback_registers(ctxt);
5908 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5912 case 0x09: /* wbinvd */
5913 (ctxt->ops->wbinvd)(ctxt);
5915 case 0x08: /* invd */
5916 case 0x0d: /* GrpP (prefetch) */
5917 case 0x18: /* Grp16 (prefetch/nop) */
5918 case 0x1f: /* nop */
5920 case 0x20: /* mov cr, reg */
5921 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5923 case 0x21: /* mov from dr to reg */
5924 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5926 case 0x40 ... 0x4f: /* cmov */
5927 if (test_cc(ctxt->b, ctxt->eflags))
5928 ctxt->dst.val = ctxt->src.val;
5929 else if (ctxt->op_bytes != 4)
5930 ctxt->dst.type = OP_NONE; /* no writeback */
5932 case 0x80 ... 0x8f: /* jnz rel, etc*/
5933 if (test_cc(ctxt->b, ctxt->eflags))
5934 rc = jmp_rel(ctxt, ctxt->src.val);
5936 case 0x90 ... 0x9f: /* setcc r/m8 */
5937 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5939 case 0xb6 ... 0xb7: /* movzx */
5940 ctxt->dst.bytes = ctxt->op_bytes;
5941 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5942 : (u16) ctxt->src.val;
5944 case 0xbe ... 0xbf: /* movsx */
5945 ctxt->dst.bytes = ctxt->op_bytes;
5946 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5947 (s16) ctxt->src.val;
5950 goto cannot_emulate;
5955 if (rc != X86EMUL_CONTINUE)
5961 return EMULATION_FAILED;
5964 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5966 invalidate_registers(ctxt);
5969 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5971 writeback_registers(ctxt);
5974 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5976 if (ctxt->rep_prefix && (ctxt->d & String))
5979 if (ctxt->d & TwoMemOp)