1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
28 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
195 * fastop functions have a special calling convention:
200 * flags: rflags (in/out)
201 * ex: rsi (in:fastop pointer, out:zero if exception)
203 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
204 * different operand sizes can be reached by calculation, rather than a jump
205 * table (which would be bigger than the code).
207 * fastop functions are declared as taking a never-defined fastop parameter,
208 * so they can't be called from C directly.
217 int (*execute)(struct x86_emulate_ctxt *ctxt);
218 const struct opcode *group;
219 const struct group_dual *gdual;
220 const struct gprefix *gprefix;
221 const struct escape *esc;
222 const struct instr_dual *idual;
223 const struct mode_dual *mdual;
224 void (*fastop)(struct fastop *fake);
226 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
230 struct opcode mod012[8];
231 struct opcode mod3[8];
235 struct opcode pfx_no;
236 struct opcode pfx_66;
237 struct opcode pfx_f2;
238 struct opcode pfx_f3;
243 struct opcode high[64];
247 struct opcode mod012;
252 struct opcode mode32;
253 struct opcode mode64;
256 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
258 enum x86_transfer_type {
260 X86_TRANSFER_CALL_JMP,
262 X86_TRANSFER_TASK_SWITCH,
265 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 if (!(ctxt->regs_valid & (1 << nr))) {
268 ctxt->regs_valid |= 1 << nr;
269 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
271 return ctxt->_regs[nr];
274 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
276 ctxt->regs_valid |= 1 << nr;
277 ctxt->regs_dirty |= 1 << nr;
278 return &ctxt->_regs[nr];
281 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
284 return reg_write(ctxt, nr);
287 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
291 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
292 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
295 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
297 ctxt->regs_dirty = 0;
298 ctxt->regs_valid = 0;
302 * These EFLAGS bits are restored from saved value during emulation, and
303 * any changes are written back to the saved value after emulation.
305 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
306 X86_EFLAGS_PF|X86_EFLAGS_CF)
314 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
316 #define FOP_FUNC(name) \
317 ".align " __stringify(FASTOP_SIZE) " \n\t" \
318 ".type " name ", @function \n\t" \
321 #define FOP_RET "ret \n\t"
323 #define FOP_START(op) \
324 extern void em_##op(struct fastop *fake); \
325 asm(".pushsection .text, \"ax\" \n\t" \
326 ".global em_" #op " \n\t" \
333 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
336 #define FOP1E(op, dst) \
337 FOP_FUNC(#op "_" #dst) \
338 "10: " #op " %" #dst " \n\t" FOP_RET
340 #define FOP1EEX(op, dst) \
341 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
343 #define FASTOP1(op) \
348 ON64(FOP1E(op##q, rax)) \
351 /* 1-operand, using src2 (for MUL/DIV r/m) */
352 #define FASTOP1SRC2(op, name) \
357 ON64(FOP1E(op, rcx)) \
360 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
361 #define FASTOP1SRC2EX(op, name) \
366 ON64(FOP1EEX(op, rcx)) \
369 #define FOP2E(op, dst, src) \
370 FOP_FUNC(#op "_" #dst "_" #src) \
371 #op " %" #src ", %" #dst " \n\t" FOP_RET
373 #define FASTOP2(op) \
375 FOP2E(op##b, al, dl) \
376 FOP2E(op##w, ax, dx) \
377 FOP2E(op##l, eax, edx) \
378 ON64(FOP2E(op##q, rax, rdx)) \
381 /* 2 operand, word only */
382 #define FASTOP2W(op) \
385 FOP2E(op##w, ax, dx) \
386 FOP2E(op##l, eax, edx) \
387 ON64(FOP2E(op##q, rax, rdx)) \
390 /* 2 operand, src is CL */
391 #define FASTOP2CL(op) \
393 FOP2E(op##b, al, cl) \
394 FOP2E(op##w, ax, cl) \
395 FOP2E(op##l, eax, cl) \
396 ON64(FOP2E(op##q, rax, cl)) \
399 /* 2 operand, src and dest are reversed */
400 #define FASTOP2R(op, name) \
402 FOP2E(op##b, dl, al) \
403 FOP2E(op##w, dx, ax) \
404 FOP2E(op##l, edx, eax) \
405 ON64(FOP2E(op##q, rdx, rax)) \
408 #define FOP3E(op, dst, src, src2) \
409 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
410 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
412 /* 3-operand, word-only, src2=cl */
413 #define FASTOP3WCL(op) \
416 FOP3E(op##w, ax, dx, cl) \
417 FOP3E(op##l, eax, edx, cl) \
418 ON64(FOP3E(op##q, rax, rdx, cl)) \
421 /* Special case for SETcc - 1 instruction per cc */
422 #define FOP_SETCC(op) \
424 ".type " #op ", @function \n\t" \
429 asm(".pushsection .fixup, \"ax\"\n"
430 ".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret\n"
453 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
457 * XXX: inoutclob user must know where the argument is being expanded.
458 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
460 #define asm_safe(insn, inoutclob...) \
464 asm volatile("1:" insn "\n" \
466 ".pushsection .fixup, \"ax\"\n" \
467 "3: movl $1, %[_fault]\n" \
470 _ASM_EXTABLE(1b, 3b) \
471 : [_fault] "+qm"(_fault) inoutclob ); \
473 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
476 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
477 enum x86_intercept intercept,
478 enum x86_intercept_stage stage)
480 struct x86_instruction_info info = {
481 .intercept = intercept,
482 .rep_prefix = ctxt->rep_prefix,
483 .modrm_mod = ctxt->modrm_mod,
484 .modrm_reg = ctxt->modrm_reg,
485 .modrm_rm = ctxt->modrm_rm,
486 .src_val = ctxt->src.val64,
487 .dst_val = ctxt->dst.val64,
488 .src_bytes = ctxt->src.bytes,
489 .dst_bytes = ctxt->dst.bytes,
490 .ad_bytes = ctxt->ad_bytes,
491 .next_rip = ctxt->eip,
494 return ctxt->ops->intercept(ctxt, &info, stage);
497 static void assign_masked(ulong *dest, ulong src, ulong mask)
499 *dest = (*dest & ~mask) | (src & mask);
502 static void assign_register(unsigned long *reg, u64 val, int bytes)
504 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
507 *(u8 *)reg = (u8)val;
510 *(u16 *)reg = (u16)val;
514 break; /* 64b: zero-extend */
521 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
523 return (1UL << (ctxt->ad_bytes << 3)) - 1;
526 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
529 struct desc_struct ss;
531 if (ctxt->mode == X86EMUL_MODE_PROT64)
533 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
534 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
537 static int stack_size(struct x86_emulate_ctxt *ctxt)
539 return (__fls(stack_mask(ctxt)) + 1) >> 3;
542 /* Access/update address held in a register, based on addressing mode. */
543 static inline unsigned long
544 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
546 if (ctxt->ad_bytes == sizeof(unsigned long))
549 return reg & ad_mask(ctxt);
552 static inline unsigned long
553 register_address(struct x86_emulate_ctxt *ctxt, int reg)
555 return address_mask(ctxt, reg_read(ctxt, reg));
558 static void masked_increment(ulong *reg, ulong mask, int inc)
560 assign_masked(reg, *reg + inc, mask);
564 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
566 ulong *preg = reg_rmw(ctxt, reg);
568 assign_register(preg, *preg + inc, ctxt->ad_bytes);
571 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
573 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
576 static u32 desc_limit_scaled(struct desc_struct *desc)
578 u32 limit = get_desc_limit(desc);
580 return desc->g ? (limit << 12) | 0xfff : limit;
583 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
585 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
588 return ctxt->ops->get_cached_segment_base(ctxt, seg);
591 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
592 u32 error, bool valid)
595 ctxt->exception.vector = vec;
596 ctxt->exception.error_code = error;
597 ctxt->exception.error_code_valid = valid;
598 return X86EMUL_PROPAGATE_FAULT;
601 static int emulate_db(struct x86_emulate_ctxt *ctxt)
603 return emulate_exception(ctxt, DB_VECTOR, 0, false);
606 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
608 return emulate_exception(ctxt, GP_VECTOR, err, true);
611 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
613 return emulate_exception(ctxt, SS_VECTOR, err, true);
616 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
618 return emulate_exception(ctxt, UD_VECTOR, 0, false);
621 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
623 return emulate_exception(ctxt, TS_VECTOR, err, true);
626 static int emulate_de(struct x86_emulate_ctxt *ctxt)
628 return emulate_exception(ctxt, DE_VECTOR, 0, false);
631 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
633 return emulate_exception(ctxt, NM_VECTOR, 0, false);
636 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
639 struct desc_struct desc;
641 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
645 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
650 struct desc_struct desc;
652 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
653 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
657 * x86 defines three classes of vector instructions: explicitly
658 * aligned, explicitly unaligned, and the rest, which change behaviour
659 * depending on whether they're AVX encoded or not.
661 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
662 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
663 * 512 bytes of data must be aligned to a 16 byte boundary.
665 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
667 u64 alignment = ctxt->d & AlignMask;
669 if (likely(size < 16))
684 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
685 struct segmented_address addr,
686 unsigned *max_size, unsigned size,
687 bool write, bool fetch,
688 enum x86emul_mode mode, ulong *linear)
690 struct desc_struct desc;
697 la = seg_base(ctxt, addr.seg) + addr.ea;
700 case X86EMUL_MODE_PROT64:
702 va_bits = ctxt_virt_addr_bits(ctxt);
703 if (get_canonical(la, va_bits) != la)
706 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
707 if (size > *max_size)
711 *linear = la = (u32)la;
712 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
716 /* code segment in protected mode or read-only data segment */
717 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
718 || !(desc.type & 2)) && write)
720 /* unreadable code segment */
721 if (!fetch && (desc.type & 8) && !(desc.type & 2))
723 lim = desc_limit_scaled(&desc);
724 if (!(desc.type & 8) && (desc.type & 4)) {
725 /* expand-down segment */
728 lim = desc.d ? 0xffffffff : 0xffff;
732 if (lim == 0xffffffff)
735 *max_size = (u64)lim + 1 - addr.ea;
736 if (size > *max_size)
741 if (la & (insn_alignment(ctxt, size) - 1))
742 return emulate_gp(ctxt, 0);
743 return X86EMUL_CONTINUE;
745 if (addr.seg == VCPU_SREG_SS)
746 return emulate_ss(ctxt, 0);
748 return emulate_gp(ctxt, 0);
751 static int linearize(struct x86_emulate_ctxt *ctxt,
752 struct segmented_address addr,
753 unsigned size, bool write,
757 return __linearize(ctxt, addr, &max_size, size, write, false,
761 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)
766 struct segmented_address addr = { .seg = VCPU_SREG_CS,
769 if (ctxt->op_bytes != sizeof(unsigned long))
770 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
771 rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);
772 if (rc == X86EMUL_CONTINUE)
773 ctxt->_eip = addr.ea;
777 static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt)
780 struct desc_struct cs;
784 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
786 if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) {
787 /* Real mode. cpu must not have long mode active */
789 return X86EMUL_UNHANDLEABLE;
790 ctxt->mode = X86EMUL_MODE_REAL;
791 return X86EMUL_CONTINUE;
794 if (ctxt->eflags & X86_EFLAGS_VM) {
795 /* Protected/VM86 mode. cpu must not have long mode active */
797 return X86EMUL_UNHANDLEABLE;
798 ctxt->mode = X86EMUL_MODE_VM86;
799 return X86EMUL_CONTINUE;
802 if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS))
803 return X86EMUL_UNHANDLEABLE;
805 if (efer & EFER_LMA) {
807 /* Proper long mode */
808 ctxt->mode = X86EMUL_MODE_PROT64;
810 /* 32 bit compatibility mode*/
811 ctxt->mode = X86EMUL_MODE_PROT32;
813 ctxt->mode = X86EMUL_MODE_PROT16;
816 /* Legacy 32 bit / 16 bit mode */
817 ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
820 return X86EMUL_CONTINUE;
823 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
825 return assign_eip(ctxt, dst);
828 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst)
830 int rc = emulator_recalc_and_set_mode(ctxt);
832 if (rc != X86EMUL_CONTINUE)
835 return assign_eip(ctxt, dst);
838 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
840 return assign_eip_near(ctxt, ctxt->_eip + rel);
843 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
844 void *data, unsigned size)
846 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
849 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
850 ulong linear, void *data,
853 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
856 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
857 struct segmented_address addr,
864 rc = linearize(ctxt, addr, size, false, &linear);
865 if (rc != X86EMUL_CONTINUE)
867 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
870 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
871 struct segmented_address addr,
878 rc = linearize(ctxt, addr, size, true, &linear);
879 if (rc != X86EMUL_CONTINUE)
881 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
885 * Prefetch the remaining bytes of the instruction without crossing page
886 * boundary if they are not in fetch_cache yet.
888 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
891 unsigned size, max_size;
892 unsigned long linear;
893 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
894 struct segmented_address addr = { .seg = VCPU_SREG_CS,
895 .ea = ctxt->eip + cur_size };
898 * We do not know exactly how many bytes will be needed, and
899 * __linearize is expensive, so fetch as much as possible. We
900 * just have to avoid going beyond the 15 byte limit, the end
901 * of the segment, or the end of the page.
903 * __linearize is called with size 0 so that it does not do any
904 * boundary check itself. Instead, we use max_size to check
907 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
909 if (unlikely(rc != X86EMUL_CONTINUE))
912 size = min_t(unsigned, 15UL ^ cur_size, max_size);
913 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
916 * One instruction can only straddle two pages,
917 * and one has been loaded at the beginning of
918 * x86_decode_insn. So, if not enough bytes
919 * still, we must have hit the 15-byte boundary.
921 if (unlikely(size < op_size))
922 return emulate_gp(ctxt, 0);
924 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
925 size, &ctxt->exception);
926 if (unlikely(rc != X86EMUL_CONTINUE))
928 ctxt->fetch.end += size;
929 return X86EMUL_CONTINUE;
932 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
935 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
937 if (unlikely(done_size < size))
938 return __do_insn_fetch_bytes(ctxt, size - done_size);
940 return X86EMUL_CONTINUE;
943 /* Fetch next part of the instruction being emulated. */
944 #define insn_fetch(_type, _ctxt) \
947 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
948 if (rc != X86EMUL_CONTINUE) \
950 ctxt->_eip += sizeof(_type); \
951 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
952 ctxt->fetch.ptr += sizeof(_type); \
956 #define insn_fetch_arr(_arr, _size, _ctxt) \
958 rc = do_insn_fetch_bytes(_ctxt, _size); \
959 if (rc != X86EMUL_CONTINUE) \
961 ctxt->_eip += (_size); \
962 memcpy(_arr, ctxt->fetch.ptr, _size); \
963 ctxt->fetch.ptr += (_size); \
967 * Given the 'reg' portion of a ModRM byte, and a register block, return a
968 * pointer into the block that addresses the relevant register.
969 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
971 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
975 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
977 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
978 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
980 p = reg_rmw(ctxt, modrm_reg);
984 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
985 struct segmented_address addr,
986 u16 *size, unsigned long *address, int op_bytes)
993 rc = segmented_read_std(ctxt, addr, size, 2);
994 if (rc != X86EMUL_CONTINUE)
997 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1011 FASTOP1SRC2(mul, mul_ex);
1012 FASTOP1SRC2(imul, imul_ex);
1013 FASTOP1SRC2EX(div, div_ex);
1014 FASTOP1SRC2EX(idiv, idiv_ex);
1043 FASTOP2R(cmp, cmp_r);
1045 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1047 /* If src is zero, do not writeback, but update flags */
1048 if (ctxt->src.val == 0)
1049 ctxt->dst.type = OP_NONE;
1050 return fastop(ctxt, em_bsf);
1053 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1055 /* If src is zero, do not writeback, but update flags */
1056 if (ctxt->src.val == 0)
1057 ctxt->dst.type = OP_NONE;
1058 return fastop(ctxt, em_bsr);
1061 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1064 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1066 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1067 asm("push %[flags]; popf; " CALL_NOSPEC
1068 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1072 static void fetch_register_operand(struct operand *op)
1074 switch (op->bytes) {
1076 op->val = *(u8 *)op->addr.reg;
1079 op->val = *(u16 *)op->addr.reg;
1082 op->val = *(u32 *)op->addr.reg;
1085 op->val = *(u64 *)op->addr.reg;
1090 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1092 ctxt->ops->get_fpu(ctxt);
1094 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1095 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1096 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1097 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1098 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1099 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1100 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1101 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1102 #ifdef CONFIG_X86_64
1103 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1104 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1105 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1106 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1107 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1108 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1109 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1110 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1114 ctxt->ops->put_fpu(ctxt);
1117 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1120 ctxt->ops->get_fpu(ctxt);
1122 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1123 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1124 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1125 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1126 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1127 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1128 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1129 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1130 #ifdef CONFIG_X86_64
1131 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1132 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1133 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1134 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1135 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1136 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1137 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1138 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1142 ctxt->ops->put_fpu(ctxt);
1145 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1147 ctxt->ops->get_fpu(ctxt);
1149 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1150 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1151 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1152 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1153 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1154 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1155 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1156 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1159 ctxt->ops->put_fpu(ctxt);
1162 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1164 ctxt->ops->get_fpu(ctxt);
1166 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1167 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1168 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1169 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1170 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1171 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1172 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1173 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1176 ctxt->ops->put_fpu(ctxt);
1179 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1181 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1182 return emulate_nm(ctxt);
1184 ctxt->ops->get_fpu(ctxt);
1185 asm volatile("fninit");
1186 ctxt->ops->put_fpu(ctxt);
1187 return X86EMUL_CONTINUE;
1190 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1194 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1195 return emulate_nm(ctxt);
1197 ctxt->ops->get_fpu(ctxt);
1198 asm volatile("fnstcw %0": "+m"(fcw));
1199 ctxt->ops->put_fpu(ctxt);
1201 ctxt->dst.val = fcw;
1203 return X86EMUL_CONTINUE;
1206 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1210 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1211 return emulate_nm(ctxt);
1213 ctxt->ops->get_fpu(ctxt);
1214 asm volatile("fnstsw %0": "+m"(fsw));
1215 ctxt->ops->put_fpu(ctxt);
1217 ctxt->dst.val = fsw;
1219 return X86EMUL_CONTINUE;
1222 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1225 unsigned reg = ctxt->modrm_reg;
1227 if (!(ctxt->d & ModRM))
1228 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1230 if (ctxt->d & Sse) {
1234 read_sse_reg(ctxt, &op->vec_val, reg);
1237 if (ctxt->d & Mmx) {
1246 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1247 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1249 fetch_register_operand(op);
1250 op->orig_val = op->val;
1253 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1255 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1256 ctxt->modrm_seg = VCPU_SREG_SS;
1259 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1263 int index_reg, base_reg, scale;
1264 int rc = X86EMUL_CONTINUE;
1267 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1268 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1269 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1271 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1272 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1273 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1274 ctxt->modrm_seg = VCPU_SREG_DS;
1276 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1278 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1279 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1281 if (ctxt->d & Sse) {
1284 op->addr.xmm = ctxt->modrm_rm;
1285 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1288 if (ctxt->d & Mmx) {
1291 op->addr.mm = ctxt->modrm_rm & 7;
1294 fetch_register_operand(op);
1300 if (ctxt->ad_bytes == 2) {
1301 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1302 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1303 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1304 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1306 /* 16-bit ModR/M decode. */
1307 switch (ctxt->modrm_mod) {
1309 if (ctxt->modrm_rm == 6)
1310 modrm_ea += insn_fetch(u16, ctxt);
1313 modrm_ea += insn_fetch(s8, ctxt);
1316 modrm_ea += insn_fetch(u16, ctxt);
1319 switch (ctxt->modrm_rm) {
1321 modrm_ea += bx + si;
1324 modrm_ea += bx + di;
1327 modrm_ea += bp + si;
1330 modrm_ea += bp + di;
1339 if (ctxt->modrm_mod != 0)
1346 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1347 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1348 ctxt->modrm_seg = VCPU_SREG_SS;
1349 modrm_ea = (u16)modrm_ea;
1351 /* 32/64-bit ModR/M decode. */
1352 if ((ctxt->modrm_rm & 7) == 4) {
1353 sib = insn_fetch(u8, ctxt);
1354 index_reg |= (sib >> 3) & 7;
1355 base_reg |= sib & 7;
1358 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1359 modrm_ea += insn_fetch(s32, ctxt);
1361 modrm_ea += reg_read(ctxt, base_reg);
1362 adjust_modrm_seg(ctxt, base_reg);
1363 /* Increment ESP on POP [ESP] */
1364 if ((ctxt->d & IncSP) &&
1365 base_reg == VCPU_REGS_RSP)
1366 modrm_ea += ctxt->op_bytes;
1369 modrm_ea += reg_read(ctxt, index_reg) << scale;
1370 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1371 modrm_ea += insn_fetch(s32, ctxt);
1372 if (ctxt->mode == X86EMUL_MODE_PROT64)
1373 ctxt->rip_relative = 1;
1375 base_reg = ctxt->modrm_rm;
1376 modrm_ea += reg_read(ctxt, base_reg);
1377 adjust_modrm_seg(ctxt, base_reg);
1379 switch (ctxt->modrm_mod) {
1381 modrm_ea += insn_fetch(s8, ctxt);
1384 modrm_ea += insn_fetch(s32, ctxt);
1388 op->addr.mem.ea = modrm_ea;
1389 if (ctxt->ad_bytes != 8)
1390 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1396 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1399 int rc = X86EMUL_CONTINUE;
1402 switch (ctxt->ad_bytes) {
1404 op->addr.mem.ea = insn_fetch(u16, ctxt);
1407 op->addr.mem.ea = insn_fetch(u32, ctxt);
1410 op->addr.mem.ea = insn_fetch(u64, ctxt);
1417 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1421 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1422 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1424 if (ctxt->src.bytes == 2)
1425 sv = (s16)ctxt->src.val & (s16)mask;
1426 else if (ctxt->src.bytes == 4)
1427 sv = (s32)ctxt->src.val & (s32)mask;
1429 sv = (s64)ctxt->src.val & (s64)mask;
1431 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1432 ctxt->dst.addr.mem.ea + (sv >> 3));
1435 /* only subword offset */
1436 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1439 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1440 unsigned long addr, void *dest, unsigned size)
1443 struct read_cache *mc = &ctxt->mem_read;
1445 if (mc->pos < mc->end)
1448 WARN_ON((mc->end + size) >= sizeof(mc->data));
1450 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1452 if (rc != X86EMUL_CONTINUE)
1458 memcpy(dest, mc->data + mc->pos, size);
1460 return X86EMUL_CONTINUE;
1463 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1464 struct segmented_address addr,
1471 rc = linearize(ctxt, addr, size, false, &linear);
1472 if (rc != X86EMUL_CONTINUE)
1474 return read_emulated(ctxt, linear, data, size);
1477 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1478 struct segmented_address addr,
1485 rc = linearize(ctxt, addr, size, true, &linear);
1486 if (rc != X86EMUL_CONTINUE)
1488 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1492 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1493 struct segmented_address addr,
1494 const void *orig_data, const void *data,
1500 rc = linearize(ctxt, addr, size, true, &linear);
1501 if (rc != X86EMUL_CONTINUE)
1503 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1504 size, &ctxt->exception);
1507 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1508 unsigned int size, unsigned short port,
1511 struct read_cache *rc = &ctxt->io_read;
1513 if (rc->pos == rc->end) { /* refill pio read ahead */
1514 unsigned int in_page, n;
1515 unsigned int count = ctxt->rep_prefix ?
1516 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1517 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1518 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1519 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1520 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1523 rc->pos = rc->end = 0;
1524 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1529 if (ctxt->rep_prefix && (ctxt->d & String) &&
1530 !(ctxt->eflags & X86_EFLAGS_DF)) {
1531 ctxt->dst.data = rc->data + rc->pos;
1532 ctxt->dst.type = OP_MEM_STR;
1533 ctxt->dst.count = (rc->end - rc->pos) / size;
1536 memcpy(dest, rc->data + rc->pos, size);
1542 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1543 u16 index, struct desc_struct *desc)
1548 ctxt->ops->get_idt(ctxt, &dt);
1550 if (dt.size < index * 8 + 7)
1551 return emulate_gp(ctxt, index << 3 | 0x2);
1553 addr = dt.address + index * 8;
1554 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1557 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1558 u16 selector, struct desc_ptr *dt)
1560 const struct x86_emulate_ops *ops = ctxt->ops;
1563 if (selector & 1 << 2) {
1564 struct desc_struct desc;
1567 memset (dt, 0, sizeof *dt);
1568 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1572 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1573 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1575 ops->get_gdt(ctxt, dt);
1578 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1579 u16 selector, ulong *desc_addr_p)
1582 u16 index = selector >> 3;
1585 get_descriptor_table_ptr(ctxt, selector, &dt);
1587 if (dt.size < index * 8 + 7)
1588 return emulate_gp(ctxt, selector & 0xfffc);
1590 addr = dt.address + index * 8;
1592 #ifdef CONFIG_X86_64
1593 if (addr >> 32 != 0) {
1596 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1597 if (!(efer & EFER_LMA))
1602 *desc_addr_p = addr;
1603 return X86EMUL_CONTINUE;
1606 /* allowed just for 8 bytes segments */
1607 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1608 u16 selector, struct desc_struct *desc,
1613 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1614 if (rc != X86EMUL_CONTINUE)
1617 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1620 /* allowed just for 8 bytes segments */
1621 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1622 u16 selector, struct desc_struct *desc)
1627 rc = get_descriptor_ptr(ctxt, selector, &addr);
1628 if (rc != X86EMUL_CONTINUE)
1631 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1634 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1635 u16 selector, int seg, u8 cpl,
1636 enum x86_transfer_type transfer,
1637 struct desc_struct *desc)
1639 struct desc_struct seg_desc, old_desc;
1641 unsigned err_vec = GP_VECTOR;
1643 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1649 memset(&seg_desc, 0, sizeof seg_desc);
1651 if (ctxt->mode == X86EMUL_MODE_REAL) {
1652 /* set real mode segment descriptor (keep limit etc. for
1654 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1655 set_desc_base(&seg_desc, selector << 4);
1657 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1658 /* VM86 needs a clean new segment descriptor */
1659 set_desc_base(&seg_desc, selector << 4);
1660 set_desc_limit(&seg_desc, 0xffff);
1670 /* TR should be in GDT only */
1671 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1674 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1675 if (null_selector) {
1676 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1679 if (seg == VCPU_SREG_SS) {
1680 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1684 * ctxt->ops->set_segment expects the CPL to be in
1685 * SS.DPL, so fake an expand-up 32-bit data segment.
1695 /* Skip all following checks */
1699 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1700 if (ret != X86EMUL_CONTINUE)
1703 err_code = selector & 0xfffc;
1704 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1707 /* can't load system descriptor into segment selector */
1708 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1709 if (transfer == X86_TRANSFER_CALL_JMP)
1710 return X86EMUL_UNHANDLEABLE;
1719 * segment is not a writable data segment or segment
1720 * selector's RPL != CPL or segment selector's RPL != CPL
1722 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1726 if (!(seg_desc.type & 8))
1729 if (seg_desc.type & 4) {
1735 if (rpl > cpl || dpl != cpl)
1738 /* in long-mode d/b must be clear if l is set */
1739 if (seg_desc.d && seg_desc.l) {
1742 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1743 if (efer & EFER_LMA)
1747 /* CS(RPL) <- CPL */
1748 selector = (selector & 0xfffc) | cpl;
1751 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1754 case VCPU_SREG_LDTR:
1755 if (seg_desc.s || seg_desc.type != 2)
1758 default: /* DS, ES, FS, or GS */
1760 * segment is not a data or readable code segment or
1761 * ((segment is a data or nonconforming code segment)
1762 * and (both RPL and CPL > DPL))
1764 if ((seg_desc.type & 0xa) == 0x8 ||
1765 (((seg_desc.type & 0xc) != 0xc) &&
1766 (rpl > dpl && cpl > dpl)))
1772 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1777 /* mark segment as accessed */
1778 if (!(seg_desc.type & 1)) {
1780 ret = write_segment_descriptor(ctxt, selector,
1782 if (ret != X86EMUL_CONTINUE)
1785 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1786 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1787 if (ret != X86EMUL_CONTINUE)
1789 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1790 ((u64)base3 << 32), ctxt))
1791 return emulate_gp(ctxt, err_code);
1794 if (seg == VCPU_SREG_TR) {
1795 old_desc = seg_desc;
1796 seg_desc.type |= 2; /* busy */
1797 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1798 sizeof(seg_desc), &ctxt->exception);
1799 if (ret != X86EMUL_CONTINUE)
1803 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1806 return X86EMUL_CONTINUE;
1808 return emulate_exception(ctxt, err_vec, err_code, true);
1811 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1812 u16 selector, int seg)
1814 u8 cpl = ctxt->ops->cpl(ctxt);
1817 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1818 * they can load it at CPL<3 (Intel's manual says only LSS can,
1821 * However, the Intel manual says that putting IST=1/DPL=3 in
1822 * an interrupt gate will result in SS=3 (the AMD manual instead
1823 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1824 * and only forbid it here.
1826 if (seg == VCPU_SREG_SS && selector == 3 &&
1827 ctxt->mode == X86EMUL_MODE_PROT64)
1828 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1830 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1831 X86_TRANSFER_NONE, NULL);
1834 static void write_register_operand(struct operand *op)
1836 return assign_register(op->addr.reg, op->val, op->bytes);
1839 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1843 write_register_operand(op);
1846 if (ctxt->lock_prefix)
1847 return segmented_cmpxchg(ctxt,
1853 return segmented_write(ctxt,
1859 return segmented_write(ctxt,
1862 op->bytes * op->count);
1865 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1868 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1876 return X86EMUL_CONTINUE;
1879 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1881 struct segmented_address addr;
1883 rsp_increment(ctxt, -bytes);
1884 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1885 addr.seg = VCPU_SREG_SS;
1887 return segmented_write(ctxt, addr, data, bytes);
1890 static int em_push(struct x86_emulate_ctxt *ctxt)
1892 /* Disable writeback. */
1893 ctxt->dst.type = OP_NONE;
1894 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1897 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1898 void *dest, int len)
1901 struct segmented_address addr;
1903 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1904 addr.seg = VCPU_SREG_SS;
1905 rc = segmented_read(ctxt, addr, dest, len);
1906 if (rc != X86EMUL_CONTINUE)
1909 rsp_increment(ctxt, len);
1913 static int em_pop(struct x86_emulate_ctxt *ctxt)
1915 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1918 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1919 void *dest, int len)
1922 unsigned long val, change_mask;
1923 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1924 int cpl = ctxt->ops->cpl(ctxt);
1926 rc = emulate_pop(ctxt, &val, len);
1927 if (rc != X86EMUL_CONTINUE)
1930 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1931 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1932 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1933 X86_EFLAGS_AC | X86_EFLAGS_ID;
1935 switch(ctxt->mode) {
1936 case X86EMUL_MODE_PROT64:
1937 case X86EMUL_MODE_PROT32:
1938 case X86EMUL_MODE_PROT16:
1940 change_mask |= X86_EFLAGS_IOPL;
1942 change_mask |= X86_EFLAGS_IF;
1944 case X86EMUL_MODE_VM86:
1946 return emulate_gp(ctxt, 0);
1947 change_mask |= X86_EFLAGS_IF;
1949 default: /* real mode */
1950 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1954 *(unsigned long *)dest =
1955 (ctxt->eflags & ~change_mask) | (val & change_mask);
1960 static int em_popf(struct x86_emulate_ctxt *ctxt)
1962 ctxt->dst.type = OP_REG;
1963 ctxt->dst.addr.reg = &ctxt->eflags;
1964 ctxt->dst.bytes = ctxt->op_bytes;
1965 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1968 static int em_enter(struct x86_emulate_ctxt *ctxt)
1971 unsigned frame_size = ctxt->src.val;
1972 unsigned nesting_level = ctxt->src2.val & 31;
1976 return X86EMUL_UNHANDLEABLE;
1978 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1979 rc = push(ctxt, &rbp, stack_size(ctxt));
1980 if (rc != X86EMUL_CONTINUE)
1982 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1984 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1985 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1987 return X86EMUL_CONTINUE;
1990 static int em_leave(struct x86_emulate_ctxt *ctxt)
1992 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1994 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1997 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1999 int seg = ctxt->src2.val;
2001 ctxt->src.val = get_segment_selector(ctxt, seg);
2002 if (ctxt->op_bytes == 4) {
2003 rsp_increment(ctxt, -2);
2007 return em_push(ctxt);
2010 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
2012 int seg = ctxt->src2.val;
2013 unsigned long selector;
2016 rc = emulate_pop(ctxt, &selector, 2);
2017 if (rc != X86EMUL_CONTINUE)
2020 if (seg == VCPU_SREG_SS)
2021 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2022 if (ctxt->op_bytes > 2)
2023 rsp_increment(ctxt, ctxt->op_bytes - 2);
2025 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2029 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2031 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2032 int rc = X86EMUL_CONTINUE;
2033 int reg = VCPU_REGS_RAX;
2035 while (reg <= VCPU_REGS_RDI) {
2036 (reg == VCPU_REGS_RSP) ?
2037 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2040 if (rc != X86EMUL_CONTINUE)
2049 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2051 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2052 return em_push(ctxt);
2055 static int em_popa(struct x86_emulate_ctxt *ctxt)
2057 int rc = X86EMUL_CONTINUE;
2058 int reg = VCPU_REGS_RDI;
2061 while (reg >= VCPU_REGS_RAX) {
2062 if (reg == VCPU_REGS_RSP) {
2063 rsp_increment(ctxt, ctxt->op_bytes);
2067 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2068 if (rc != X86EMUL_CONTINUE)
2070 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2076 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2078 const struct x86_emulate_ops *ops = ctxt->ops;
2085 /* TODO: Add limit checks */
2086 ctxt->src.val = ctxt->eflags;
2088 if (rc != X86EMUL_CONTINUE)
2091 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2093 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2095 if (rc != X86EMUL_CONTINUE)
2098 ctxt->src.val = ctxt->_eip;
2100 if (rc != X86EMUL_CONTINUE)
2103 ops->get_idt(ctxt, &dt);
2105 eip_addr = dt.address + (irq << 2);
2106 cs_addr = dt.address + (irq << 2) + 2;
2108 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2109 if (rc != X86EMUL_CONTINUE)
2112 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2113 if (rc != X86EMUL_CONTINUE)
2116 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2117 if (rc != X86EMUL_CONTINUE)
2125 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2129 invalidate_registers(ctxt);
2130 rc = __emulate_int_real(ctxt, irq);
2131 if (rc == X86EMUL_CONTINUE)
2132 writeback_registers(ctxt);
2136 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2138 switch(ctxt->mode) {
2139 case X86EMUL_MODE_REAL:
2140 return __emulate_int_real(ctxt, irq);
2141 case X86EMUL_MODE_VM86:
2142 case X86EMUL_MODE_PROT16:
2143 case X86EMUL_MODE_PROT32:
2144 case X86EMUL_MODE_PROT64:
2146 /* Protected mode interrupts unimplemented yet */
2147 return X86EMUL_UNHANDLEABLE;
2151 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2153 int rc = X86EMUL_CONTINUE;
2154 unsigned long temp_eip = 0;
2155 unsigned long temp_eflags = 0;
2156 unsigned long cs = 0;
2157 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2158 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2159 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2160 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2161 X86_EFLAGS_AC | X86_EFLAGS_ID |
2163 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2166 /* TODO: Add stack limit check */
2168 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2170 if (rc != X86EMUL_CONTINUE)
2173 if (temp_eip & ~0xffff)
2174 return emulate_gp(ctxt, 0);
2176 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2178 if (rc != X86EMUL_CONTINUE)
2181 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2183 if (rc != X86EMUL_CONTINUE)
2186 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2188 if (rc != X86EMUL_CONTINUE)
2191 ctxt->_eip = temp_eip;
2193 if (ctxt->op_bytes == 4)
2194 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2195 else if (ctxt->op_bytes == 2) {
2196 ctxt->eflags &= ~0xffff;
2197 ctxt->eflags |= temp_eflags;
2200 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2201 ctxt->eflags |= X86_EFLAGS_FIXED;
2202 ctxt->ops->set_nmi_mask(ctxt, false);
2207 static int em_iret(struct x86_emulate_ctxt *ctxt)
2209 switch(ctxt->mode) {
2210 case X86EMUL_MODE_REAL:
2211 return emulate_iret_real(ctxt);
2212 case X86EMUL_MODE_VM86:
2213 case X86EMUL_MODE_PROT16:
2214 case X86EMUL_MODE_PROT32:
2215 case X86EMUL_MODE_PROT64:
2217 /* iret from protected mode unimplemented yet */
2218 return X86EMUL_UNHANDLEABLE;
2222 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2226 struct desc_struct new_desc;
2227 u8 cpl = ctxt->ops->cpl(ctxt);
2229 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2231 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2232 X86_TRANSFER_CALL_JMP,
2234 if (rc != X86EMUL_CONTINUE)
2237 rc = assign_eip_far(ctxt, ctxt->src.val);
2238 /* Error handling is not implemented. */
2239 if (rc != X86EMUL_CONTINUE)
2240 return X86EMUL_UNHANDLEABLE;
2245 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2247 return assign_eip_near(ctxt, ctxt->src.val);
2250 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2255 old_eip = ctxt->_eip;
2256 rc = assign_eip_near(ctxt, ctxt->src.val);
2257 if (rc != X86EMUL_CONTINUE)
2259 ctxt->src.val = old_eip;
2264 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2266 u64 old = ctxt->dst.orig_val64;
2268 if (ctxt->dst.bytes == 16)
2269 return X86EMUL_UNHANDLEABLE;
2271 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2272 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2273 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2274 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2275 ctxt->eflags &= ~X86_EFLAGS_ZF;
2277 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2278 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2280 ctxt->eflags |= X86_EFLAGS_ZF;
2282 return X86EMUL_CONTINUE;
2285 static int em_ret(struct x86_emulate_ctxt *ctxt)
2290 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2291 if (rc != X86EMUL_CONTINUE)
2294 return assign_eip_near(ctxt, eip);
2297 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2300 unsigned long eip, cs;
2301 int cpl = ctxt->ops->cpl(ctxt);
2302 struct desc_struct new_desc;
2304 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2305 if (rc != X86EMUL_CONTINUE)
2307 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2308 if (rc != X86EMUL_CONTINUE)
2310 /* Outer-privilege level return is not implemented */
2311 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2312 return X86EMUL_UNHANDLEABLE;
2313 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2316 if (rc != X86EMUL_CONTINUE)
2318 rc = assign_eip_far(ctxt, eip);
2319 /* Error handling is not implemented. */
2320 if (rc != X86EMUL_CONTINUE)
2321 return X86EMUL_UNHANDLEABLE;
2326 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2330 rc = em_ret_far(ctxt);
2331 if (rc != X86EMUL_CONTINUE)
2333 rsp_increment(ctxt, ctxt->src.val);
2334 return X86EMUL_CONTINUE;
2337 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2339 /* Save real source value, then compare EAX against destination. */
2340 ctxt->dst.orig_val = ctxt->dst.val;
2341 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2342 ctxt->src.orig_val = ctxt->src.val;
2343 ctxt->src.val = ctxt->dst.orig_val;
2344 fastop(ctxt, em_cmp);
2346 if (ctxt->eflags & X86_EFLAGS_ZF) {
2347 /* Success: write back to memory; no update of EAX */
2348 ctxt->src.type = OP_NONE;
2349 ctxt->dst.val = ctxt->src.orig_val;
2351 /* Failure: write the value we saw to EAX. */
2352 ctxt->src.type = OP_REG;
2353 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2354 ctxt->src.val = ctxt->dst.orig_val;
2355 /* Create write-cycle to dest by writing the same value */
2356 ctxt->dst.val = ctxt->dst.orig_val;
2358 return X86EMUL_CONTINUE;
2361 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2363 int seg = ctxt->src2.val;
2367 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2369 rc = load_segment_descriptor(ctxt, sel, seg);
2370 if (rc != X86EMUL_CONTINUE)
2373 ctxt->dst.val = ctxt->src.val;
2377 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2379 u32 eax, ebx, ecx, edx;
2383 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2384 return edx & bit(X86_FEATURE_LM);
2387 #define GET_SMSTATE(type, smbase, offset) \
2390 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2392 if (r != X86EMUL_CONTINUE) \
2393 return X86EMUL_UNHANDLEABLE; \
2397 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2399 desc->g = (flags >> 23) & 1;
2400 desc->d = (flags >> 22) & 1;
2401 desc->l = (flags >> 21) & 1;
2402 desc->avl = (flags >> 20) & 1;
2403 desc->p = (flags >> 15) & 1;
2404 desc->dpl = (flags >> 13) & 3;
2405 desc->s = (flags >> 12) & 1;
2406 desc->type = (flags >> 8) & 15;
2409 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2411 struct desc_struct desc;
2415 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2418 offset = 0x7f84 + n * 12;
2420 offset = 0x7f2c + (n - 3) * 12;
2422 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2423 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2424 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2425 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2426 return X86EMUL_CONTINUE;
2429 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2431 struct desc_struct desc;
2436 offset = 0x7e00 + n * 16;
2438 selector = GET_SMSTATE(u16, smbase, offset);
2439 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2440 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2441 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2442 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2444 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2445 return X86EMUL_CONTINUE;
2448 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2449 u64 cr0, u64 cr3, u64 cr4)
2454 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2456 if (cr4 & X86_CR4_PCIDE) {
2461 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2463 return X86EMUL_UNHANDLEABLE;
2466 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2467 * Then enable protected mode. However, PCID cannot be enabled
2468 * if EFER.LMA=0, so set it separately.
2470 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2472 return X86EMUL_UNHANDLEABLE;
2474 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2476 return X86EMUL_UNHANDLEABLE;
2478 if (cr4 & X86_CR4_PCIDE) {
2479 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2481 return X86EMUL_UNHANDLEABLE;
2483 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2485 return X86EMUL_UNHANDLEABLE;
2490 return X86EMUL_CONTINUE;
2493 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2495 struct desc_struct desc;
2498 u32 val, cr0, cr3, cr4;
2501 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2502 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2503 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2504 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2506 for (i = 0; i < 8; i++)
2507 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2509 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2510 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2511 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2512 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2514 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2515 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2516 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2517 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2518 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2520 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2521 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2522 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2523 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2524 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2526 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2527 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2528 ctxt->ops->set_gdt(ctxt, &dt);
2530 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2531 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2532 ctxt->ops->set_idt(ctxt, &dt);
2534 for (i = 0; i < 6; i++) {
2535 int r = rsm_load_seg_32(ctxt, smbase, i);
2536 if (r != X86EMUL_CONTINUE)
2540 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2542 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2544 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2547 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2549 struct desc_struct desc;
2551 u64 val, cr0, cr3, cr4;
2556 for (i = 0; i < 16; i++)
2557 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2559 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2560 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2562 val = GET_SMSTATE(u32, smbase, 0x7f68);
2563 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2564 val = GET_SMSTATE(u32, smbase, 0x7f60);
2565 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2567 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2568 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2569 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2570 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2571 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2572 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2574 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2575 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2576 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2577 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2578 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2579 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2581 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2582 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2583 ctxt->ops->set_idt(ctxt, &dt);
2585 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2586 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2587 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2588 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2589 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2590 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2592 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2593 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2594 ctxt->ops->set_gdt(ctxt, &dt);
2596 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2597 if (r != X86EMUL_CONTINUE)
2600 for (i = 0; i < 6; i++) {
2601 r = rsm_load_seg_64(ctxt, smbase, i);
2602 if (r != X86EMUL_CONTINUE)
2606 return X86EMUL_CONTINUE;
2609 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2611 unsigned long cr0, cr4, efer;
2615 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2616 return emulate_ud(ctxt);
2619 * Get back to real mode, to prepare a safe state in which to load
2620 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2621 * supports long mode.
2623 if (emulator_has_longmode(ctxt)) {
2624 struct desc_struct cs_desc;
2626 /* Zero CR4.PCIDE before CR0.PG. */
2627 cr4 = ctxt->ops->get_cr(ctxt, 4);
2628 if (cr4 & X86_CR4_PCIDE)
2629 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2631 /* A 32-bit code segment is required to clear EFER.LMA. */
2632 memset(&cs_desc, 0, sizeof(cs_desc));
2634 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2635 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2638 /* For the 64-bit case, this will clear EFER.LMA. */
2639 cr0 = ctxt->ops->get_cr(ctxt, 0);
2640 if (cr0 & X86_CR0_PE)
2641 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2643 if (emulator_has_longmode(ctxt)) {
2644 /* Clear CR4.PAE before clearing EFER.LME. */
2645 cr4 = ctxt->ops->get_cr(ctxt, 4);
2646 if (cr4 & X86_CR4_PAE)
2647 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2649 /* And finally go back to 32-bit mode. */
2651 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2654 smbase = ctxt->ops->get_smbase(ctxt);
2655 if (emulator_has_longmode(ctxt))
2656 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2658 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2660 if (ret != X86EMUL_CONTINUE) {
2661 /* FIXME: should triple fault */
2662 return X86EMUL_UNHANDLEABLE;
2665 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2666 ctxt->ops->set_nmi_mask(ctxt, false);
2668 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2669 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2670 return X86EMUL_CONTINUE;
2674 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2675 struct desc_struct *cs, struct desc_struct *ss)
2677 cs->l = 0; /* will be adjusted later */
2678 set_desc_base(cs, 0); /* flat segment */
2679 cs->g = 1; /* 4kb granularity */
2680 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2681 cs->type = 0x0b; /* Read, Execute, Accessed */
2683 cs->dpl = 0; /* will be adjusted later */
2688 set_desc_base(ss, 0); /* flat segment */
2689 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2690 ss->g = 1; /* 4kb granularity */
2692 ss->type = 0x03; /* Read/Write, Accessed */
2693 ss->d = 1; /* 32bit stack segment */
2700 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2702 u32 eax, ebx, ecx, edx;
2705 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2706 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2707 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2708 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2711 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2713 const struct x86_emulate_ops *ops = ctxt->ops;
2714 u32 eax, ebx, ecx, edx;
2717 * syscall should always be enabled in longmode - so only become
2718 * vendor specific (cpuid) if other modes are active...
2720 if (ctxt->mode == X86EMUL_MODE_PROT64)
2725 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2727 * Intel ("GenuineIntel")
2728 * remark: Intel CPUs only support "syscall" in 64bit
2729 * longmode. Also an 64bit guest with a
2730 * 32bit compat-app running will #UD !! While this
2731 * behaviour can be fixed (by emulating) into AMD
2732 * response - CPUs of AMD can't behave like Intel.
2734 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2735 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2736 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2739 /* AMD ("AuthenticAMD") */
2740 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2741 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2742 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2745 /* AMD ("AMDisbetter!") */
2746 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2747 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2748 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2751 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2755 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2757 const struct x86_emulate_ops *ops = ctxt->ops;
2758 struct desc_struct cs, ss;
2763 /* syscall is not available in real mode */
2764 if (ctxt->mode == X86EMUL_MODE_REAL ||
2765 ctxt->mode == X86EMUL_MODE_VM86)
2766 return emulate_ud(ctxt);
2768 if (!(em_syscall_is_enabled(ctxt)))
2769 return emulate_ud(ctxt);
2771 ops->get_msr(ctxt, MSR_EFER, &efer);
2772 setup_syscalls_segments(ctxt, &cs, &ss);
2774 if (!(efer & EFER_SCE))
2775 return emulate_ud(ctxt);
2777 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2779 cs_sel = (u16)(msr_data & 0xfffc);
2780 ss_sel = (u16)(msr_data + 8);
2782 if (efer & EFER_LMA) {
2786 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2787 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2789 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2790 if (efer & EFER_LMA) {
2791 #ifdef CONFIG_X86_64
2792 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2795 ctxt->mode == X86EMUL_MODE_PROT64 ?
2796 MSR_LSTAR : MSR_CSTAR, &msr_data);
2797 ctxt->_eip = msr_data;
2799 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2800 ctxt->eflags &= ~msr_data;
2801 ctxt->eflags |= X86_EFLAGS_FIXED;
2805 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2806 ctxt->_eip = (u32)msr_data;
2808 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2811 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2812 return X86EMUL_CONTINUE;
2815 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2817 const struct x86_emulate_ops *ops = ctxt->ops;
2818 struct desc_struct cs, ss;
2823 ops->get_msr(ctxt, MSR_EFER, &efer);
2824 /* inject #GP if in real mode */
2825 if (ctxt->mode == X86EMUL_MODE_REAL)
2826 return emulate_gp(ctxt, 0);
2829 * Not recognized on AMD in compat mode (but is recognized in legacy
2832 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2833 && !vendor_intel(ctxt))
2834 return emulate_ud(ctxt);
2836 /* sysenter/sysexit have not been tested in 64bit mode. */
2837 if (ctxt->mode == X86EMUL_MODE_PROT64)
2838 return X86EMUL_UNHANDLEABLE;
2840 setup_syscalls_segments(ctxt, &cs, &ss);
2842 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2843 if ((msr_data & 0xfffc) == 0x0)
2844 return emulate_gp(ctxt, 0);
2846 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2847 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2848 ss_sel = cs_sel + 8;
2849 if (efer & EFER_LMA) {
2854 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2855 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2857 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2858 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2860 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2861 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2864 return X86EMUL_CONTINUE;
2867 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2869 const struct x86_emulate_ops *ops = ctxt->ops;
2870 struct desc_struct cs, ss;
2871 u64 msr_data, rcx, rdx;
2873 u16 cs_sel = 0, ss_sel = 0;
2875 /* inject #GP if in real mode or Virtual 8086 mode */
2876 if (ctxt->mode == X86EMUL_MODE_REAL ||
2877 ctxt->mode == X86EMUL_MODE_VM86)
2878 return emulate_gp(ctxt, 0);
2880 setup_syscalls_segments(ctxt, &cs, &ss);
2882 if ((ctxt->rex_prefix & 0x8) != 0x0)
2883 usermode = X86EMUL_MODE_PROT64;
2885 usermode = X86EMUL_MODE_PROT32;
2887 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2888 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2892 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2894 case X86EMUL_MODE_PROT32:
2895 cs_sel = (u16)(msr_data + 16);
2896 if ((msr_data & 0xfffc) == 0x0)
2897 return emulate_gp(ctxt, 0);
2898 ss_sel = (u16)(msr_data + 24);
2902 case X86EMUL_MODE_PROT64:
2903 cs_sel = (u16)(msr_data + 32);
2904 if (msr_data == 0x0)
2905 return emulate_gp(ctxt, 0);
2906 ss_sel = cs_sel + 8;
2909 if (emul_is_noncanonical_address(rcx, ctxt) ||
2910 emul_is_noncanonical_address(rdx, ctxt))
2911 return emulate_gp(ctxt, 0);
2914 cs_sel |= SEGMENT_RPL_MASK;
2915 ss_sel |= SEGMENT_RPL_MASK;
2917 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2918 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2921 ctxt->mode = usermode;
2922 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2924 return X86EMUL_CONTINUE;
2927 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2930 if (ctxt->mode == X86EMUL_MODE_REAL)
2932 if (ctxt->mode == X86EMUL_MODE_VM86)
2934 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2935 return ctxt->ops->cpl(ctxt) > iopl;
2938 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2941 const struct x86_emulate_ops *ops = ctxt->ops;
2942 struct desc_struct tr_seg;
2945 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2946 unsigned mask = (1 << len) - 1;
2949 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2952 if (desc_limit_scaled(&tr_seg) < 103)
2954 base = get_desc_base(&tr_seg);
2955 #ifdef CONFIG_X86_64
2956 base |= ((u64)base3) << 32;
2958 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2959 if (r != X86EMUL_CONTINUE)
2961 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2963 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2964 if (r != X86EMUL_CONTINUE)
2966 if ((perm >> bit_idx) & mask)
2971 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2977 if (emulator_bad_iopl(ctxt))
2978 if (!emulator_io_port_access_allowed(ctxt, port, len))
2981 ctxt->perm_ok = true;
2986 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2989 * Intel CPUs mask the counter and pointers in quite strange
2990 * manner when ECX is zero due to REP-string optimizations.
2992 #ifdef CONFIG_X86_64
2993 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2996 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2999 case 0xa4: /* movsb */
3000 case 0xa5: /* movsd/w */
3001 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3003 case 0xaa: /* stosb */
3004 case 0xab: /* stosd/w */
3005 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3010 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3011 struct tss_segment_16 *tss)
3013 tss->ip = ctxt->_eip;
3014 tss->flag = ctxt->eflags;
3015 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3016 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3017 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3018 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3019 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3020 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3021 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3022 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3024 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3025 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3026 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3027 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3028 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3031 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3032 struct tss_segment_16 *tss)
3037 ctxt->_eip = tss->ip;
3038 ctxt->eflags = tss->flag | 2;
3039 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3040 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3041 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3042 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3043 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3044 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3045 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3046 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3049 * SDM says that segment selectors are loaded before segment
3052 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3053 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3054 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3055 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3056 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3061 * Now load segment descriptors. If fault happens at this stage
3062 * it is handled in a context of new task
3064 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3065 X86_TRANSFER_TASK_SWITCH, NULL);
3066 if (ret != X86EMUL_CONTINUE)
3068 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3069 X86_TRANSFER_TASK_SWITCH, NULL);
3070 if (ret != X86EMUL_CONTINUE)
3072 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3073 X86_TRANSFER_TASK_SWITCH, NULL);
3074 if (ret != X86EMUL_CONTINUE)
3076 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3077 X86_TRANSFER_TASK_SWITCH, NULL);
3078 if (ret != X86EMUL_CONTINUE)
3080 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3081 X86_TRANSFER_TASK_SWITCH, NULL);
3082 if (ret != X86EMUL_CONTINUE)
3085 return X86EMUL_CONTINUE;
3088 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3089 u16 tss_selector, u16 old_tss_sel,
3090 ulong old_tss_base, struct desc_struct *new_desc)
3092 struct tss_segment_16 tss_seg;
3094 u32 new_tss_base = get_desc_base(new_desc);
3096 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3097 if (ret != X86EMUL_CONTINUE)
3100 save_state_to_tss16(ctxt, &tss_seg);
3102 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3103 if (ret != X86EMUL_CONTINUE)
3106 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3107 if (ret != X86EMUL_CONTINUE)
3110 if (old_tss_sel != 0xffff) {
3111 tss_seg.prev_task_link = old_tss_sel;
3113 ret = linear_write_system(ctxt, new_tss_base,
3114 &tss_seg.prev_task_link,
3115 sizeof tss_seg.prev_task_link);
3116 if (ret != X86EMUL_CONTINUE)
3120 return load_state_from_tss16(ctxt, &tss_seg);
3123 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3124 struct tss_segment_32 *tss)
3126 /* CR3 and ldt selector are not saved intentionally */
3127 tss->eip = ctxt->_eip;
3128 tss->eflags = ctxt->eflags;
3129 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3130 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3131 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3132 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3133 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3134 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3135 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3136 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3138 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3139 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3140 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3141 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3142 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3143 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3146 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3147 struct tss_segment_32 *tss)
3152 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3153 return emulate_gp(ctxt, 0);
3154 ctxt->_eip = tss->eip;
3155 ctxt->eflags = tss->eflags | 2;
3157 /* General purpose registers */
3158 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3159 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3160 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3161 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3162 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3163 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3164 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3165 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3168 * SDM says that segment selectors are loaded before segment
3169 * descriptors. This is important because CPL checks will
3172 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3173 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3174 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3175 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3176 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3177 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3178 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3181 * If we're switching between Protected Mode and VM86, we need to make
3182 * sure to update the mode before loading the segment descriptors so
3183 * that the selectors are interpreted correctly.
3185 if (ctxt->eflags & X86_EFLAGS_VM) {
3186 ctxt->mode = X86EMUL_MODE_VM86;
3189 ctxt->mode = X86EMUL_MODE_PROT32;
3194 * Now load segment descriptors. If fault happenes at this stage
3195 * it is handled in a context of new task
3197 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3198 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3199 if (ret != X86EMUL_CONTINUE)
3201 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3202 X86_TRANSFER_TASK_SWITCH, NULL);
3203 if (ret != X86EMUL_CONTINUE)
3205 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3206 X86_TRANSFER_TASK_SWITCH, NULL);
3207 if (ret != X86EMUL_CONTINUE)
3209 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3210 X86_TRANSFER_TASK_SWITCH, NULL);
3211 if (ret != X86EMUL_CONTINUE)
3213 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3214 X86_TRANSFER_TASK_SWITCH, NULL);
3215 if (ret != X86EMUL_CONTINUE)
3217 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3218 X86_TRANSFER_TASK_SWITCH, NULL);
3219 if (ret != X86EMUL_CONTINUE)
3221 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3222 X86_TRANSFER_TASK_SWITCH, NULL);
3227 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3228 u16 tss_selector, u16 old_tss_sel,
3229 ulong old_tss_base, struct desc_struct *new_desc)
3231 struct tss_segment_32 tss_seg;
3233 u32 new_tss_base = get_desc_base(new_desc);
3234 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3235 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3237 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3238 if (ret != X86EMUL_CONTINUE)
3241 save_state_to_tss32(ctxt, &tss_seg);
3243 /* Only GP registers and segment selectors are saved */
3244 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3245 ldt_sel_offset - eip_offset);
3246 if (ret != X86EMUL_CONTINUE)
3249 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3250 if (ret != X86EMUL_CONTINUE)
3253 if (old_tss_sel != 0xffff) {
3254 tss_seg.prev_task_link = old_tss_sel;
3256 ret = linear_write_system(ctxt, new_tss_base,
3257 &tss_seg.prev_task_link,
3258 sizeof tss_seg.prev_task_link);
3259 if (ret != X86EMUL_CONTINUE)
3263 return load_state_from_tss32(ctxt, &tss_seg);
3266 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3267 u16 tss_selector, int idt_index, int reason,
3268 bool has_error_code, u32 error_code)
3270 const struct x86_emulate_ops *ops = ctxt->ops;
3271 struct desc_struct curr_tss_desc, next_tss_desc;
3273 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3274 ulong old_tss_base =
3275 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3277 ulong desc_addr, dr7;
3279 /* FIXME: old_tss_base == ~0 ? */
3281 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3282 if (ret != X86EMUL_CONTINUE)
3284 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3285 if (ret != X86EMUL_CONTINUE)
3288 /* FIXME: check that next_tss_desc is tss */
3291 * Check privileges. The three cases are task switch caused by...
3293 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3294 * 2. Exception/IRQ/iret: No check is performed
3295 * 3. jmp/call to TSS/task-gate: No check is performed since the
3296 * hardware checks it before exiting.
3298 if (reason == TASK_SWITCH_GATE) {
3299 if (idt_index != -1) {
3300 /* Software interrupts */
3301 struct desc_struct task_gate_desc;
3304 ret = read_interrupt_descriptor(ctxt, idt_index,
3306 if (ret != X86EMUL_CONTINUE)
3309 dpl = task_gate_desc.dpl;
3310 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3311 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3315 desc_limit = desc_limit_scaled(&next_tss_desc);
3316 if (!next_tss_desc.p ||
3317 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3318 desc_limit < 0x2b)) {
3319 return emulate_ts(ctxt, tss_selector & 0xfffc);
3322 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3323 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3324 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3327 if (reason == TASK_SWITCH_IRET)
3328 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3330 /* set back link to prev task only if NT bit is set in eflags
3331 note that old_tss_sel is not used after this point */
3332 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3333 old_tss_sel = 0xffff;
3335 if (next_tss_desc.type & 8)
3336 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3337 old_tss_base, &next_tss_desc);
3339 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3340 old_tss_base, &next_tss_desc);
3341 if (ret != X86EMUL_CONTINUE)
3344 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3345 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3347 if (reason != TASK_SWITCH_IRET) {
3348 next_tss_desc.type |= (1 << 1); /* set busy flag */
3349 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3352 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3353 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3355 if (has_error_code) {
3356 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3357 ctxt->lock_prefix = 0;
3358 ctxt->src.val = (unsigned long) error_code;
3359 ret = em_push(ctxt);
3362 ops->get_dr(ctxt, 7, &dr7);
3363 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3368 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3369 u16 tss_selector, int idt_index, int reason,
3370 bool has_error_code, u32 error_code)
3374 invalidate_registers(ctxt);
3375 ctxt->_eip = ctxt->eip;
3376 ctxt->dst.type = OP_NONE;
3378 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3379 has_error_code, error_code);
3381 if (rc == X86EMUL_CONTINUE) {
3382 ctxt->eip = ctxt->_eip;
3383 writeback_registers(ctxt);
3386 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3389 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3392 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3394 register_address_increment(ctxt, reg, df * op->bytes);
3395 op->addr.mem.ea = register_address(ctxt, reg);
3398 static int em_das(struct x86_emulate_ctxt *ctxt)
3401 bool af, cf, old_cf;
3403 cf = ctxt->eflags & X86_EFLAGS_CF;
3409 af = ctxt->eflags & X86_EFLAGS_AF;
3410 if ((al & 0x0f) > 9 || af) {
3412 cf = old_cf | (al >= 250);
3417 if (old_al > 0x99 || old_cf) {
3423 /* Set PF, ZF, SF */
3424 ctxt->src.type = OP_IMM;
3426 ctxt->src.bytes = 1;
3427 fastop(ctxt, em_or);
3428 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3430 ctxt->eflags |= X86_EFLAGS_CF;
3432 ctxt->eflags |= X86_EFLAGS_AF;
3433 return X86EMUL_CONTINUE;
3436 static int em_aam(struct x86_emulate_ctxt *ctxt)
3440 if (ctxt->src.val == 0)
3441 return emulate_de(ctxt);
3443 al = ctxt->dst.val & 0xff;
3444 ah = al / ctxt->src.val;
3445 al %= ctxt->src.val;
3447 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3449 /* Set PF, ZF, SF */
3450 ctxt->src.type = OP_IMM;
3452 ctxt->src.bytes = 1;
3453 fastop(ctxt, em_or);
3455 return X86EMUL_CONTINUE;
3458 static int em_aad(struct x86_emulate_ctxt *ctxt)
3460 u8 al = ctxt->dst.val & 0xff;
3461 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3463 al = (al + (ah * ctxt->src.val)) & 0xff;
3465 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3467 /* Set PF, ZF, SF */
3468 ctxt->src.type = OP_IMM;
3470 ctxt->src.bytes = 1;
3471 fastop(ctxt, em_or);
3473 return X86EMUL_CONTINUE;
3476 static int em_call(struct x86_emulate_ctxt *ctxt)
3479 long rel = ctxt->src.val;
3481 ctxt->src.val = (unsigned long)ctxt->_eip;
3482 rc = jmp_rel(ctxt, rel);
3483 if (rc != X86EMUL_CONTINUE)
3485 return em_push(ctxt);
3488 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3493 struct desc_struct old_desc, new_desc;
3494 const struct x86_emulate_ops *ops = ctxt->ops;
3495 int cpl = ctxt->ops->cpl(ctxt);
3496 enum x86emul_mode prev_mode = ctxt->mode;
3498 old_eip = ctxt->_eip;
3499 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3501 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3502 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3503 X86_TRANSFER_CALL_JMP, &new_desc);
3504 if (rc != X86EMUL_CONTINUE)
3507 rc = assign_eip_far(ctxt, ctxt->src.val);
3508 if (rc != X86EMUL_CONTINUE)
3511 ctxt->src.val = old_cs;
3513 if (rc != X86EMUL_CONTINUE)
3516 ctxt->src.val = old_eip;
3518 /* If we failed, we tainted the memory, but the very least we should
3520 if (rc != X86EMUL_CONTINUE) {
3521 pr_warn_once("faulting far call emulation tainted memory\n");
3526 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3527 ctxt->mode = prev_mode;
3532 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3537 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3538 if (rc != X86EMUL_CONTINUE)
3540 rc = assign_eip_near(ctxt, eip);
3541 if (rc != X86EMUL_CONTINUE)
3543 rsp_increment(ctxt, ctxt->src.val);
3544 return X86EMUL_CONTINUE;
3547 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3549 /* Write back the register source. */
3550 ctxt->src.val = ctxt->dst.val;
3551 write_register_operand(&ctxt->src);
3553 /* Write back the memory destination with implicit LOCK prefix. */
3554 ctxt->dst.val = ctxt->src.orig_val;
3555 ctxt->lock_prefix = 1;
3556 return X86EMUL_CONTINUE;
3559 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3561 ctxt->dst.val = ctxt->src2.val;
3562 return fastop(ctxt, em_imul);
3565 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3567 ctxt->dst.type = OP_REG;
3568 ctxt->dst.bytes = ctxt->src.bytes;
3569 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3570 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3572 return X86EMUL_CONTINUE;
3575 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3579 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3580 return emulate_ud(ctxt);
3581 ctxt->dst.val = tsc_aux;
3582 return X86EMUL_CONTINUE;
3585 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3589 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3590 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3591 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3592 return X86EMUL_CONTINUE;
3595 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3599 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3600 return emulate_gp(ctxt, 0);
3601 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3602 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3603 return X86EMUL_CONTINUE;
3606 static int em_mov(struct x86_emulate_ctxt *ctxt)
3608 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3609 return X86EMUL_CONTINUE;
3612 #define FFL(x) bit(X86_FEATURE_##x)
3614 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3616 u32 ebx, ecx, edx, eax = 1;
3620 * Check MOVBE is set in the guest-visible CPUID leaf.
3622 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3623 if (!(ecx & FFL(MOVBE)))
3624 return emulate_ud(ctxt);
3626 switch (ctxt->op_bytes) {
3629 * From MOVBE definition: "...When the operand size is 16 bits,
3630 * the upper word of the destination register remains unchanged
3633 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3634 * rules so we have to do the operation almost per hand.
3636 tmp = (u16)ctxt->src.val;
3637 ctxt->dst.val &= ~0xffffUL;
3638 ctxt->dst.val |= (unsigned long)swab16(tmp);
3641 ctxt->dst.val = swab32((u32)ctxt->src.val);
3644 ctxt->dst.val = swab64(ctxt->src.val);
3649 return X86EMUL_CONTINUE;
3652 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3654 int cr_num = ctxt->modrm_reg;
3657 if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val))
3658 return emulate_gp(ctxt, 0);
3660 /* Disable writeback. */
3661 ctxt->dst.type = OP_NONE;
3665 * CR0 write might have updated CR0.PE and/or CR0.PG
3666 * which can affect the cpu's execution mode.
3668 r = emulator_recalc_and_set_mode(ctxt);
3669 if (r != X86EMUL_CONTINUE)
3673 return X86EMUL_CONTINUE;
3676 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3680 if (ctxt->mode == X86EMUL_MODE_PROT64)
3681 val = ctxt->src.val & ~0ULL;
3683 val = ctxt->src.val & ~0U;
3685 /* #UD condition is already handled. */
3686 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3687 return emulate_gp(ctxt, 0);
3689 /* Disable writeback. */
3690 ctxt->dst.type = OP_NONE;
3691 return X86EMUL_CONTINUE;
3694 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3698 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3699 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3700 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3701 return emulate_gp(ctxt, 0);
3703 return X86EMUL_CONTINUE;
3706 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3710 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3711 return emulate_gp(ctxt, 0);
3713 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3714 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3715 return X86EMUL_CONTINUE;
3718 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3720 if (ctxt->modrm_reg > VCPU_SREG_GS)
3721 return emulate_ud(ctxt);
3723 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3724 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3725 ctxt->dst.bytes = 2;
3726 return X86EMUL_CONTINUE;
3729 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3731 u16 sel = ctxt->src.val;
3733 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3734 return emulate_ud(ctxt);
3736 if (ctxt->modrm_reg == VCPU_SREG_SS)
3737 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3739 /* Disable writeback. */
3740 ctxt->dst.type = OP_NONE;
3741 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3744 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3746 u16 sel = ctxt->src.val;
3748 /* Disable writeback. */
3749 ctxt->dst.type = OP_NONE;
3750 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3753 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3755 u16 sel = ctxt->src.val;
3757 /* Disable writeback. */
3758 ctxt->dst.type = OP_NONE;
3759 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3762 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3767 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3768 if (rc == X86EMUL_CONTINUE)
3769 ctxt->ops->invlpg(ctxt, linear);
3770 /* Disable writeback. */
3771 ctxt->dst.type = OP_NONE;
3772 return X86EMUL_CONTINUE;
3775 static int em_clts(struct x86_emulate_ctxt *ctxt)
3779 cr0 = ctxt->ops->get_cr(ctxt, 0);
3781 ctxt->ops->set_cr(ctxt, 0, cr0);
3782 return X86EMUL_CONTINUE;
3785 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3787 int rc = ctxt->ops->fix_hypercall(ctxt);
3789 if (rc != X86EMUL_CONTINUE)
3792 /* Let the processor re-execute the fixed hypercall */
3793 ctxt->_eip = ctxt->eip;
3794 /* Disable writeback. */
3795 ctxt->dst.type = OP_NONE;
3796 return X86EMUL_CONTINUE;
3799 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3800 void (*get)(struct x86_emulate_ctxt *ctxt,
3801 struct desc_ptr *ptr))
3803 struct desc_ptr desc_ptr;
3805 if (ctxt->mode == X86EMUL_MODE_PROT64)
3807 get(ctxt, &desc_ptr);
3808 if (ctxt->op_bytes == 2) {
3810 desc_ptr.address &= 0x00ffffff;
3812 /* Disable writeback. */
3813 ctxt->dst.type = OP_NONE;
3814 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3815 &desc_ptr, 2 + ctxt->op_bytes);
3818 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3820 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3823 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3825 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3828 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3830 struct desc_ptr desc_ptr;
3833 if (ctxt->mode == X86EMUL_MODE_PROT64)
3835 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3836 &desc_ptr.size, &desc_ptr.address,
3838 if (rc != X86EMUL_CONTINUE)
3840 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3841 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3842 return emulate_gp(ctxt, 0);
3844 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3846 ctxt->ops->set_idt(ctxt, &desc_ptr);
3847 /* Disable writeback. */
3848 ctxt->dst.type = OP_NONE;
3849 return X86EMUL_CONTINUE;
3852 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3854 return em_lgdt_lidt(ctxt, true);
3857 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3859 return em_lgdt_lidt(ctxt, false);
3862 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3864 if (ctxt->dst.type == OP_MEM)
3865 ctxt->dst.bytes = 2;
3866 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3867 return X86EMUL_CONTINUE;
3870 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3872 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3873 | (ctxt->src.val & 0x0f));
3874 ctxt->dst.type = OP_NONE;
3875 return X86EMUL_CONTINUE;
3878 static int em_loop(struct x86_emulate_ctxt *ctxt)
3880 int rc = X86EMUL_CONTINUE;
3882 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3883 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3884 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3885 rc = jmp_rel(ctxt, ctxt->src.val);
3890 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3892 int rc = X86EMUL_CONTINUE;
3894 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3895 rc = jmp_rel(ctxt, ctxt->src.val);
3900 static int em_in(struct x86_emulate_ctxt *ctxt)
3902 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3904 return X86EMUL_IO_NEEDED;
3906 return X86EMUL_CONTINUE;
3909 static int em_out(struct x86_emulate_ctxt *ctxt)
3911 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3913 /* Disable writeback. */
3914 ctxt->dst.type = OP_NONE;
3915 return X86EMUL_CONTINUE;
3918 static int em_cli(struct x86_emulate_ctxt *ctxt)
3920 if (emulator_bad_iopl(ctxt))
3921 return emulate_gp(ctxt, 0);
3923 ctxt->eflags &= ~X86_EFLAGS_IF;
3924 return X86EMUL_CONTINUE;
3927 static int em_sti(struct x86_emulate_ctxt *ctxt)
3929 if (emulator_bad_iopl(ctxt))
3930 return emulate_gp(ctxt, 0);
3932 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3933 ctxt->eflags |= X86_EFLAGS_IF;
3934 return X86EMUL_CONTINUE;
3937 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3939 u32 eax, ebx, ecx, edx;
3942 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3943 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3944 ctxt->ops->cpl(ctxt)) {
3945 return emulate_gp(ctxt, 0);
3948 eax = reg_read(ctxt, VCPU_REGS_RAX);
3949 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3950 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3951 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3952 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3953 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3954 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3955 return X86EMUL_CONTINUE;
3958 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3962 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3964 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3966 ctxt->eflags &= ~0xffUL;
3967 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3968 return X86EMUL_CONTINUE;
3971 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3973 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3974 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3975 return X86EMUL_CONTINUE;
3978 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3980 switch (ctxt->op_bytes) {
3981 #ifdef CONFIG_X86_64
3983 asm("bswap %0" : "+r"(ctxt->dst.val));
3987 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3990 return X86EMUL_CONTINUE;
3993 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3995 /* emulating clflush regardless of cpuid */
3996 return X86EMUL_CONTINUE;
3999 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4001 /* emulating clflushopt regardless of cpuid */
4002 return X86EMUL_CONTINUE;
4005 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4007 ctxt->dst.val = (s32) ctxt->src.val;
4008 return X86EMUL_CONTINUE;
4011 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4013 u32 eax = 1, ebx, ecx = 0, edx;
4015 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4016 if (!(edx & FFL(FXSR)))
4017 return emulate_ud(ctxt);
4019 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4020 return emulate_nm(ctxt);
4023 * Don't emulate a case that should never be hit, instead of working
4024 * around a lack of fxsave64/fxrstor64 on old compilers.
4026 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4027 return X86EMUL_UNHANDLEABLE;
4029 return X86EMUL_CONTINUE;
4033 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4034 * and restore MXCSR.
4036 static size_t __fxstate_size(int nregs)
4038 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4041 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4044 if (ctxt->mode == X86EMUL_MODE_PROT64)
4045 return __fxstate_size(16);
4047 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4048 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4052 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4055 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4056 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4058 * 3) 64-bit mode with REX.W prefix
4059 * - like (2), but XMM 8-15 are being saved and restored
4060 * 4) 64-bit mode without REX.W prefix
4061 * - like (3), but FIP and FDP are 64 bit
4063 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4064 * desired result. (4) is not emulated.
4066 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4067 * and FPU DS) should match.
4069 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4071 struct fxregs_state fx_state;
4074 rc = check_fxsr(ctxt);
4075 if (rc != X86EMUL_CONTINUE)
4078 ctxt->ops->get_fpu(ctxt);
4080 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4082 ctxt->ops->put_fpu(ctxt);
4084 if (rc != X86EMUL_CONTINUE)
4087 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4088 fxstate_size(ctxt));
4092 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4093 * in the host registers (via FXSAVE) instead, so they won't be modified.
4094 * (preemption has to stay disabled until FXRSTOR).
4096 * Use noinline to keep the stack for other functions called by callers small.
4098 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4099 const size_t used_size)
4101 struct fxregs_state fx_tmp;
4104 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4105 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4106 __fxstate_size(16) - used_size);
4111 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4113 struct fxregs_state fx_state;
4117 rc = check_fxsr(ctxt);
4118 if (rc != X86EMUL_CONTINUE)
4121 size = fxstate_size(ctxt);
4122 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4123 if (rc != X86EMUL_CONTINUE)
4126 ctxt->ops->get_fpu(ctxt);
4128 if (size < __fxstate_size(16)) {
4129 rc = fxregs_fixup(&fx_state, size);
4130 if (rc != X86EMUL_CONTINUE)
4134 if (fx_state.mxcsr >> 16) {
4135 rc = emulate_gp(ctxt, 0);
4139 if (rc == X86EMUL_CONTINUE)
4140 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4143 ctxt->ops->put_fpu(ctxt);
4148 static bool valid_cr(int nr)
4160 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4162 if (!valid_cr(ctxt->modrm_reg))
4163 return emulate_ud(ctxt);
4165 return X86EMUL_CONTINUE;
4168 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4170 u64 new_val = ctxt->src.val64;
4171 int cr = ctxt->modrm_reg;
4174 static u64 cr_reserved_bits[] = {
4175 0xffffffff00000000ULL,
4176 0, 0, 0, /* CR3 checked later */
4183 return emulate_ud(ctxt);
4185 if (new_val & cr_reserved_bits[cr])
4186 return emulate_gp(ctxt, 0);
4191 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4192 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4193 return emulate_gp(ctxt, 0);
4195 cr4 = ctxt->ops->get_cr(ctxt, 4);
4196 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4198 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4199 !(cr4 & X86_CR4_PAE))
4200 return emulate_gp(ctxt, 0);
4207 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4208 if (efer & EFER_LMA) {
4210 u32 eax, ebx, ecx, edx;
4214 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4216 maxphyaddr = eax & 0xff;
4219 rsvd = rsvd_bits(maxphyaddr, 63);
4220 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4221 rsvd &= ~CR3_PCID_INVD;
4225 return emulate_gp(ctxt, 0);
4230 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4232 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4233 return emulate_gp(ctxt, 0);
4239 return X86EMUL_CONTINUE;
4242 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4246 ctxt->ops->get_dr(ctxt, 7, &dr7);
4248 /* Check if DR7.Global_Enable is set */
4249 return dr7 & (1 << 13);
4252 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4254 int dr = ctxt->modrm_reg;
4258 return emulate_ud(ctxt);
4260 cr4 = ctxt->ops->get_cr(ctxt, 4);
4261 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4262 return emulate_ud(ctxt);
4264 if (check_dr7_gd(ctxt)) {
4267 ctxt->ops->get_dr(ctxt, 6, &dr6);
4269 dr6 |= DR6_BD | DR6_RTM;
4270 ctxt->ops->set_dr(ctxt, 6, dr6);
4271 return emulate_db(ctxt);
4274 return X86EMUL_CONTINUE;
4277 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4279 u64 new_val = ctxt->src.val64;
4280 int dr = ctxt->modrm_reg;
4282 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4283 return emulate_gp(ctxt, 0);
4285 return check_dr_read(ctxt);
4288 static int check_svme(struct x86_emulate_ctxt *ctxt)
4292 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4294 if (!(efer & EFER_SVME))
4295 return emulate_ud(ctxt);
4297 return X86EMUL_CONTINUE;
4300 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4302 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4304 /* Valid physical address? */
4305 if (rax & 0xffff000000000000ULL)
4306 return emulate_gp(ctxt, 0);
4308 return check_svme(ctxt);
4311 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4313 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4315 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4316 return emulate_ud(ctxt);
4318 return X86EMUL_CONTINUE;
4321 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4323 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4324 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4326 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4327 ctxt->ops->check_pmc(ctxt, rcx))
4328 return emulate_gp(ctxt, 0);
4330 return X86EMUL_CONTINUE;
4333 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4335 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4336 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4337 return emulate_gp(ctxt, 0);
4339 return X86EMUL_CONTINUE;
4342 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4344 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4345 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4346 return emulate_gp(ctxt, 0);
4348 return X86EMUL_CONTINUE;
4351 #define D(_y) { .flags = (_y) }
4352 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4353 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4354 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4355 #define N D(NotImpl)
4356 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4357 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4358 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4359 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4360 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4361 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4362 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4363 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4364 #define II(_f, _e, _i) \
4365 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4366 #define IIP(_f, _e, _i, _p) \
4367 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4368 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4369 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4371 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4372 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4373 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4374 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4375 #define I2bvIP(_f, _e, _i, _p) \
4376 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4378 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4379 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4380 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4382 static const struct opcode group7_rm0[] = {
4384 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4388 static const struct opcode group7_rm1[] = {
4389 DI(SrcNone | Priv, monitor),
4390 DI(SrcNone | Priv, mwait),
4394 static const struct opcode group7_rm3[] = {
4395 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4396 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4397 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4398 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4399 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4400 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4401 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4402 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4405 static const struct opcode group7_rm7[] = {
4407 DIP(SrcNone, rdtscp, check_rdtsc),
4411 static const struct opcode group1[] = {
4413 F(Lock | PageTable, em_or),
4416 F(Lock | PageTable, em_and),
4422 static const struct opcode group1A[] = {
4423 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4426 static const struct opcode group2[] = {
4427 F(DstMem | ModRM, em_rol),
4428 F(DstMem | ModRM, em_ror),
4429 F(DstMem | ModRM, em_rcl),
4430 F(DstMem | ModRM, em_rcr),
4431 F(DstMem | ModRM, em_shl),
4432 F(DstMem | ModRM, em_shr),
4433 F(DstMem | ModRM, em_shl),
4434 F(DstMem | ModRM, em_sar),
4437 static const struct opcode group3[] = {
4438 F(DstMem | SrcImm | NoWrite, em_test),
4439 F(DstMem | SrcImm | NoWrite, em_test),
4440 F(DstMem | SrcNone | Lock, em_not),
4441 F(DstMem | SrcNone | Lock, em_neg),
4442 F(DstXacc | Src2Mem, em_mul_ex),
4443 F(DstXacc | Src2Mem, em_imul_ex),
4444 F(DstXacc | Src2Mem, em_div_ex),
4445 F(DstXacc | Src2Mem, em_idiv_ex),
4448 static const struct opcode group4[] = {
4449 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4450 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4454 static const struct opcode group5[] = {
4455 F(DstMem | SrcNone | Lock, em_inc),
4456 F(DstMem | SrcNone | Lock, em_dec),
4457 I(SrcMem | NearBranch, em_call_near_abs),
4458 I(SrcMemFAddr | ImplicitOps, em_call_far),
4459 I(SrcMem | NearBranch, em_jmp_abs),
4460 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4461 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4464 static const struct opcode group6[] = {
4465 DI(Prot | DstMem, sldt),
4466 DI(Prot | DstMem, str),
4467 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4468 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4472 static const struct group_dual group7 = { {
4473 II(Mov | DstMem, em_sgdt, sgdt),
4474 II(Mov | DstMem, em_sidt, sidt),
4475 II(SrcMem | Priv, em_lgdt, lgdt),
4476 II(SrcMem | Priv, em_lidt, lidt),
4477 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4478 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4479 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4483 N, EXT(0, group7_rm3),
4484 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4485 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4489 static const struct opcode group8[] = {
4491 F(DstMem | SrcImmByte | NoWrite, em_bt),
4492 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4493 F(DstMem | SrcImmByte | Lock, em_btr),
4494 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4498 * The "memory" destination is actually always a register, since we come
4499 * from the register case of group9.
4501 static const struct gprefix pfx_0f_c7_7 = {
4502 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4506 static const struct group_dual group9 = { {
4507 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4509 N, N, N, N, N, N, N,
4510 GP(0, &pfx_0f_c7_7),
4513 static const struct opcode group11[] = {
4514 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4518 static const struct gprefix pfx_0f_ae_7 = {
4519 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4522 static const struct group_dual group15 = { {
4523 I(ModRM | Aligned16, em_fxsave),
4524 I(ModRM | Aligned16, em_fxrstor),
4525 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4527 N, N, N, N, N, N, N, N,
4530 static const struct gprefix pfx_0f_6f_0f_7f = {
4531 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4534 static const struct instr_dual instr_dual_0f_2b = {
4538 static const struct gprefix pfx_0f_2b = {
4539 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4542 static const struct gprefix pfx_0f_28_0f_29 = {
4543 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4546 static const struct gprefix pfx_0f_e7 = {
4547 N, I(Sse, em_mov), N, N,
4550 static const struct escape escape_d9 = { {
4551 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4554 N, N, N, N, N, N, N, N,
4556 N, N, N, N, N, N, N, N,
4558 N, N, N, N, N, N, N, N,
4560 N, N, N, N, N, N, N, N,
4562 N, N, N, N, N, N, N, N,
4564 N, N, N, N, N, N, N, N,
4566 N, N, N, N, N, N, N, N,
4568 N, N, N, N, N, N, N, N,
4571 static const struct escape escape_db = { {
4572 N, N, N, N, N, N, N, N,
4575 N, N, N, N, N, N, N, N,
4577 N, N, N, N, N, N, N, N,
4579 N, N, N, N, N, N, N, N,
4581 N, N, N, N, N, N, N, N,
4583 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4585 N, N, N, N, N, N, N, N,
4587 N, N, N, N, N, N, N, N,
4589 N, N, N, N, N, N, N, N,
4592 static const struct escape escape_dd = { {
4593 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4596 N, N, N, N, N, N, N, N,
4598 N, N, N, N, N, N, N, N,
4600 N, N, N, N, N, N, N, N,
4602 N, N, N, N, N, N, N, N,
4604 N, N, N, N, N, N, N, N,
4606 N, N, N, N, N, N, N, N,
4608 N, N, N, N, N, N, N, N,
4610 N, N, N, N, N, N, N, N,
4613 static const struct instr_dual instr_dual_0f_c3 = {
4614 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4617 static const struct mode_dual mode_dual_63 = {
4618 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4621 static const struct opcode opcode_table[256] = {
4623 F6ALU(Lock, em_add),
4624 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4625 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4627 F6ALU(Lock | PageTable, em_or),
4628 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4631 F6ALU(Lock, em_adc),
4632 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4633 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4635 F6ALU(Lock, em_sbb),
4636 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4637 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4639 F6ALU(Lock | PageTable, em_and), N, N,
4641 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4643 F6ALU(Lock, em_xor), N, N,
4645 F6ALU(NoWrite, em_cmp), N, N,
4647 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4649 X8(I(SrcReg | Stack, em_push)),
4651 X8(I(DstReg | Stack, em_pop)),
4653 I(ImplicitOps | Stack | No64, em_pusha),
4654 I(ImplicitOps | Stack | No64, em_popa),
4655 N, MD(ModRM, &mode_dual_63),
4658 I(SrcImm | Mov | Stack, em_push),
4659 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4660 I(SrcImmByte | Mov | Stack, em_push),
4661 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4662 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4663 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4665 X16(D(SrcImmByte | NearBranch)),
4667 G(ByteOp | DstMem | SrcImm, group1),
4668 G(DstMem | SrcImm, group1),
4669 G(ByteOp | DstMem | SrcImm | No64, group1),
4670 G(DstMem | SrcImmByte, group1),
4671 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4672 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4674 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4675 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4676 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4677 D(ModRM | SrcMem | NoAccess | DstReg),
4678 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4681 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4683 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4684 I(SrcImmFAddr | No64, em_call_far), N,
4685 II(ImplicitOps | Stack, em_pushf, pushf),
4686 II(ImplicitOps | Stack, em_popf, popf),
4687 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4689 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4690 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4691 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4692 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4694 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4695 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4696 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4697 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4699 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4701 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4703 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4704 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4705 I(ImplicitOps | NearBranch, em_ret),
4706 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4707 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4708 G(ByteOp, group11), G(0, group11),
4710 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4711 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4712 I(ImplicitOps, em_ret_far),
4713 D(ImplicitOps), DI(SrcImmByte, intn),
4714 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4716 G(Src2One | ByteOp, group2), G(Src2One, group2),
4717 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4718 I(DstAcc | SrcImmUByte | No64, em_aam),
4719 I(DstAcc | SrcImmUByte | No64, em_aad),
4720 F(DstAcc | ByteOp | No64, em_salc),
4721 I(DstAcc | SrcXLat | ByteOp, em_mov),
4723 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4725 X3(I(SrcImmByte | NearBranch, em_loop)),
4726 I(SrcImmByte | NearBranch, em_jcxz),
4727 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4728 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4730 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4731 I(SrcImmFAddr | No64, em_jmp_far),
4732 D(SrcImmByte | ImplicitOps | NearBranch),
4733 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4734 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4736 N, DI(ImplicitOps, icebp), N, N,
4737 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4738 G(ByteOp, group3), G(0, group3),
4740 D(ImplicitOps), D(ImplicitOps),
4741 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4742 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4745 static const struct opcode twobyte_table[256] = {
4747 G(0, group6), GD(0, &group7), N, N,
4748 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4749 II(ImplicitOps | Priv, em_clts, clts), N,
4750 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4751 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4753 N, N, N, N, N, N, N, N,
4754 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4755 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4757 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4758 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4759 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4761 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4764 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4765 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4766 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4769 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4770 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4771 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4772 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4773 I(ImplicitOps | EmulateOnUD, em_sysenter),
4774 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4776 N, N, N, N, N, N, N, N,
4778 X16(D(DstReg | SrcMem | ModRM)),
4780 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4785 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4790 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4792 X16(D(SrcImm | NearBranch)),
4794 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4796 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4797 II(ImplicitOps, em_cpuid, cpuid),
4798 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4799 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4800 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4802 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4803 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4804 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4805 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4806 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4807 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4809 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4810 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4811 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4812 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4813 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4814 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4818 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4819 I(DstReg | SrcMem | ModRM, em_bsf_c),
4820 I(DstReg | SrcMem | ModRM, em_bsr_c),
4821 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4823 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4824 N, ID(0, &instr_dual_0f_c3),
4825 N, N, N, GD(0, &group9),
4827 X8(I(DstReg, em_bswap)),
4829 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4831 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4832 N, N, N, N, N, N, N, N,
4834 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4837 static const struct instr_dual instr_dual_0f_38_f0 = {
4838 I(DstReg | SrcMem | Mov, em_movbe), N
4841 static const struct instr_dual instr_dual_0f_38_f1 = {
4842 I(DstMem | SrcReg | Mov, em_movbe), N
4845 static const struct gprefix three_byte_0f_38_f0 = {
4846 ID(0, &instr_dual_0f_38_f0), N, N, N
4849 static const struct gprefix three_byte_0f_38_f1 = {
4850 ID(0, &instr_dual_0f_38_f1), N, N, N
4854 * Insns below are selected by the prefix which indexed by the third opcode
4857 static const struct opcode opcode_map_0f_38[256] = {
4859 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4861 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4863 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4864 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4885 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4889 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4895 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4896 unsigned size, bool sign_extension)
4898 int rc = X86EMUL_CONTINUE;
4902 op->addr.mem.ea = ctxt->_eip;
4903 /* NB. Immediates are sign-extended as necessary. */
4904 switch (op->bytes) {
4906 op->val = insn_fetch(s8, ctxt);
4909 op->val = insn_fetch(s16, ctxt);
4912 op->val = insn_fetch(s32, ctxt);
4915 op->val = insn_fetch(s64, ctxt);
4918 if (!sign_extension) {
4919 switch (op->bytes) {
4927 op->val &= 0xffffffff;
4935 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4938 int rc = X86EMUL_CONTINUE;
4942 decode_register_operand(ctxt, op);
4945 rc = decode_imm(ctxt, op, 1, false);
4948 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4952 if (ctxt->d & BitOp)
4953 fetch_bit_operand(ctxt);
4954 op->orig_val = op->val;
4957 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4961 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4962 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4963 fetch_register_operand(op);
4964 op->orig_val = op->val;
4968 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4969 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4970 fetch_register_operand(op);
4971 op->orig_val = op->val;
4974 if (ctxt->d & ByteOp) {
4979 op->bytes = ctxt->op_bytes;
4980 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4981 fetch_register_operand(op);
4982 op->orig_val = op->val;
4986 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4988 register_address(ctxt, VCPU_REGS_RDI);
4989 op->addr.mem.seg = VCPU_SREG_ES;
4996 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4997 fetch_register_operand(op);
5002 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5005 rc = decode_imm(ctxt, op, 1, true);
5013 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5016 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5019 ctxt->memop.bytes = 1;
5020 if (ctxt->memop.type == OP_REG) {
5021 ctxt->memop.addr.reg = decode_register(ctxt,
5022 ctxt->modrm_rm, true);
5023 fetch_register_operand(&ctxt->memop);
5027 ctxt->memop.bytes = 2;
5030 ctxt->memop.bytes = 4;
5033 rc = decode_imm(ctxt, op, 2, false);
5036 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5040 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5042 register_address(ctxt, VCPU_REGS_RSI);
5043 op->addr.mem.seg = ctxt->seg_override;
5049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5052 reg_read(ctxt, VCPU_REGS_RBX) +
5053 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5054 op->addr.mem.seg = ctxt->seg_override;
5059 op->addr.mem.ea = ctxt->_eip;
5060 op->bytes = ctxt->op_bytes + 2;
5061 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5064 ctxt->memop.bytes = ctxt->op_bytes + 2;
5068 op->val = VCPU_SREG_ES;
5072 op->val = VCPU_SREG_CS;
5076 op->val = VCPU_SREG_SS;
5080 op->val = VCPU_SREG_DS;
5084 op->val = VCPU_SREG_FS;
5088 op->val = VCPU_SREG_GS;
5091 /* Special instructions do their own operand decoding. */
5093 op->type = OP_NONE; /* Disable writeback. */
5101 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5103 int rc = X86EMUL_CONTINUE;
5104 int mode = ctxt->mode;
5105 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5106 bool op_prefix = false;
5107 bool has_seg_override = false;
5108 struct opcode opcode;
5110 struct desc_struct desc;
5112 ctxt->memop.type = OP_NONE;
5113 ctxt->memopp = NULL;
5114 ctxt->_eip = ctxt->eip;
5115 ctxt->fetch.ptr = ctxt->fetch.data;
5116 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5117 ctxt->opcode_len = 1;
5118 ctxt->intercept = x86_intercept_none;
5120 memcpy(ctxt->fetch.data, insn, insn_len);
5122 rc = __do_insn_fetch_bytes(ctxt, 1);
5123 if (rc != X86EMUL_CONTINUE)
5128 case X86EMUL_MODE_REAL:
5129 case X86EMUL_MODE_VM86:
5130 def_op_bytes = def_ad_bytes = 2;
5131 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5133 def_op_bytes = def_ad_bytes = 4;
5135 case X86EMUL_MODE_PROT16:
5136 def_op_bytes = def_ad_bytes = 2;
5138 case X86EMUL_MODE_PROT32:
5139 def_op_bytes = def_ad_bytes = 4;
5141 #ifdef CONFIG_X86_64
5142 case X86EMUL_MODE_PROT64:
5148 return EMULATION_FAILED;
5151 ctxt->op_bytes = def_op_bytes;
5152 ctxt->ad_bytes = def_ad_bytes;
5154 /* Legacy prefixes. */
5156 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5157 case 0x66: /* operand-size override */
5159 /* switch between 2/4 bytes */
5160 ctxt->op_bytes = def_op_bytes ^ 6;
5162 case 0x67: /* address-size override */
5163 if (mode == X86EMUL_MODE_PROT64)
5164 /* switch between 4/8 bytes */
5165 ctxt->ad_bytes = def_ad_bytes ^ 12;
5167 /* switch between 2/4 bytes */
5168 ctxt->ad_bytes = def_ad_bytes ^ 6;
5170 case 0x26: /* ES override */
5171 has_seg_override = true;
5172 ctxt->seg_override = VCPU_SREG_ES;
5174 case 0x2e: /* CS override */
5175 has_seg_override = true;
5176 ctxt->seg_override = VCPU_SREG_CS;
5178 case 0x36: /* SS override */
5179 has_seg_override = true;
5180 ctxt->seg_override = VCPU_SREG_SS;
5182 case 0x3e: /* DS override */
5183 has_seg_override = true;
5184 ctxt->seg_override = VCPU_SREG_DS;
5186 case 0x64: /* FS override */
5187 has_seg_override = true;
5188 ctxt->seg_override = VCPU_SREG_FS;
5190 case 0x65: /* GS override */
5191 has_seg_override = true;
5192 ctxt->seg_override = VCPU_SREG_GS;
5194 case 0x40 ... 0x4f: /* REX */
5195 if (mode != X86EMUL_MODE_PROT64)
5197 ctxt->rex_prefix = ctxt->b;
5199 case 0xf0: /* LOCK */
5200 ctxt->lock_prefix = 1;
5202 case 0xf2: /* REPNE/REPNZ */
5203 case 0xf3: /* REP/REPE/REPZ */
5204 ctxt->rep_prefix = ctxt->b;
5210 /* Any legacy prefix after a REX prefix nullifies its effect. */
5212 ctxt->rex_prefix = 0;
5218 if (ctxt->rex_prefix & 8)
5219 ctxt->op_bytes = 8; /* REX.W */
5221 /* Opcode byte(s). */
5222 opcode = opcode_table[ctxt->b];
5223 /* Two-byte opcode? */
5224 if (ctxt->b == 0x0f) {
5225 ctxt->opcode_len = 2;
5226 ctxt->b = insn_fetch(u8, ctxt);
5227 opcode = twobyte_table[ctxt->b];
5229 /* 0F_38 opcode map */
5230 if (ctxt->b == 0x38) {
5231 ctxt->opcode_len = 3;
5232 ctxt->b = insn_fetch(u8, ctxt);
5233 opcode = opcode_map_0f_38[ctxt->b];
5236 ctxt->d = opcode.flags;
5238 if (ctxt->d & ModRM)
5239 ctxt->modrm = insn_fetch(u8, ctxt);
5241 /* vex-prefix instructions are not implemented */
5242 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5243 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5247 while (ctxt->d & GroupMask) {
5248 switch (ctxt->d & GroupMask) {
5250 goffset = (ctxt->modrm >> 3) & 7;
5251 opcode = opcode.u.group[goffset];
5254 goffset = (ctxt->modrm >> 3) & 7;
5255 if ((ctxt->modrm >> 6) == 3)
5256 opcode = opcode.u.gdual->mod3[goffset];
5258 opcode = opcode.u.gdual->mod012[goffset];
5261 goffset = ctxt->modrm & 7;
5262 opcode = opcode.u.group[goffset];
5265 if (ctxt->rep_prefix && op_prefix)
5266 return EMULATION_FAILED;
5267 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5268 switch (simd_prefix) {
5269 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5270 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5271 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5272 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5276 if (ctxt->modrm > 0xbf) {
5277 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5278 u32 index = array_index_nospec(
5279 ctxt->modrm - 0xc0, size);
5281 opcode = opcode.u.esc->high[index];
5283 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5287 if ((ctxt->modrm >> 6) == 3)
5288 opcode = opcode.u.idual->mod3;
5290 opcode = opcode.u.idual->mod012;
5293 if (ctxt->mode == X86EMUL_MODE_PROT64)
5294 opcode = opcode.u.mdual->mode64;
5296 opcode = opcode.u.mdual->mode32;
5299 return EMULATION_FAILED;
5302 ctxt->d &= ~(u64)GroupMask;
5303 ctxt->d |= opcode.flags;
5308 return EMULATION_FAILED;
5310 ctxt->execute = opcode.u.execute;
5312 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5313 return EMULATION_FAILED;
5315 if (unlikely(ctxt->d &
5316 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5319 * These are copied unconditionally here, and checked unconditionally
5320 * in x86_emulate_insn.
5322 ctxt->check_perm = opcode.check_perm;
5323 ctxt->intercept = opcode.intercept;
5325 if (ctxt->d & NotImpl)
5326 return EMULATION_FAILED;
5328 if (mode == X86EMUL_MODE_PROT64) {
5329 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5331 else if (ctxt->d & NearBranch)
5335 if (ctxt->d & Op3264) {
5336 if (mode == X86EMUL_MODE_PROT64)
5342 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5346 ctxt->op_bytes = 16;
5347 else if (ctxt->d & Mmx)
5351 /* ModRM and SIB bytes. */
5352 if (ctxt->d & ModRM) {
5353 rc = decode_modrm(ctxt, &ctxt->memop);
5354 if (!has_seg_override) {
5355 has_seg_override = true;
5356 ctxt->seg_override = ctxt->modrm_seg;
5358 } else if (ctxt->d & MemAbs)
5359 rc = decode_abs(ctxt, &ctxt->memop);
5360 if (rc != X86EMUL_CONTINUE)
5363 if (!has_seg_override)
5364 ctxt->seg_override = VCPU_SREG_DS;
5366 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5369 * Decode and fetch the source operand: register, memory
5372 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5373 if (rc != X86EMUL_CONTINUE)
5377 * Decode and fetch the second source operand: register, memory
5380 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5381 if (rc != X86EMUL_CONTINUE)
5384 /* Decode and fetch the destination operand: register or memory. */
5385 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5387 if (ctxt->rip_relative && likely(ctxt->memopp))
5388 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5389 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5392 if (rc == X86EMUL_PROPAGATE_FAULT)
5393 ctxt->have_exception = true;
5394 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5397 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5399 return ctxt->d & PageTable;
5402 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5404 /* The second termination condition only applies for REPE
5405 * and REPNE. Test if the repeat string operation prefix is
5406 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5407 * corresponding termination condition according to:
5408 * - if REPE/REPZ and ZF = 0 then done
5409 * - if REPNE/REPNZ and ZF = 1 then done
5411 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5412 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5413 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5414 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5415 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5416 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5422 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5426 ctxt->ops->get_fpu(ctxt);
5427 rc = asm_safe("fwait");
5428 ctxt->ops->put_fpu(ctxt);
5430 if (unlikely(rc != X86EMUL_CONTINUE))
5431 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5433 return X86EMUL_CONTINUE;
5436 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5439 if (op->type == OP_MM)
5440 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5443 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5445 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5447 if (!(ctxt->d & ByteOp))
5448 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5450 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5451 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5452 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5453 : "c"(ctxt->src2.val));
5455 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5456 if (!fop) /* exception is returned in fop variable */
5457 return emulate_de(ctxt);
5458 return X86EMUL_CONTINUE;
5461 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5463 memset(&ctxt->rip_relative, 0,
5464 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5466 ctxt->io_read.pos = 0;
5467 ctxt->io_read.end = 0;
5468 ctxt->mem_read.end = 0;
5471 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5473 const struct x86_emulate_ops *ops = ctxt->ops;
5474 int rc = X86EMUL_CONTINUE;
5475 int saved_dst_type = ctxt->dst.type;
5476 unsigned emul_flags;
5478 ctxt->mem_read.pos = 0;
5480 /* LOCK prefix is allowed only with some instructions */
5481 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5482 rc = emulate_ud(ctxt);
5486 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5487 rc = emulate_ud(ctxt);
5491 emul_flags = ctxt->ops->get_hflags(ctxt);
5492 if (unlikely(ctxt->d &
5493 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5494 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5495 (ctxt->d & Undefined)) {
5496 rc = emulate_ud(ctxt);
5500 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5501 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5502 rc = emulate_ud(ctxt);
5506 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5507 rc = emulate_nm(ctxt);
5511 if (ctxt->d & Mmx) {
5512 rc = flush_pending_x87_faults(ctxt);
5513 if (rc != X86EMUL_CONTINUE)
5516 * Now that we know the fpu is exception safe, we can fetch
5519 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5520 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5521 if (!(ctxt->d & Mov))
5522 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5525 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5526 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5527 X86_ICPT_PRE_EXCEPT);
5528 if (rc != X86EMUL_CONTINUE)
5532 /* Instruction can only be executed in protected mode */
5533 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5534 rc = emulate_ud(ctxt);
5538 /* Privileged instruction can be executed only in CPL=0 */
5539 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5540 if (ctxt->d & PrivUD)
5541 rc = emulate_ud(ctxt);
5543 rc = emulate_gp(ctxt, 0);
5547 /* Do instruction specific permission checks */
5548 if (ctxt->d & CheckPerm) {
5549 rc = ctxt->check_perm(ctxt);
5550 if (rc != X86EMUL_CONTINUE)
5554 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5555 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5556 X86_ICPT_POST_EXCEPT);
5557 if (rc != X86EMUL_CONTINUE)
5561 if (ctxt->rep_prefix && (ctxt->d & String)) {
5562 /* All REP prefixes have the same first termination condition */
5563 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5564 string_registers_quirk(ctxt);
5565 ctxt->eip = ctxt->_eip;
5566 ctxt->eflags &= ~X86_EFLAGS_RF;
5572 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5573 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5574 ctxt->src.valptr, ctxt->src.bytes);
5575 if (rc != X86EMUL_CONTINUE)
5577 ctxt->src.orig_val64 = ctxt->src.val64;
5580 if (ctxt->src2.type == OP_MEM) {
5581 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5582 &ctxt->src2.val, ctxt->src2.bytes);
5583 if (rc != X86EMUL_CONTINUE)
5587 if ((ctxt->d & DstMask) == ImplicitOps)
5591 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5592 /* optimisation - avoid slow emulated read if Mov */
5593 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5594 &ctxt->dst.val, ctxt->dst.bytes);
5595 if (rc != X86EMUL_CONTINUE) {
5596 if (!(ctxt->d & NoWrite) &&
5597 rc == X86EMUL_PROPAGATE_FAULT &&
5598 ctxt->exception.vector == PF_VECTOR)
5599 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5603 /* Copy full 64-bit value for CMPXCHG8B. */
5604 ctxt->dst.orig_val64 = ctxt->dst.val64;
5608 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5609 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5610 X86_ICPT_POST_MEMACCESS);
5611 if (rc != X86EMUL_CONTINUE)
5615 if (ctxt->rep_prefix && (ctxt->d & String))
5616 ctxt->eflags |= X86_EFLAGS_RF;
5618 ctxt->eflags &= ~X86_EFLAGS_RF;
5620 if (ctxt->execute) {
5621 if (ctxt->d & Fastop) {
5622 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5623 rc = fastop(ctxt, fop);
5624 if (rc != X86EMUL_CONTINUE)
5628 rc = ctxt->execute(ctxt);
5629 if (rc != X86EMUL_CONTINUE)
5634 if (ctxt->opcode_len == 2)
5636 else if (ctxt->opcode_len == 3)
5637 goto threebyte_insn;
5640 case 0x70 ... 0x7f: /* jcc (short) */
5641 if (test_cc(ctxt->b, ctxt->eflags))
5642 rc = jmp_rel(ctxt, ctxt->src.val);
5644 case 0x8d: /* lea r16/r32, m */
5645 ctxt->dst.val = ctxt->src.addr.mem.ea;
5647 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5648 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5649 ctxt->dst.type = OP_NONE;
5653 case 0x98: /* cbw/cwde/cdqe */
5654 switch (ctxt->op_bytes) {
5655 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5656 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5657 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5660 case 0xcc: /* int3 */
5661 rc = emulate_int(ctxt, 3);
5663 case 0xcd: /* int n */
5664 rc = emulate_int(ctxt, ctxt->src.val);
5666 case 0xce: /* into */
5667 if (ctxt->eflags & X86_EFLAGS_OF)
5668 rc = emulate_int(ctxt, 4);
5670 case 0xe9: /* jmp rel */
5671 case 0xeb: /* jmp rel short */
5672 rc = jmp_rel(ctxt, ctxt->src.val);
5673 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5675 case 0xf4: /* hlt */
5676 ctxt->ops->halt(ctxt);
5678 case 0xf5: /* cmc */
5679 /* complement carry flag from eflags reg */
5680 ctxt->eflags ^= X86_EFLAGS_CF;
5682 case 0xf8: /* clc */
5683 ctxt->eflags &= ~X86_EFLAGS_CF;
5685 case 0xf9: /* stc */
5686 ctxt->eflags |= X86_EFLAGS_CF;
5688 case 0xfc: /* cld */
5689 ctxt->eflags &= ~X86_EFLAGS_DF;
5691 case 0xfd: /* std */
5692 ctxt->eflags |= X86_EFLAGS_DF;
5695 goto cannot_emulate;
5698 if (rc != X86EMUL_CONTINUE)
5702 if (ctxt->d & SrcWrite) {
5703 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5704 rc = writeback(ctxt, &ctxt->src);
5705 if (rc != X86EMUL_CONTINUE)
5708 if (!(ctxt->d & NoWrite)) {
5709 rc = writeback(ctxt, &ctxt->dst);
5710 if (rc != X86EMUL_CONTINUE)
5715 * restore dst type in case the decoding will be reused
5716 * (happens for string instruction )
5718 ctxt->dst.type = saved_dst_type;
5720 if ((ctxt->d & SrcMask) == SrcSI)
5721 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5723 if ((ctxt->d & DstMask) == DstDI)
5724 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5726 if (ctxt->rep_prefix && (ctxt->d & String)) {
5728 struct read_cache *r = &ctxt->io_read;
5729 if ((ctxt->d & SrcMask) == SrcSI)
5730 count = ctxt->src.count;
5732 count = ctxt->dst.count;
5733 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5735 if (!string_insn_completed(ctxt)) {
5737 * Re-enter guest when pio read ahead buffer is empty
5738 * or, if it is not used, after each 1024 iteration.
5740 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5741 (r->end == 0 || r->end != r->pos)) {
5743 * Reset read cache. Usually happens before
5744 * decode, but since instruction is restarted
5745 * we have to do it here.
5747 ctxt->mem_read.end = 0;
5748 writeback_registers(ctxt);
5749 return EMULATION_RESTART;
5751 goto done; /* skip rip writeback */
5753 ctxt->eflags &= ~X86_EFLAGS_RF;
5756 ctxt->eip = ctxt->_eip;
5759 if (rc == X86EMUL_PROPAGATE_FAULT) {
5760 WARN_ON(ctxt->exception.vector > 0x1f);
5761 ctxt->have_exception = true;
5763 if (rc == X86EMUL_INTERCEPTED)
5764 return EMULATION_INTERCEPTED;
5766 if (rc == X86EMUL_CONTINUE)
5767 writeback_registers(ctxt);
5769 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5773 case 0x09: /* wbinvd */
5774 (ctxt->ops->wbinvd)(ctxt);
5776 case 0x08: /* invd */
5777 case 0x0d: /* GrpP (prefetch) */
5778 case 0x18: /* Grp16 (prefetch/nop) */
5779 case 0x1f: /* nop */
5781 case 0x20: /* mov cr, reg */
5782 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5784 case 0x21: /* mov from dr to reg */
5785 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5787 case 0x40 ... 0x4f: /* cmov */
5788 if (test_cc(ctxt->b, ctxt->eflags))
5789 ctxt->dst.val = ctxt->src.val;
5790 else if (ctxt->op_bytes != 4)
5791 ctxt->dst.type = OP_NONE; /* no writeback */
5793 case 0x80 ... 0x8f: /* jnz rel, etc*/
5794 if (test_cc(ctxt->b, ctxt->eflags))
5795 rc = jmp_rel(ctxt, ctxt->src.val);
5797 case 0x90 ... 0x9f: /* setcc r/m8 */
5798 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5800 case 0xb6 ... 0xb7: /* movzx */
5801 ctxt->dst.bytes = ctxt->op_bytes;
5802 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5803 : (u16) ctxt->src.val;
5805 case 0xbe ... 0xbf: /* movsx */
5806 ctxt->dst.bytes = ctxt->op_bytes;
5807 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5808 (s16) ctxt->src.val;
5811 goto cannot_emulate;
5816 if (rc != X86EMUL_CONTINUE)
5822 return EMULATION_FAILED;
5825 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5827 invalidate_registers(ctxt);
5830 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5832 writeback_registers(ctxt);
5835 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5837 if (ctxt->rep_prefix && (ctxt->d & String))
5840 if (ctxt->d & TwoMemOp)