2 * Copyright (C) 2017 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/kvm_host.h>
19 #include <linux/random.h>
20 #include <linux/memblock.h>
21 #include <asm/alternative.h>
22 #include <asm/debug-monitors.h>
24 #include <asm/kvm_mmu.h>
27 * The LSB of the random hyp VA tag or 0 if no randomization is used.
31 * The random hyp VA tag value with the region bit if hyp randomization is used
36 static void compute_layout(void)
38 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
42 /* Where is my RAM region? */
43 hyp_va_msb = idmap_addr & BIT(VA_BITS - 1);
44 hyp_va_msb ^= BIT(VA_BITS - 1);
46 kva_msb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
47 (u64)(high_memory - 1));
49 if (kva_msb == (VA_BITS - 1)) {
51 * No space in the address, let's compute the mask so
52 * that it covers (VA_BITS - 1) bits, and the region
53 * bit. The tag stays set to zero.
55 va_mask = BIT(VA_BITS - 1) - 1;
56 va_mask |= hyp_va_msb;
59 * We do have some free bits to insert a random tag.
60 * Hyp VAs are now created from kernel linear map VAs
61 * using the following formula (with V == VA_BITS):
63 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0
64 * ---------------------------------------------------------
65 * | 0000000 | hyp_va_msb | random tag | kern linear VA |
68 va_mask = GENMASK_ULL(tag_lsb - 1, 0);
69 tag_val = get_random_long() & GENMASK_ULL(VA_BITS - 2, tag_lsb);
70 tag_val |= hyp_va_msb;
75 static u32 compute_instruction(int n, u32 rd, u32 rn)
77 u32 insn = AARCH64_BREAK_FAULT;
81 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
82 AARCH64_INSN_VARIANT_64BIT,
87 /* ROR is a variant of EXTR with Rm = Rn */
88 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
94 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
95 tag_val & GENMASK(11, 0),
96 AARCH64_INSN_VARIANT_64BIT,
97 AARCH64_INSN_ADSB_ADD);
101 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
102 tag_val & GENMASK(23, 12),
103 AARCH64_INSN_VARIANT_64BIT,
104 AARCH64_INSN_ADSB_ADD);
108 /* ROR is a variant of EXTR with Rm = Rn */
109 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
110 rn, rn, rd, 64 - tag_lsb);
117 void __init kvm_update_va_mask(struct alt_instr *alt,
118 __le32 *origptr, __le32 *updptr, int nr_inst)
122 BUG_ON(nr_inst != 5);
124 if (!has_vhe() && !va_mask)
127 for (i = 0; i < nr_inst; i++) {
128 u32 rd, rn, insn, oinsn;
131 * VHE doesn't need any address translation, let's NOP
134 * Alternatively, if we don't have any spare bits in
135 * the address, NOP everything after masking that
138 if (has_vhe() || (!tag_lsb && i > 0)) {
139 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
143 oinsn = le32_to_cpu(origptr[i]);
144 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
145 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
147 insn = compute_instruction(i, rd, rn);
148 BUG_ON(insn == AARCH64_BREAK_FAULT);
150 updptr[i] = cpu_to_le32(insn);
154 void *__kvm_bp_vect_base;
155 int __kvm_harden_el2_vector_slot;
157 void kvm_patch_vector_branch(struct alt_instr *alt,
158 __le32 *origptr, __le32 *updptr, int nr_inst)
163 BUG_ON(nr_inst != 5);
165 if (has_vhe() || !cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
166 WARN_ON_ONCE(cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS));
174 * Compute HYP VA by using the same computation as kern_hyp_va()
176 addr = (uintptr_t)kvm_ksym_ref(__kvm_hyp_vector);
178 addr |= tag_val << tag_lsb;
180 /* Use PC[10:7] to branch to the same vector in KVM */
181 addr |= ((u64)origptr & GENMASK_ULL(10, 7));
184 * Branch to the second instruction in the vectors in order to
185 * avoid the initial store on the stack (which we already
186 * perform in the hardening vectors).
188 addr += AARCH64_INSN_SIZE;
190 /* stp x0, x1, [sp, #-16]! */
191 insn = aarch64_insn_gen_load_store_pair(AARCH64_INSN_REG_0,
195 AARCH64_INSN_VARIANT_64BIT,
196 AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX);
197 *updptr++ = cpu_to_le32(insn);
199 /* movz x0, #(addr & 0xffff) */
200 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
203 AARCH64_INSN_VARIANT_64BIT,
204 AARCH64_INSN_MOVEWIDE_ZERO);
205 *updptr++ = cpu_to_le32(insn);
207 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
208 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
211 AARCH64_INSN_VARIANT_64BIT,
212 AARCH64_INSN_MOVEWIDE_KEEP);
213 *updptr++ = cpu_to_le32(insn);
215 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
216 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
219 AARCH64_INSN_VARIANT_64BIT,
220 AARCH64_INSN_MOVEWIDE_KEEP);
221 *updptr++ = cpu_to_le32(insn);
224 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
225 AARCH64_INSN_BRANCH_NOLINK);
226 *updptr++ = cpu_to_le32(insn);