GNU Linux-libre 6.9-gnu
[releases.git] / mm / kmsan / hooks.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * KMSAN hooks for kernel subsystems.
4  *
5  * These functions handle creation of KMSAN metadata for memory allocations.
6  *
7  * Copyright (C) 2018-2022 Google LLC
8  * Author: Alexander Potapenko <glider@google.com>
9  *
10  */
11
12 #include <linux/cacheflush.h>
13 #include <linux/dma-direction.h>
14 #include <linux/gfp.h>
15 #include <linux/kmsan.h>
16 #include <linux/mm.h>
17 #include <linux/mm_types.h>
18 #include <linux/scatterlist.h>
19 #include <linux/slab.h>
20 #include <linux/uaccess.h>
21 #include <linux/usb.h>
22
23 #include "../internal.h"
24 #include "../slab.h"
25 #include "kmsan.h"
26
27 /*
28  * Instrumented functions shouldn't be called under
29  * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
30  * skipping effects of functions like memset() inside instrumented code.
31  */
32
33 void kmsan_task_create(struct task_struct *task)
34 {
35         kmsan_enter_runtime();
36         kmsan_internal_task_create(task);
37         kmsan_leave_runtime();
38 }
39
40 void kmsan_task_exit(struct task_struct *task)
41 {
42         struct kmsan_ctx *ctx = &task->kmsan_ctx;
43
44         if (!kmsan_enabled || kmsan_in_runtime())
45                 return;
46
47         ctx->allow_reporting = false;
48 }
49
50 void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
51 {
52         if (unlikely(object == NULL))
53                 return;
54         if (!kmsan_enabled || kmsan_in_runtime())
55                 return;
56         /*
57          * There's a ctor or this is an RCU cache - do nothing. The memory
58          * status hasn't changed since last use.
59          */
60         if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
61                 return;
62
63         kmsan_enter_runtime();
64         if (flags & __GFP_ZERO)
65                 kmsan_internal_unpoison_memory(object, s->object_size,
66                                                KMSAN_POISON_CHECK);
67         else
68                 kmsan_internal_poison_memory(object, s->object_size, flags,
69                                              KMSAN_POISON_CHECK);
70         kmsan_leave_runtime();
71 }
72
73 void kmsan_slab_free(struct kmem_cache *s, void *object)
74 {
75         if (!kmsan_enabled || kmsan_in_runtime())
76                 return;
77
78         /* RCU slabs could be legally used after free within the RCU period */
79         if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
80                 return;
81         /*
82          * If there's a constructor, freed memory must remain in the same state
83          * until the next allocation. We cannot save its state to detect
84          * use-after-free bugs, instead we just keep it unpoisoned.
85          */
86         if (s->ctor)
87                 return;
88         kmsan_enter_runtime();
89         kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
90                                      KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
91         kmsan_leave_runtime();
92 }
93
94 void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
95 {
96         if (unlikely(ptr == NULL))
97                 return;
98         if (!kmsan_enabled || kmsan_in_runtime())
99                 return;
100         kmsan_enter_runtime();
101         if (flags & __GFP_ZERO)
102                 kmsan_internal_unpoison_memory((void *)ptr, size,
103                                                /*checked*/ true);
104         else
105                 kmsan_internal_poison_memory((void *)ptr, size, flags,
106                                              KMSAN_POISON_CHECK);
107         kmsan_leave_runtime();
108 }
109
110 void kmsan_kfree_large(const void *ptr)
111 {
112         struct page *page;
113
114         if (!kmsan_enabled || kmsan_in_runtime())
115                 return;
116         kmsan_enter_runtime();
117         page = virt_to_head_page((void *)ptr);
118         KMSAN_WARN_ON(ptr != page_address(page));
119         kmsan_internal_poison_memory((void *)ptr,
120                                      page_size(page),
121                                      GFP_KERNEL,
122                                      KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
123         kmsan_leave_runtime();
124 }
125
126 static unsigned long vmalloc_shadow(unsigned long addr)
127 {
128         return (unsigned long)kmsan_get_metadata((void *)addr,
129                                                  KMSAN_META_SHADOW);
130 }
131
132 static unsigned long vmalloc_origin(unsigned long addr)
133 {
134         return (unsigned long)kmsan_get_metadata((void *)addr,
135                                                  KMSAN_META_ORIGIN);
136 }
137
138 void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
139 {
140         __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
141         __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
142         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
143         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
144 }
145
146 /*
147  * This function creates new shadow/origin pages for the physical pages mapped
148  * into the virtual memory. If those physical pages already had shadow/origin,
149  * those are ignored.
150  */
151 int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
152                              phys_addr_t phys_addr, pgprot_t prot,
153                              unsigned int page_shift)
154 {
155         gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;
156         struct page *shadow, *origin;
157         unsigned long off = 0;
158         int nr, err = 0, clean = 0, mapped;
159
160         if (!kmsan_enabled || kmsan_in_runtime())
161                 return 0;
162
163         nr = (end - start) / PAGE_SIZE;
164         kmsan_enter_runtime();
165         for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
166                 shadow = alloc_pages(gfp_mask, 1);
167                 origin = alloc_pages(gfp_mask, 1);
168                 if (!shadow || !origin) {
169                         err = -ENOMEM;
170                         goto ret;
171                 }
172                 mapped = __vmap_pages_range_noflush(
173                         vmalloc_shadow(start + off),
174                         vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
175                         PAGE_SHIFT);
176                 if (mapped) {
177                         err = mapped;
178                         goto ret;
179                 }
180                 shadow = NULL;
181                 mapped = __vmap_pages_range_noflush(
182                         vmalloc_origin(start + off),
183                         vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
184                         PAGE_SHIFT);
185                 if (mapped) {
186                         __vunmap_range_noflush(
187                                 vmalloc_shadow(start + off),
188                                 vmalloc_shadow(start + off + PAGE_SIZE));
189                         err = mapped;
190                         goto ret;
191                 }
192                 origin = NULL;
193         }
194         /* Page mapping loop finished normally, nothing to clean up. */
195         clean = 0;
196
197 ret:
198         if (clean > 0) {
199                 /*
200                  * Something went wrong. Clean up shadow/origin pages allocated
201                  * on the last loop iteration, then delete mappings created
202                  * during the previous iterations.
203                  */
204                 if (shadow)
205                         __free_pages(shadow, 1);
206                 if (origin)
207                         __free_pages(origin, 1);
208                 __vunmap_range_noflush(
209                         vmalloc_shadow(start),
210                         vmalloc_shadow(start + clean * PAGE_SIZE));
211                 __vunmap_range_noflush(
212                         vmalloc_origin(start),
213                         vmalloc_origin(start + clean * PAGE_SIZE));
214         }
215         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
216         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
217         kmsan_leave_runtime();
218         return err;
219 }
220
221 void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
222 {
223         unsigned long v_shadow, v_origin;
224         struct page *shadow, *origin;
225         int nr;
226
227         if (!kmsan_enabled || kmsan_in_runtime())
228                 return;
229
230         nr = (end - start) / PAGE_SIZE;
231         kmsan_enter_runtime();
232         v_shadow = (unsigned long)vmalloc_shadow(start);
233         v_origin = (unsigned long)vmalloc_origin(start);
234         for (int i = 0; i < nr;
235              i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
236                 shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
237                 origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
238                 __vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
239                 __vunmap_range_noflush(v_origin, vmalloc_origin(end));
240                 if (shadow)
241                         __free_pages(shadow, 1);
242                 if (origin)
243                         __free_pages(origin, 1);
244         }
245         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
246         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
247         kmsan_leave_runtime();
248 }
249
250 void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
251                         size_t left)
252 {
253         unsigned long ua_flags;
254
255         if (!kmsan_enabled || kmsan_in_runtime())
256                 return;
257         /*
258          * At this point we've copied the memory already. It's hard to check it
259          * before copying, as the size of actually copied buffer is unknown.
260          */
261
262         /* copy_to_user() may copy zero bytes. No need to check. */
263         if (!to_copy)
264                 return;
265         /* Or maybe copy_to_user() failed to copy anything. */
266         if (to_copy <= left)
267                 return;
268
269         ua_flags = user_access_save();
270         if ((u64)to < TASK_SIZE) {
271                 /* This is a user memory access, check it. */
272                 kmsan_internal_check_memory((void *)from, to_copy - left, to,
273                                             REASON_COPY_TO_USER);
274         } else {
275                 /* Otherwise this is a kernel memory access. This happens when a
276                  * compat syscall passes an argument allocated on the kernel
277                  * stack to a real syscall.
278                  * Don't check anything, just copy the shadow of the copied
279                  * bytes.
280                  */
281                 kmsan_internal_memmove_metadata((void *)to, (void *)from,
282                                                 to_copy - left);
283         }
284         user_access_restore(ua_flags);
285 }
286 EXPORT_SYMBOL(kmsan_copy_to_user);
287
288 /* Helper function to check an URB. */
289 void kmsan_handle_urb(const struct urb *urb, bool is_out)
290 {
291         if (!urb)
292                 return;
293         if (is_out)
294                 kmsan_internal_check_memory(urb->transfer_buffer,
295                                             urb->transfer_buffer_length,
296                                             /*user_addr*/ 0, REASON_SUBMIT_URB);
297         else
298                 kmsan_internal_unpoison_memory(urb->transfer_buffer,
299                                                urb->transfer_buffer_length,
300                                                /*checked*/ false);
301 }
302 EXPORT_SYMBOL_GPL(kmsan_handle_urb);
303
304 static void kmsan_handle_dma_page(const void *addr, size_t size,
305                                   enum dma_data_direction dir)
306 {
307         switch (dir) {
308         case DMA_BIDIRECTIONAL:
309                 kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
310                                             REASON_ANY);
311                 kmsan_internal_unpoison_memory((void *)addr, size,
312                                                /*checked*/ false);
313                 break;
314         case DMA_TO_DEVICE:
315                 kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
316                                             REASON_ANY);
317                 break;
318         case DMA_FROM_DEVICE:
319                 kmsan_internal_unpoison_memory((void *)addr, size,
320                                                /*checked*/ false);
321                 break;
322         case DMA_NONE:
323                 break;
324         }
325 }
326
327 /* Helper function to handle DMA data transfers. */
328 void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
329                       enum dma_data_direction dir)
330 {
331         u64 page_offset, to_go, addr;
332
333         if (PageHighMem(page))
334                 return;
335         addr = (u64)page_address(page) + offset;
336         /*
337          * The kernel may occasionally give us adjacent DMA pages not belonging
338          * to the same allocation. Process them separately to avoid triggering
339          * internal KMSAN checks.
340          */
341         while (size > 0) {
342                 page_offset = offset_in_page(addr);
343                 to_go = min(PAGE_SIZE - page_offset, (u64)size);
344                 kmsan_handle_dma_page((void *)addr, to_go, dir);
345                 addr += to_go;
346                 size -= to_go;
347         }
348 }
349
350 void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
351                          enum dma_data_direction dir)
352 {
353         struct scatterlist *item;
354         int i;
355
356         for_each_sg(sg, item, nents, i)
357                 kmsan_handle_dma(sg_page(item), item->offset, item->length,
358                                  dir);
359 }
360
361 /* Functions from kmsan-checks.h follow. */
362
363 /*
364  * To create an origin, kmsan_poison_memory() unwinds the stacks and stores it
365  * into the stack depot. This may cause deadlocks if done from within KMSAN
366  * runtime, therefore we bail out if kmsan_in_runtime().
367  */
368 void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
369 {
370         if (!kmsan_enabled || kmsan_in_runtime())
371                 return;
372         kmsan_enter_runtime();
373         /* The users may want to poison/unpoison random memory. */
374         kmsan_internal_poison_memory((void *)address, size, flags,
375                                      KMSAN_POISON_NOCHECK);
376         kmsan_leave_runtime();
377 }
378 EXPORT_SYMBOL(kmsan_poison_memory);
379
380 /*
381  * Unlike kmsan_poison_memory(), this function can be used from within KMSAN
382  * runtime, because it does not trigger allocations or call instrumented code.
383  */
384 void kmsan_unpoison_memory(const void *address, size_t size)
385 {
386         unsigned long ua_flags;
387
388         if (!kmsan_enabled)
389                 return;
390
391         ua_flags = user_access_save();
392         /* The users may want to poison/unpoison random memory. */
393         kmsan_internal_unpoison_memory((void *)address, size,
394                                        KMSAN_POISON_NOCHECK);
395         user_access_restore(ua_flags);
396 }
397 EXPORT_SYMBOL(kmsan_unpoison_memory);
398
399 /*
400  * Version of kmsan_unpoison_memory() called from IRQ entry functions.
401  */
402 void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
403 {
404         kmsan_unpoison_memory((void *)regs, sizeof(*regs));
405 }
406
407 void kmsan_check_memory(const void *addr, size_t size)
408 {
409         if (!kmsan_enabled)
410                 return;
411         return kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
412                                            REASON_ANY);
413 }
414 EXPORT_SYMBOL(kmsan_check_memory);