mm/kmsan/hooks.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KMSAN hooks for kernel subsystems.
   4  *
   5  * These functions handle creation of KMSAN metadata for memory allocations.
   6  *
   7  * Copyright (C) 2018-2022 Google LLC
   8  * Author: Alexander Potapenko <glider@google.com>
   9  *
  10  */
  11
  12 #include <linux/cacheflush.h>
  13 #include <linux/dma-direction.h>
  14 #include <linux/gfp.h>
  15 #include <linux/kmsan.h>
  16 #include <linux/mm.h>
  17 #include <linux/mm_types.h>
  18 #include <linux/scatterlist.h>
  19 #include <linux/slab.h>
  20 #include <linux/uaccess.h>
  21 #include <linux/usb.h>
  22
  23 #include "../internal.h"
  24 #include "../slab.h"
  25 #include "kmsan.h"
  26
  27 /*
  28  * Instrumented functions shouldn't be called under
  29  * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
  30  * skipping effects of functions like memset() inside instrumented code.
  31  */
  32
  33 void kmsan_task_create(struct task_struct *task)
  34 {
  35         kmsan_enter_runtime();
  36         kmsan_internal_task_create(task);
  37         kmsan_leave_runtime();
  38 }
  39
  40 void kmsan_task_exit(struct task_struct *task)
  41 {
  42         struct kmsan_ctx *ctx = &task->kmsan_ctx;
  43
  44         if (!kmsan_enabled || kmsan_in_runtime())
  45                 return;
  46
  47         ctx->allow_reporting = false;
  48 }
  49
  50 void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
  51 {
  52         if (unlikely(object == NULL))
  53                 return;
  54         if (!kmsan_enabled || kmsan_in_runtime())
  55                 return;
  56         /*
  57          * There's a ctor or this is an RCU cache - do nothing. The memory
  58          * status hasn't changed since last use.
  59          */
  60         if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
  61                 return;
  62
  63         kmsan_enter_runtime();
  64         if (flags & __GFP_ZERO)
  65                 kmsan_internal_unpoison_memory(object, s->object_size,
  66                                                KMSAN_POISON_CHECK);
  67         else
  68                 kmsan_internal_poison_memory(object, s->object_size, flags,
  69                                              KMSAN_POISON_CHECK);
  70         kmsan_leave_runtime();
  71 }
  72
  73 void kmsan_slab_free(struct kmem_cache *s, void *object)
  74 {
  75         if (!kmsan_enabled || kmsan_in_runtime())
  76                 return;
  77
  78         /* RCU slabs could be legally used after free within the RCU period */
  79         if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
  80                 return;
  81         /*
  82          * If there's a constructor, freed memory must remain in the same state
  83          * until the next allocation. We cannot save its state to detect
  84          * use-after-free bugs, instead we just keep it unpoisoned.
  85          */
  86         if (s->ctor)
  87                 return;
  88         kmsan_enter_runtime();
  89         kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
  90                                      KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
  91         kmsan_leave_runtime();
  92 }
  93
  94 void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
  95 {
  96         if (unlikely(ptr == NULL))
  97                 return;
  98         if (!kmsan_enabled || kmsan_in_runtime())
  99                 return;
 100         kmsan_enter_runtime();
 101         if (flags & __GFP_ZERO)
 102                 kmsan_internal_unpoison_memory((void *)ptr, size,
 103                                                /*checked*/ true);
 104         else
 105                 kmsan_internal_poison_memory((void *)ptr, size, flags,
 106                                              KMSAN_POISON_CHECK);
 107         kmsan_leave_runtime();
 108 }
 109
 110 void kmsan_kfree_large(const void *ptr)
 111 {
 112         struct page *page;
 113
 114         if (!kmsan_enabled || kmsan_in_runtime())
 115                 return;
 116         kmsan_enter_runtime();
 117         page = virt_to_head_page((void *)ptr);
 118         KMSAN_WARN_ON(ptr != page_address(page));
 119         kmsan_internal_poison_memory((void *)ptr,
 120                                      page_size(page),
 121                                      GFP_KERNEL,
 122                                      KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
 123         kmsan_leave_runtime();
 124 }
 125
 126 static unsigned long vmalloc_shadow(unsigned long addr)
 127 {
 128         return (unsigned long)kmsan_get_metadata((void *)addr,
 129                                                  KMSAN_META_SHADOW);
 130 }
 131
 132 static unsigned long vmalloc_origin(unsigned long addr)
 133 {
 134         return (unsigned long)kmsan_get_metadata((void *)addr,
 135                                                  KMSAN_META_ORIGIN);
 136 }
 137
 138 void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
 139 {
 140         __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
 141         __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
 142         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
 143         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
 144 }
 145
 146 /*
 147  * This function creates new shadow/origin pages for the physical pages mapped
 148  * into the virtual memory. If those physical pages already had shadow/origin,
 149  * those are ignored.
 150  */
 151 int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
 152                              phys_addr_t phys_addr, pgprot_t prot,
 153                              unsigned int page_shift)
 154 {
 155         gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;
 156         struct page *shadow, *origin;
 157         unsigned long off = 0;
 158         int nr, err = 0, clean = 0, mapped;
 159
 160         if (!kmsan_enabled || kmsan_in_runtime())
 161                 return 0;
 162
 163         nr = (end - start) / PAGE_SIZE;
 164         kmsan_enter_runtime();
 165         for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
 166                 shadow = alloc_pages(gfp_mask, 1);
 167                 origin = alloc_pages(gfp_mask, 1);
 168                 if (!shadow || !origin) {
 169                         err = -ENOMEM;
 170                         goto ret;
 171                 }
 172                 mapped = __vmap_pages_range_noflush(
 173                         vmalloc_shadow(start + off),
 174                         vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
 175                         PAGE_SHIFT);
 176                 if (mapped) {
 177                         err = mapped;
 178                         goto ret;
 179                 }
 180                 shadow = NULL;
 181                 mapped = __vmap_pages_range_noflush(
 182                         vmalloc_origin(start + off),
 183                         vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
 184                         PAGE_SHIFT);
 185                 if (mapped) {
 186                         __vunmap_range_noflush(
 187                                 vmalloc_shadow(start + off),
 188                                 vmalloc_shadow(start + off + PAGE_SIZE));
 189                         err = mapped;
 190                         goto ret;
 191                 }
 192                 origin = NULL;
 193         }
 194         /* Page mapping loop finished normally, nothing to clean up. */
 195         clean = 0;
 196
 197 ret:
 198         if (clean > 0) {
 199                 /*
 200                  * Something went wrong. Clean up shadow/origin pages allocated
 201                  * on the last loop iteration, then delete mappings created
 202                  * during the previous iterations.
 203                  */
 204                 if (shadow)
 205                         __free_pages(shadow, 1);
 206                 if (origin)
 207                         __free_pages(origin, 1);
 208                 __vunmap_range_noflush(
 209                         vmalloc_shadow(start),
 210                         vmalloc_shadow(start + clean * PAGE_SIZE));
 211                 __vunmap_range_noflush(
 212                         vmalloc_origin(start),
 213                         vmalloc_origin(start + clean * PAGE_SIZE));
 214         }
 215         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
 216         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
 217         kmsan_leave_runtime();
 218         return err;
 219 }
 220
 221 void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
 222 {
 223         unsigned long v_shadow, v_origin;
 224         struct page *shadow, *origin;
 225         int nr;
 226
 227         if (!kmsan_enabled || kmsan_in_runtime())
 228                 return;
 229
 230         nr = (end - start) / PAGE_SIZE;
 231         kmsan_enter_runtime();
 232         v_shadow = (unsigned long)vmalloc_shadow(start);
 233         v_origin = (unsigned long)vmalloc_origin(start);
 234         for (int i = 0; i < nr;
 235              i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
 236                 shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
 237                 origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
 238                 __vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
 239                 __vunmap_range_noflush(v_origin, vmalloc_origin(end));
 240                 if (shadow)
 241                         __free_pages(shadow, 1);
 242                 if (origin)
 243                         __free_pages(origin, 1);
 244         }
 245         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
 246         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
 247         kmsan_leave_runtime();
 248 }
 249
 250 void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
 251                         size_t left)
 252 {
 253         unsigned long ua_flags;
 254
 255         if (!kmsan_enabled || kmsan_in_runtime())
 256                 return;
 257         /*
 258          * At this point we've copied the memory already. It's hard to check it
 259          * before copying, as the size of actually copied buffer is unknown.
 260          */
 261
 262         /* copy_to_user() may copy zero bytes. No need to check. */
 263         if (!to_copy)
 264                 return;
 265         /* Or maybe copy_to_user() failed to copy anything. */
 266         if (to_copy <= left)
 267                 return;
 268
 269         ua_flags = user_access_save();
 270         if ((u64)to < TASK_SIZE) {
 271                 /* This is a user memory access, check it. */
 272                 kmsan_internal_check_memory((void *)from, to_copy - left, to,
 273                                             REASON_COPY_TO_USER);
 274         } else {
 275                 /* Otherwise this is a kernel memory access. This happens when a
 276                  * compat syscall passes an argument allocated on the kernel
 277                  * stack to a real syscall.
 278                  * Don't check anything, just copy the shadow of the copied
 279                  * bytes.
 280                  */
 281                 kmsan_internal_memmove_metadata((void *)to, (void *)from,
 282                                                 to_copy - left);
 283         }
 284         user_access_restore(ua_flags);
 285 }
 286 EXPORT_SYMBOL(kmsan_copy_to_user);
 287
 288 /* Helper function to check an URB. */
 289 void kmsan_handle_urb(const struct urb *urb, bool is_out)
 290 {
 291         if (!urb)
 292                 return;
 293         if (is_out)
 294                 kmsan_internal_check_memory(urb->transfer_buffer,
 295                                             urb->transfer_buffer_length,
 296                                             /*user_addr*/ 0, REASON_SUBMIT_URB);
 297         else
 298                 kmsan_internal_unpoison_memory(urb->transfer_buffer,
 299                                                urb->transfer_buffer_length,
 300                                                /*checked*/ false);
 301 }
 302 EXPORT_SYMBOL_GPL(kmsan_handle_urb);
 303
 304 static void kmsan_handle_dma_page(const void *addr, size_t size,
 305                                   enum dma_data_direction dir)
 306 {
 307         switch (dir) {
 308         case DMA_BIDIRECTIONAL:
 309                 kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
 310                                             REASON_ANY);
 311                 kmsan_internal_unpoison_memory((void *)addr, size,
 312                                                /*checked*/ false);
 313                 break;
 314         case DMA_TO_DEVICE:
 315                 kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
 316                                             REASON_ANY);
 317                 break;
 318         case DMA_FROM_DEVICE:
 319                 kmsan_internal_unpoison_memory((void *)addr, size,
 320                                                /*checked*/ false);
 321                 break;
 322         case DMA_NONE:
 323                 break;
 324         }
 325 }
 326
 327 /* Helper function to handle DMA data transfers. */
 328 void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
 329                       enum dma_data_direction dir)
 330 {
 331         u64 page_offset, to_go, addr;
 332
 333         if (PageHighMem(page))
 334                 return;
 335         addr = (u64)page_address(page) + offset;
 336         /*
 337          * The kernel may occasionally give us adjacent DMA pages not belonging
 338          * to the same allocation. Process them separately to avoid triggering
 339          * internal KMSAN checks.
 340          */
 341         while (size > 0) {
 342                 page_offset = offset_in_page(addr);
 343                 to_go = min(PAGE_SIZE - page_offset, (u64)size);
 344                 kmsan_handle_dma_page((void *)addr, to_go, dir);
 345                 addr += to_go;
 346                 size -= to_go;
 347         }
 348 }
 349
 350 void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
 351                          enum dma_data_direction dir)
 352 {
 353         struct scatterlist *item;
 354         int i;
 355
 356         for_each_sg(sg, item, nents, i)
 357                 kmsan_handle_dma(sg_page(item), item->offset, item->length,
 358                                  dir);
 359 }
 360
 361 /* Functions from kmsan-checks.h follow. */
 362
 363 /*
 364  * To create an origin, kmsan_poison_memory() unwinds the stacks and stores it
 365  * into the stack depot. This may cause deadlocks if done from within KMSAN
 366  * runtime, therefore we bail out if kmsan_in_runtime().
 367  */
 368 void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
 369 {
 370         if (!kmsan_enabled || kmsan_in_runtime())
 371                 return;
 372         kmsan_enter_runtime();
 373         /* The users may want to poison/unpoison random memory. */
 374         kmsan_internal_poison_memory((void *)address, size, flags,
 375                                      KMSAN_POISON_NOCHECK);
 376         kmsan_leave_runtime();
 377 }
 378 EXPORT_SYMBOL(kmsan_poison_memory);
 379
 380 /*
 381  * Unlike kmsan_poison_memory(), this function can be used from within KMSAN
 382  * runtime, because it does not trigger allocations or call instrumented code.
 383  */
 384 void kmsan_unpoison_memory(const void *address, size_t size)
 385 {
 386         unsigned long ua_flags;
 387
 388         if (!kmsan_enabled)
 389                 return;
 390
 391         ua_flags = user_access_save();
 392         /* The users may want to poison/unpoison random memory. */
 393         kmsan_internal_unpoison_memory((void *)address, size,
 394                                        KMSAN_POISON_NOCHECK);
 395         user_access_restore(ua_flags);
 396 }
 397 EXPORT_SYMBOL(kmsan_unpoison_memory);
 398
 399 /*
 400  * Version of kmsan_unpoison_memory() called from IRQ entry functions.
 401  */
 402 void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
 403 {
 404         kmsan_unpoison_memory((void *)regs, sizeof(*regs));
 405 }
 406
 407 void kmsan_check_memory(const void *addr, size_t size)
 408 {
 409         if (!kmsan_enabled)
 410                 return;
 411         return kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
 412                                            REASON_ANY);
 413 }
 414 EXPORT_SYMBOL(kmsan_check_memory);