1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 1999 Linus Torvalds
6 * Copyright (C) 2002 Christoph Hellwig
9 #include <linux/mman.h>
10 #include <linux/pagemap.h>
11 #include <linux/syscalls.h>
12 #include <linux/mempolicy.h>
13 #include <linux/page-isolation.h>
14 #include <linux/page_idle.h>
15 #include <linux/userfaultfd_k.h>
16 #include <linux/hugetlb.h>
17 #include <linux/falloc.h>
18 #include <linux/fadvise.h>
19 #include <linux/sched.h>
20 #include <linux/ksm.h>
22 #include <linux/file.h>
23 #include <linux/blkdev.h>
24 #include <linux/backing-dev.h>
25 #include <linux/pagewalk.h>
26 #include <linux/swap.h>
27 #include <linux/swapops.h>
28 #include <linux/shmem_fs.h>
29 #include <linux/mmu_notifier.h>
35 struct madvise_walk_private {
36 struct mmu_gather *tlb;
41 * Any behaviour which results in changes to the vma->vm_flags needs to
42 * take mmap_sem for writing. Others, which simply traverse vmas, need
43 * to only take it for reading.
45 static int madvise_need_mmap_write(int behavior)
56 /* be safe, default to 1. list exceptions explicitly */
62 * We can potentially split a vm area into separate
63 * areas, each area with its own behavior.
65 static long madvise_behavior(struct vm_area_struct *vma,
66 struct vm_area_struct **prev,
67 unsigned long start, unsigned long end, int behavior)
69 struct mm_struct *mm = vma->vm_mm;
72 unsigned long new_flags = vma->vm_flags;
76 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
79 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
82 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
85 new_flags |= VM_DONTCOPY;
88 if (vma->vm_flags & VM_IO) {
92 new_flags &= ~VM_DONTCOPY;
95 /* MADV_WIPEONFORK is only supported on anonymous memory. */
96 if (vma->vm_file || vma->vm_flags & VM_SHARED) {
100 new_flags |= VM_WIPEONFORK;
102 case MADV_KEEPONFORK:
103 new_flags &= ~VM_WIPEONFORK;
106 new_flags |= VM_DONTDUMP;
109 if (!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) {
113 new_flags &= ~VM_DONTDUMP;
116 case MADV_UNMERGEABLE:
117 error = ksm_madvise(vma, start, end, behavior, &new_flags);
119 goto out_convert_errno;
122 case MADV_NOHUGEPAGE:
123 error = hugepage_madvise(vma, &new_flags, behavior);
125 goto out_convert_errno;
129 if (new_flags == vma->vm_flags) {
134 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
135 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
136 vma->vm_file, pgoff, vma_policy(vma),
137 vma->vm_userfaultfd_ctx);
145 if (start != vma->vm_start) {
146 if (unlikely(mm->map_count >= sysctl_max_map_count)) {
150 error = __split_vma(mm, vma, start, 1);
152 goto out_convert_errno;
155 if (end != vma->vm_end) {
156 if (unlikely(mm->map_count >= sysctl_max_map_count)) {
160 error = __split_vma(mm, vma, end, 0);
162 goto out_convert_errno;
167 * vm_flags is protected by the mmap_sem held in write mode.
169 vma->vm_flags = new_flags;
173 * madvise() returns EAGAIN if kernel resources, such as
174 * slab, are temporarily unavailable.
176 if (error == -ENOMEM)
183 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
184 unsigned long end, struct mm_walk *walk)
187 struct vm_area_struct *vma = walk->private;
190 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
193 for (index = start; index != end; index += PAGE_SIZE) {
199 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
200 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
201 pte_unmap_unlock(orig_pte, ptl);
203 if (pte_present(pte) || pte_none(pte))
205 entry = pte_to_swp_entry(pte);
206 if (unlikely(non_swap_entry(entry)))
209 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
218 static const struct mm_walk_ops swapin_walk_ops = {
219 .pmd_entry = swapin_walk_pmd_entry,
222 static void force_shm_swapin_readahead(struct vm_area_struct *vma,
223 unsigned long start, unsigned long end,
224 struct address_space *mapping)
230 for (; start < end; start += PAGE_SIZE) {
231 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
233 page = find_get_entry(mapping, index);
234 if (!xa_is_value(page)) {
239 swap = radix_to_swp_entry(page);
240 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
246 lru_add_drain(); /* Push any new pages onto the LRU now */
248 #endif /* CONFIG_SWAP */
251 * Schedule all required I/O operations. Do not wait for completion.
253 static long madvise_willneed(struct vm_area_struct *vma,
254 struct vm_area_struct **prev,
255 unsigned long start, unsigned long end)
257 struct file *file = vma->vm_file;
263 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
264 lru_add_drain(); /* Push any new pages onto the LRU now */
268 if (shmem_mapping(file->f_mapping)) {
269 force_shm_swapin_readahead(vma, start, end,
278 if (IS_DAX(file_inode(file))) {
279 /* no bad return value, but ignore advice */
284 * Filesystem's fadvise may need to take various locks. We need to
285 * explicitly grab a reference because the vma (and hence the
286 * vma's reference to the file) can go away as soon as we drop
289 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
291 offset = (loff_t)(start - vma->vm_start)
292 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
293 up_read(¤t->mm->mmap_sem);
294 vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
296 down_read(¤t->mm->mmap_sem);
300 static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
301 unsigned long addr, unsigned long end,
302 struct mm_walk *walk)
304 struct madvise_walk_private *private = walk->private;
305 struct mmu_gather *tlb = private->tlb;
306 bool pageout = private->pageout;
307 struct mm_struct *mm = tlb->mm;
308 struct vm_area_struct *vma = walk->vma;
309 pte_t *orig_pte, *pte, ptent;
311 struct page *page = NULL;
312 LIST_HEAD(page_list);
314 if (fatal_signal_pending(current))
317 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
318 if (pmd_trans_huge(*pmd)) {
320 unsigned long next = pmd_addr_end(addr, end);
322 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
323 ptl = pmd_trans_huge_lock(pmd, vma);
328 if (is_huge_zero_pmd(orig_pmd))
331 if (unlikely(!pmd_present(orig_pmd))) {
332 VM_BUG_ON(thp_migration_supported() &&
333 !is_pmd_migration_entry(orig_pmd));
337 page = pmd_page(orig_pmd);
339 /* Do not interfere with other mappings of this page */
340 if (page_mapcount(page) != 1)
343 if (next - addr != HPAGE_PMD_SIZE) {
349 err = split_huge_page(page);
357 if (pmd_young(orig_pmd)) {
358 pmdp_invalidate(vma, addr, pmd);
359 orig_pmd = pmd_mkold(orig_pmd);
361 set_pmd_at(mm, addr, pmd, orig_pmd);
362 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
365 ClearPageReferenced(page);
366 test_and_clear_page_young(page);
368 if (!isolate_lru_page(page)) {
369 if (PageUnevictable(page))
370 putback_lru_page(page);
372 list_add(&page->lru, &page_list);
375 deactivate_page(page);
379 reclaim_pages(&page_list);
384 if (pmd_trans_unstable(pmd))
387 tlb_change_page_size(tlb, PAGE_SIZE);
388 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
389 flush_tlb_batched_pending(mm);
390 arch_enter_lazy_mmu_mode();
391 for (; addr < end; pte++, addr += PAGE_SIZE) {
397 if (!pte_present(ptent))
400 page = vm_normal_page(vma, addr, ptent);
405 * Creating a THP page is expensive so split it only if we
406 * are sure it's worth. Split it if we are only owner.
408 if (PageTransCompound(page)) {
409 if (page_mapcount(page) != 1)
412 if (!trylock_page(page)) {
416 pte_unmap_unlock(orig_pte, ptl);
417 if (split_huge_page(page)) {
420 pte_offset_map_lock(mm, pmd, addr, &ptl);
425 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
432 * Do not interfere with other mappings of this page and
435 if (!PageLRU(page) || page_mapcount(page) != 1)
438 VM_BUG_ON_PAGE(PageTransCompound(page), page);
440 if (pte_young(ptent)) {
441 ptent = ptep_get_and_clear_full(mm, addr, pte,
443 ptent = pte_mkold(ptent);
444 set_pte_at(mm, addr, pte, ptent);
445 tlb_remove_tlb_entry(tlb, pte, addr);
449 * We are deactivating a page for accelerating reclaiming.
450 * VM couldn't reclaim the page unless we clear PG_young.
451 * As a side effect, it makes confuse idle-page tracking
452 * because they will miss recent referenced history.
454 ClearPageReferenced(page);
455 test_and_clear_page_young(page);
457 if (!isolate_lru_page(page)) {
458 if (PageUnevictable(page))
459 putback_lru_page(page);
461 list_add(&page->lru, &page_list);
464 deactivate_page(page);
467 arch_leave_lazy_mmu_mode();
468 pte_unmap_unlock(orig_pte, ptl);
470 reclaim_pages(&page_list);
476 static const struct mm_walk_ops cold_walk_ops = {
477 .pmd_entry = madvise_cold_or_pageout_pte_range,
480 static void madvise_cold_page_range(struct mmu_gather *tlb,
481 struct vm_area_struct *vma,
482 unsigned long addr, unsigned long end)
484 struct madvise_walk_private walk_private = {
489 tlb_start_vma(tlb, vma);
490 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
491 tlb_end_vma(tlb, vma);
494 static long madvise_cold(struct vm_area_struct *vma,
495 struct vm_area_struct **prev,
496 unsigned long start_addr, unsigned long end_addr)
498 struct mm_struct *mm = vma->vm_mm;
499 struct mmu_gather tlb;
502 if (!can_madv_lru_vma(vma))
506 tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
507 madvise_cold_page_range(&tlb, vma, start_addr, end_addr);
508 tlb_finish_mmu(&tlb, start_addr, end_addr);
513 static void madvise_pageout_page_range(struct mmu_gather *tlb,
514 struct vm_area_struct *vma,
515 unsigned long addr, unsigned long end)
517 struct madvise_walk_private walk_private = {
522 tlb_start_vma(tlb, vma);
523 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
524 tlb_end_vma(tlb, vma);
527 static inline bool can_do_pageout(struct vm_area_struct *vma)
529 if (vma_is_anonymous(vma))
534 * paging out pagecache only for non-anonymous mappings that correspond
535 * to the files the calling process could (if tried) open for writing;
536 * otherwise we'd be including shared non-exclusive mappings, which
537 * opens a side channel.
539 return inode_owner_or_capable(file_inode(vma->vm_file)) ||
540 inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0;
543 static long madvise_pageout(struct vm_area_struct *vma,
544 struct vm_area_struct **prev,
545 unsigned long start_addr, unsigned long end_addr)
547 struct mm_struct *mm = vma->vm_mm;
548 struct mmu_gather tlb;
551 if (!can_madv_lru_vma(vma))
554 if (!can_do_pageout(vma))
558 tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
559 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr);
560 tlb_finish_mmu(&tlb, start_addr, end_addr);
565 static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
566 unsigned long end, struct mm_walk *walk)
569 struct mmu_gather *tlb = walk->private;
570 struct mm_struct *mm = tlb->mm;
571 struct vm_area_struct *vma = walk->vma;
573 pte_t *orig_pte, *pte, ptent;
578 next = pmd_addr_end(addr, end);
579 if (pmd_trans_huge(*pmd))
580 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
583 if (pmd_trans_unstable(pmd))
586 tlb_change_page_size(tlb, PAGE_SIZE);
587 orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
588 flush_tlb_batched_pending(mm);
589 arch_enter_lazy_mmu_mode();
590 for (; addr != end; pte++, addr += PAGE_SIZE) {
596 * If the pte has swp_entry, just clear page table to
597 * prevent swap-in which is more expensive rather than
598 * (page allocation + zeroing).
600 if (!pte_present(ptent)) {
603 entry = pte_to_swp_entry(ptent);
604 if (non_swap_entry(entry))
607 free_swap_and_cache(entry);
608 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
612 page = vm_normal_page(vma, addr, ptent);
617 * If pmd isn't transhuge but the page is THP and
618 * is owned by only this process, split it and
619 * deactivate all pages.
621 if (PageTransCompound(page)) {
622 if (page_mapcount(page) != 1)
625 if (!trylock_page(page)) {
629 pte_unmap_unlock(orig_pte, ptl);
630 if (split_huge_page(page)) {
633 pte_offset_map_lock(mm, pmd, addr, &ptl);
638 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
644 VM_BUG_ON_PAGE(PageTransCompound(page), page);
646 if (PageSwapCache(page) || PageDirty(page)) {
647 if (!trylock_page(page))
650 * If page is shared with others, we couldn't clear
651 * PG_dirty of the page.
653 if (page_mapcount(page) != 1) {
658 if (PageSwapCache(page) && !try_to_free_swap(page)) {
663 ClearPageDirty(page);
667 if (pte_young(ptent) || pte_dirty(ptent)) {
669 * Some of architecture(ex, PPC) don't update TLB
670 * with set_pte_at and tlb_remove_tlb_entry so for
671 * the portability, remap the pte with old|clean
672 * after pte clearing.
674 ptent = ptep_get_and_clear_full(mm, addr, pte,
677 ptent = pte_mkold(ptent);
678 ptent = pte_mkclean(ptent);
679 set_pte_at(mm, addr, pte, ptent);
680 tlb_remove_tlb_entry(tlb, pte, addr);
682 mark_page_lazyfree(page);
686 if (current->mm == mm)
689 add_mm_counter(mm, MM_SWAPENTS, nr_swap);
691 arch_leave_lazy_mmu_mode();
692 pte_unmap_unlock(orig_pte, ptl);
698 static const struct mm_walk_ops madvise_free_walk_ops = {
699 .pmd_entry = madvise_free_pte_range,
702 static int madvise_free_single_vma(struct vm_area_struct *vma,
703 unsigned long start_addr, unsigned long end_addr)
705 struct mm_struct *mm = vma->vm_mm;
706 struct mmu_notifier_range range;
707 struct mmu_gather tlb;
709 /* MADV_FREE works for only anon vma at the moment */
710 if (!vma_is_anonymous(vma))
713 range.start = max(vma->vm_start, start_addr);
714 if (range.start >= vma->vm_end)
716 range.end = min(vma->vm_end, end_addr);
717 if (range.end <= vma->vm_start)
719 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
720 range.start, range.end);
723 tlb_gather_mmu(&tlb, mm, range.start, range.end);
724 update_hiwater_rss(mm);
726 mmu_notifier_invalidate_range_start(&range);
727 tlb_start_vma(&tlb, vma);
728 walk_page_range(vma->vm_mm, range.start, range.end,
729 &madvise_free_walk_ops, &tlb);
730 tlb_end_vma(&tlb, vma);
731 mmu_notifier_invalidate_range_end(&range);
732 tlb_finish_mmu(&tlb, range.start, range.end);
738 * Application no longer needs these pages. If the pages are dirty,
739 * it's OK to just throw them away. The app will be more careful about
740 * data it wants to keep. Be sure to free swap resources too. The
741 * zap_page_range call sets things up for shrink_active_list to actually free
742 * these pages later if no one else has touched them in the meantime,
743 * although we could add these pages to a global reuse list for
744 * shrink_active_list to pick up before reclaiming other pages.
746 * NB: This interface discards data rather than pushes it out to swap,
747 * as some implementations do. This has performance implications for
748 * applications like large transactional databases which want to discard
749 * pages in anonymous maps after committing to backing store the data
750 * that was kept in them. There is no reason to write this data out to
751 * the swap area if the application is discarding it.
753 * An interface that causes the system to free clean pages and flush
754 * dirty pages is already available as msync(MS_INVALIDATE).
756 static long madvise_dontneed_single_vma(struct vm_area_struct *vma,
757 unsigned long start, unsigned long end)
759 zap_page_range(vma, start, end - start);
763 static long madvise_dontneed_free(struct vm_area_struct *vma,
764 struct vm_area_struct **prev,
765 unsigned long start, unsigned long end,
769 if (!can_madv_lru_vma(vma))
772 if (!userfaultfd_remove(vma, start, end)) {
773 *prev = NULL; /* mmap_sem has been dropped, prev is stale */
775 down_read(¤t->mm->mmap_sem);
776 vma = find_vma(current->mm, start);
779 if (start < vma->vm_start) {
781 * This "vma" under revalidation is the one
782 * with the lowest vma->vm_start where start
783 * is also < vma->vm_end. If start <
784 * vma->vm_start it means an hole materialized
785 * in the user address space within the
786 * virtual range passed to MADV_DONTNEED
791 if (!can_madv_lru_vma(vma))
793 if (end > vma->vm_end) {
795 * Don't fail if end > vma->vm_end. If the old
796 * vma was splitted while the mmap_sem was
797 * released the effect of the concurrent
798 * operation may not cause madvise() to
799 * have an undefined result. There may be an
800 * adjacent next vma that we'll walk
801 * next. userfaultfd_remove() will generate an
802 * UFFD_EVENT_REMOVE repetition on the
803 * end-vma->vm_end range, but the manager can
804 * handle a repetition fine.
808 VM_WARN_ON(start >= end);
811 if (behavior == MADV_DONTNEED)
812 return madvise_dontneed_single_vma(vma, start, end);
813 else if (behavior == MADV_FREE)
814 return madvise_free_single_vma(vma, start, end);
820 * Application wants to free up the pages and associated backing store.
821 * This is effectively punching a hole into the middle of a file.
823 static long madvise_remove(struct vm_area_struct *vma,
824 struct vm_area_struct **prev,
825 unsigned long start, unsigned long end)
831 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
833 if (vma->vm_flags & VM_LOCKED)
838 if (!f || !f->f_mapping || !f->f_mapping->host) {
842 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
845 offset = (loff_t)(start - vma->vm_start)
846 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
849 * Filesystem's fallocate may need to take i_mutex. We need to
850 * explicitly grab a reference because the vma (and hence the
851 * vma's reference to the file) can go away as soon as we drop
855 if (userfaultfd_remove(vma, start, end)) {
856 /* mmap_sem was not released by userfaultfd_remove() */
857 up_read(¤t->mm->mmap_sem);
859 error = vfs_fallocate(f,
860 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
861 offset, end - start);
863 down_read(¤t->mm->mmap_sem);
867 #ifdef CONFIG_MEMORY_FAILURE
869 * Error injection support for memory error handling.
871 static int madvise_inject_error(int behavior,
872 unsigned long start, unsigned long end)
878 if (!capable(CAP_SYS_ADMIN))
882 for (; start < end; start += PAGE_SIZE << order) {
886 ret = get_user_pages_fast(start, 1, 0, &page);
889 pfn = page_to_pfn(page);
892 * When soft offlining hugepages, after migrating the page
893 * we dissolve it, therefore in the second loop "page" will
894 * no longer be a compound page, and order will be 0.
896 order = compound_order(compound_head(page));
898 if (PageHWPoison(page)) {
903 if (behavior == MADV_SOFT_OFFLINE) {
904 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
907 ret = soft_offline_page(page, MF_COUNT_INCREASED);
913 pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n",
917 * Drop the page reference taken by get_user_pages_fast(). In
918 * the absence of MF_COUNT_INCREASED the memory_failure()
919 * routine is responsible for pinning the page to prevent it
920 * from being released back to the page allocator.
923 ret = memory_failure(pfn, 0);
928 /* Ensure that all poisoned pages are removed from per-cpu lists */
929 for_each_populated_zone(zone)
930 drain_all_pages(zone);
937 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
938 unsigned long start, unsigned long end, int behavior)
942 return madvise_remove(vma, prev, start, end);
944 return madvise_willneed(vma, prev, start, end);
946 return madvise_cold(vma, prev, start, end);
948 return madvise_pageout(vma, prev, start, end);
951 return madvise_dontneed_free(vma, prev, start, end, behavior);
953 return madvise_behavior(vma, prev, start, end, behavior);
958 madvise_behavior_valid(int behavior)
964 case MADV_SEQUENTIAL:
974 case MADV_UNMERGEABLE:
976 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
978 case MADV_NOHUGEPAGE:
982 case MADV_WIPEONFORK:
983 case MADV_KEEPONFORK:
984 #ifdef CONFIG_MEMORY_FAILURE
985 case MADV_SOFT_OFFLINE:
996 * The madvise(2) system call.
998 * Applications can use madvise() to advise the kernel how it should
999 * handle paging I/O in this VM area. The idea is to help the kernel
1000 * use appropriate read-ahead and caching techniques. The information
1001 * provided is advisory only, and can be safely disregarded by the
1002 * kernel without affecting the correct operation of the application.
1005 * MADV_NORMAL - the default behavior is to read clusters. This
1006 * results in some read-ahead and read-behind.
1007 * MADV_RANDOM - the system should read the minimum amount of data
1008 * on any access, since it is unlikely that the appli-
1009 * cation will need more than what it asks for.
1010 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
1011 * once, so they can be aggressively read ahead, and
1012 * can be freed soon after they are accessed.
1013 * MADV_WILLNEED - the application is notifying the system to read
1015 * MADV_DONTNEED - the application is finished with the given range,
1016 * so the kernel can free resources associated with it.
1017 * MADV_FREE - the application marks pages in the given range as lazy free,
1018 * where actual purges are postponed until memory pressure happens.
1019 * MADV_REMOVE - the application wants to free up the given range of
1020 * pages and associated backing store.
1021 * MADV_DONTFORK - omit this area from child's address space when forking:
1022 * typically, to avoid COWing pages pinned by get_user_pages().
1023 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
1024 * MADV_WIPEONFORK - present the child process with zero-filled memory in this
1025 * range after a fork.
1026 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
1027 * MADV_HWPOISON - trigger memory error handler as if the given memory range
1028 * were corrupted by unrecoverable hardware memory failure.
1029 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
1030 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
1031 * this area with pages of identical content from other such areas.
1032 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1033 * MADV_HUGEPAGE - the application wants to back the given range by transparent
1034 * huge pages in the future. Existing pages might be coalesced and
1035 * new pages might be allocated as THP.
1036 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
1037 * transparent huge pages so the existing pages will not be
1038 * coalesced into THP and new pages will not be allocated as THP.
1039 * MADV_DONTDUMP - the application wants to prevent pages in the given range
1040 * from being included in its core dump.
1041 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
1045 * -EINVAL - start + len < 0, start is not page-aligned,
1046 * "behavior" is not a valid value, or application
1047 * is attempting to release locked or shared pages,
1048 * or the specified address range includes file, Huge TLB,
1049 * MAP_SHARED or VMPFNMAP range.
1050 * -ENOMEM - addresses in the specified range are not currently
1051 * mapped, or are outside the AS of the process.
1052 * -EIO - an I/O error occurred while paging in data.
1053 * -EBADF - map exists, but area maps something that isn't a file.
1054 * -EAGAIN - a kernel resource was temporarily unavailable.
1056 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
1058 unsigned long end, tmp;
1059 struct vm_area_struct *vma, *prev;
1060 int unmapped_error = 0;
1061 int error = -EINVAL;
1064 struct blk_plug plug;
1066 start = untagged_addr(start);
1068 if (!madvise_behavior_valid(behavior))
1071 if (start & ~PAGE_MASK)
1073 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
1075 /* Check to see whether len was rounded up from small -ve to zero */
1087 #ifdef CONFIG_MEMORY_FAILURE
1088 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
1089 return madvise_inject_error(behavior, start, start + len_in);
1092 write = madvise_need_mmap_write(behavior);
1094 if (down_write_killable(¤t->mm->mmap_sem))
1097 down_read(¤t->mm->mmap_sem);
1101 * If the interval [start,end) covers some unmapped address
1102 * ranges, just ignore them, but return -ENOMEM at the end.
1103 * - different from the way of handling in mlock etc.
1105 vma = find_vma_prev(current->mm, start, &prev);
1106 if (vma && start > vma->vm_start)
1109 blk_start_plug(&plug);
1111 /* Still start < end. */
1116 /* Here start < (end|vma->vm_end). */
1117 if (start < vma->vm_start) {
1118 unmapped_error = -ENOMEM;
1119 start = vma->vm_start;
1124 /* Here vma->vm_start <= start < (end|vma->vm_end) */
1129 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
1130 error = madvise_vma(vma, &prev, start, tmp, behavior);
1134 if (prev && start < prev->vm_end)
1135 start = prev->vm_end;
1136 error = unmapped_error;
1140 vma = prev->vm_next;
1141 else /* madvise_remove dropped mmap_sem */
1142 vma = find_vma(current->mm, start);
1145 blk_finish_plug(&plug);
1147 up_write(¤t->mm->mmap_sem);
1149 up_read(¤t->mm->mmap_sem);