mm/khugepaged.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   3
   4 #include <linux/mm.h>
   5 #include <linux/sched.h>
   6 #include <linux/sched/mm.h>
   7 #include <linux/sched/coredump.h>
   8 #include <linux/mmu_notifier.h>
   9 #include <linux/rmap.h>
  10 #include <linux/swap.h>
  11 #include <linux/mm_inline.h>
  12 #include <linux/kthread.h>
  13 #include <linux/khugepaged.h>
  14 #include <linux/freezer.h>
  15 #include <linux/mman.h>
  16 #include <linux/hashtable.h>
  17 #include <linux/userfaultfd_k.h>
  18 #include <linux/page_idle.h>
  19 #include <linux/swapops.h>
  20 #include <linux/shmem_fs.h>
  21
  22 #include <asm/tlb.h>
  23 #include <asm/pgalloc.h>
  24 #include "internal.h"
  25
  26 enum scan_result {
  27         SCAN_FAIL,
  28         SCAN_SUCCEED,
  29         SCAN_PMD_NULL,
  30         SCAN_EXCEED_NONE_PTE,
  31         SCAN_PTE_NON_PRESENT,
  32         SCAN_PAGE_RO,
  33         SCAN_LACK_REFERENCED_PAGE,
  34         SCAN_PAGE_NULL,
  35         SCAN_SCAN_ABORT,
  36         SCAN_PAGE_COUNT,
  37         SCAN_PAGE_LRU,
  38         SCAN_PAGE_LOCK,
  39         SCAN_PAGE_ANON,
  40         SCAN_PAGE_COMPOUND,
  41         SCAN_ANY_PROCESS,
  42         SCAN_VMA_NULL,
  43         SCAN_VMA_CHECK,
  44         SCAN_ADDRESS_RANGE,
  45         SCAN_SWAP_CACHE_PAGE,
  46         SCAN_DEL_PAGE_LRU,
  47         SCAN_ALLOC_HUGE_PAGE_FAIL,
  48         SCAN_CGROUP_CHARGE_FAIL,
  49         SCAN_EXCEED_SWAP_PTE,
  50         SCAN_TRUNCATED,
  51 };
  52
  53 #define CREATE_TRACE_POINTS
  54 #include <trace/events/huge_memory.h>
  55
  56 static struct task_struct *khugepaged_thread __read_mostly;
  57 static DEFINE_MUTEX(khugepaged_mutex);
  58
  59 /* default scan 8*512 pte (or vmas) every 30 second */
  60 static unsigned int khugepaged_pages_to_scan __read_mostly;
  61 static unsigned int khugepaged_pages_collapsed;
  62 static unsigned int khugepaged_full_scans;
  63 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
  64 /* during fragmentation poll the hugepage allocator once every minute */
  65 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
  66 static unsigned long khugepaged_sleep_expire;
  67 static DEFINE_SPINLOCK(khugepaged_mm_lock);
  68 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  69 /*
  70  * default collapse hugepages if there is at least one pte mapped like
  71  * it would have happened if the vma was large enough during page
  72  * fault.
  73  */
  74 static unsigned int khugepaged_max_ptes_none __read_mostly;
  75 static unsigned int khugepaged_max_ptes_swap __read_mostly;
  76
  77 #define MM_SLOTS_HASH_BITS 10
  78 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
  79
  80 static struct kmem_cache *mm_slot_cache __read_mostly;
  81
  82 /**
  83  * struct mm_slot - hash lookup from mm to mm_slot
  84  * @hash: hash collision list
  85  * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
  86  * @mm: the mm that this information is valid for
  87  */
  88 struct mm_slot {
  89         struct hlist_node hash;
  90         struct list_head mm_node;
  91         struct mm_struct *mm;
  92 };
  93
  94 /**
  95  * struct khugepaged_scan - cursor for scanning
  96  * @mm_head: the head of the mm list to scan
  97  * @mm_slot: the current mm_slot we are scanning
  98  * @address: the next address inside that to be scanned
  99  *
 100  * There is only the one khugepaged_scan instance of this cursor structure.
 101  */
 102 struct khugepaged_scan {
 103         struct list_head mm_head;
 104         struct mm_slot *mm_slot;
 105         unsigned long address;
 106 };
 107
 108 static struct khugepaged_scan khugepaged_scan = {
 109         .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
 110 };
 111
 112 #ifdef CONFIG_SYSFS
 113 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
 114                                          struct kobj_attribute *attr,
 115                                          char *buf)
 116 {
 117         return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
 118 }
 119
 120 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
 121                                           struct kobj_attribute *attr,
 122                                           const char *buf, size_t count)
 123 {
 124         unsigned long msecs;
 125         int err;
 126
 127         err = kstrtoul(buf, 10, &msecs);
 128         if (err || msecs > UINT_MAX)
 129                 return -EINVAL;
 130
 131         khugepaged_scan_sleep_millisecs = msecs;
 132         khugepaged_sleep_expire = 0;
 133         wake_up_interruptible(&khugepaged_wait);
 134
 135         return count;
 136 }
 137 static struct kobj_attribute scan_sleep_millisecs_attr =
 138         __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
 139                scan_sleep_millisecs_store);
 140
 141 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
 142                                           struct kobj_attribute *attr,
 143                                           char *buf)
 144 {
 145         return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
 146 }
 147
 148 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
 149                                            struct kobj_attribute *attr,
 150                                            const char *buf, size_t count)
 151 {
 152         unsigned long msecs;
 153         int err;
 154
 155         err = kstrtoul(buf, 10, &msecs);
 156         if (err || msecs > UINT_MAX)
 157                 return -EINVAL;
 158
 159         khugepaged_alloc_sleep_millisecs = msecs;
 160         khugepaged_sleep_expire = 0;
 161         wake_up_interruptible(&khugepaged_wait);
 162
 163         return count;
 164 }
 165 static struct kobj_attribute alloc_sleep_millisecs_attr =
 166         __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
 167                alloc_sleep_millisecs_store);
 168
 169 static ssize_t pages_to_scan_show(struct kobject *kobj,
 170                                   struct kobj_attribute *attr,
 171                                   char *buf)
 172 {
 173         return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
 174 }
 175 static ssize_t pages_to_scan_store(struct kobject *kobj,
 176                                    struct kobj_attribute *attr,
 177                                    const char *buf, size_t count)
 178 {
 179         int err;
 180         unsigned long pages;
 181
 182         err = kstrtoul(buf, 10, &pages);
 183         if (err || !pages || pages > UINT_MAX)
 184                 return -EINVAL;
 185
 186         khugepaged_pages_to_scan = pages;
 187
 188         return count;
 189 }
 190 static struct kobj_attribute pages_to_scan_attr =
 191         __ATTR(pages_to_scan, 0644, pages_to_scan_show,
 192                pages_to_scan_store);
 193
 194 static ssize_t pages_collapsed_show(struct kobject *kobj,
 195                                     struct kobj_attribute *attr,
 196                                     char *buf)
 197 {
 198         return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
 199 }
 200 static struct kobj_attribute pages_collapsed_attr =
 201         __ATTR_RO(pages_collapsed);
 202
 203 static ssize_t full_scans_show(struct kobject *kobj,
 204                                struct kobj_attribute *attr,
 205                                char *buf)
 206 {
 207         return sprintf(buf, "%u\n", khugepaged_full_scans);
 208 }
 209 static struct kobj_attribute full_scans_attr =
 210         __ATTR_RO(full_scans);
 211
 212 static ssize_t khugepaged_defrag_show(struct kobject *kobj,
 213                                       struct kobj_attribute *attr, char *buf)
 214 {
 215         return single_hugepage_flag_show(kobj, attr, buf,
 216                                 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
 217 }
 218 static ssize_t khugepaged_defrag_store(struct kobject *kobj,
 219                                        struct kobj_attribute *attr,
 220                                        const char *buf, size_t count)
 221 {
 222         return single_hugepage_flag_store(kobj, attr, buf, count,
 223                                  TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
 224 }
 225 static struct kobj_attribute khugepaged_defrag_attr =
 226         __ATTR(defrag, 0644, khugepaged_defrag_show,
 227                khugepaged_defrag_store);
 228
 229 /*
 230  * max_ptes_none controls if khugepaged should collapse hugepages over
 231  * any unmapped ptes in turn potentially increasing the memory
 232  * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
 233  * reduce the available free memory in the system as it
 234  * runs. Increasing max_ptes_none will instead potentially reduce the
 235  * free memory in the system during the khugepaged scan.
 236  */
 237 static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
 238                                              struct kobj_attribute *attr,
 239                                              char *buf)
 240 {
 241         return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
 242 }
 243 static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
 244                                               struct kobj_attribute *attr,
 245                                               const char *buf, size_t count)
 246 {
 247         int err;
 248         unsigned long max_ptes_none;
 249
 250         err = kstrtoul(buf, 10, &max_ptes_none);
 251         if (err || max_ptes_none > HPAGE_PMD_NR-1)
 252                 return -EINVAL;
 253
 254         khugepaged_max_ptes_none = max_ptes_none;
 255
 256         return count;
 257 }
 258 static struct kobj_attribute khugepaged_max_ptes_none_attr =
 259         __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
 260                khugepaged_max_ptes_none_store);
 261
 262 static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj,
 263                                              struct kobj_attribute *attr,
 264                                              char *buf)
 265 {
 266         return sprintf(buf, "%u\n", khugepaged_max_ptes_swap);
 267 }
 268
 269 static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj,
 270                                               struct kobj_attribute *attr,
 271                                               const char *buf, size_t count)
 272 {
 273         int err;
 274         unsigned long max_ptes_swap;
 275
 276         err  = kstrtoul(buf, 10, &max_ptes_swap);
 277         if (err || max_ptes_swap > HPAGE_PMD_NR-1)
 278                 return -EINVAL;
 279
 280         khugepaged_max_ptes_swap = max_ptes_swap;
 281
 282         return count;
 283 }
 284
 285 static struct kobj_attribute khugepaged_max_ptes_swap_attr =
 286         __ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show,
 287                khugepaged_max_ptes_swap_store);
 288
 289 static struct attribute *khugepaged_attr[] = {
 290         &khugepaged_defrag_attr.attr,
 291         &khugepaged_max_ptes_none_attr.attr,
 292         &pages_to_scan_attr.attr,
 293         &pages_collapsed_attr.attr,
 294         &full_scans_attr.attr,
 295         &scan_sleep_millisecs_attr.attr,
 296         &alloc_sleep_millisecs_attr.attr,
 297         &khugepaged_max_ptes_swap_attr.attr,
 298         NULL,
 299 };
 300
 301 struct attribute_group khugepaged_attr_group = {
 302         .attrs = khugepaged_attr,
 303         .name = "khugepaged",
 304 };
 305 #endif /* CONFIG_SYSFS */
 306
 307 #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
 308
 309 int hugepage_madvise(struct vm_area_struct *vma,
 310                      unsigned long *vm_flags, int advice)
 311 {
 312         switch (advice) {
 313         case MADV_HUGEPAGE:
 314 #ifdef CONFIG_S390
 315                 /*
 316                  * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
 317                  * can't handle this properly after s390_enable_sie, so we simply
 318                  * ignore the madvise to prevent qemu from causing a SIGSEGV.
 319                  */
 320                 if (mm_has_pgste(vma->vm_mm))
 321                         return 0;
 322 #endif
 323                 *vm_flags &= ~VM_NOHUGEPAGE;
 324                 *vm_flags |= VM_HUGEPAGE;
 325                 /*
 326                  * If the vma become good for khugepaged to scan,
 327                  * register it here without waiting a page fault that
 328                  * may not happen any time soon.
 329                  */
 330                 if (!(*vm_flags & VM_NO_KHUGEPAGED) &&
 331                                 khugepaged_enter_vma_merge(vma, *vm_flags))
 332                         return -ENOMEM;
 333                 break;
 334         case MADV_NOHUGEPAGE:
 335                 *vm_flags &= ~VM_HUGEPAGE;
 336                 *vm_flags |= VM_NOHUGEPAGE;
 337                 /*
 338                  * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
 339                  * this vma even if we leave the mm registered in khugepaged if
 340                  * it got registered before VM_NOHUGEPAGE was set.
 341                  */
 342                 break;
 343         }
 344
 345         return 0;
 346 }
 347
 348 int __init khugepaged_init(void)
 349 {
 350         mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
 351                                           sizeof(struct mm_slot),
 352                                           __alignof__(struct mm_slot), 0, NULL);
 353         if (!mm_slot_cache)
 354                 return -ENOMEM;
 355
 356         khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
 357         khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
 358         khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
 359
 360         return 0;
 361 }
 362
 363 void __init khugepaged_destroy(void)
 364 {
 365         kmem_cache_destroy(mm_slot_cache);
 366 }
 367
 368 static inline struct mm_slot *alloc_mm_slot(void)
 369 {
 370         if (!mm_slot_cache)     /* initialization failed */
 371                 return NULL;
 372         return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
 373 }
 374
 375 static inline void free_mm_slot(struct mm_slot *mm_slot)
 376 {
 377         kmem_cache_free(mm_slot_cache, mm_slot);
 378 }
 379
 380 static struct mm_slot *get_mm_slot(struct mm_struct *mm)
 381 {
 382         struct mm_slot *mm_slot;
 383
 384         hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
 385                 if (mm == mm_slot->mm)
 386                         return mm_slot;
 387
 388         return NULL;
 389 }
 390
 391 static void insert_to_mm_slots_hash(struct mm_struct *mm,
 392                                     struct mm_slot *mm_slot)
 393 {
 394         mm_slot->mm = mm;
 395         hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
 396 }
 397
 398 static inline int khugepaged_test_exit(struct mm_struct *mm)
 399 {
 400         return atomic_read(&mm->mm_users) == 0 || !mmget_still_valid(mm);
 401 }
 402
 403 int __khugepaged_enter(struct mm_struct *mm)
 404 {
 405         struct mm_slot *mm_slot;
 406         int wakeup;
 407
 408         mm_slot = alloc_mm_slot();
 409         if (!mm_slot)
 410                 return -ENOMEM;
 411
 412         /* __khugepaged_exit() must not run from under us */
 413         VM_BUG_ON_MM(atomic_read(&mm->mm_users) == 0, mm);
 414         if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
 415                 free_mm_slot(mm_slot);
 416                 return 0;
 417         }
 418
 419         spin_lock(&khugepaged_mm_lock);
 420         insert_to_mm_slots_hash(mm, mm_slot);
 421         /*
 422          * Insert just behind the scanning cursor, to let the area settle
 423          * down a little.
 424          */
 425         wakeup = list_empty(&khugepaged_scan.mm_head);
 426         list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
 427         spin_unlock(&khugepaged_mm_lock);
 428
 429         mmgrab(mm);
 430         if (wakeup)
 431                 wake_up_interruptible(&khugepaged_wait);
 432
 433         return 0;
 434 }
 435
 436 int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
 437                                unsigned long vm_flags)
 438 {
 439         unsigned long hstart, hend;
 440         if (!vma->anon_vma)
 441                 /*
 442                  * Not yet faulted in so we will register later in the
 443                  * page fault if needed.
 444                  */
 445                 return 0;
 446         if (vma->vm_ops || (vm_flags & VM_NO_KHUGEPAGED))
 447                 /* khugepaged not yet working on file or special mappings */
 448                 return 0;
 449         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
 450         hend = vma->vm_end & HPAGE_PMD_MASK;
 451         if (hstart < hend)
 452                 return khugepaged_enter(vma, vm_flags);
 453         return 0;
 454 }
 455
 456 void __khugepaged_exit(struct mm_struct *mm)
 457 {
 458         struct mm_slot *mm_slot;
 459         int free = 0;
 460
 461         spin_lock(&khugepaged_mm_lock);
 462         mm_slot = get_mm_slot(mm);
 463         if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
 464                 hash_del(&mm_slot->hash);
 465                 list_del(&mm_slot->mm_node);
 466                 free = 1;
 467         }
 468         spin_unlock(&khugepaged_mm_lock);
 469
 470         if (free) {
 471                 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
 472                 free_mm_slot(mm_slot);
 473                 mmdrop(mm);
 474         } else if (mm_slot) {
 475                 /*
 476                  * This is required to serialize against
 477                  * khugepaged_test_exit() (which is guaranteed to run
 478                  * under mmap sem read mode). Stop here (after we
 479                  * return all pagetables will be destroyed) until
 480                  * khugepaged has finished working on the pagetables
 481                  * under the mmap_sem.
 482                  */
 483                 down_write(&mm->mmap_sem);
 484                 up_write(&mm->mmap_sem);
 485         }
 486 }
 487
 488 static void release_pte_page(struct page *page)
 489 {
 490         dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page));
 491         unlock_page(page);
 492         putback_lru_page(page);
 493 }
 494
 495 static void release_pte_pages(pte_t *pte, pte_t *_pte)
 496 {
 497         while (--_pte >= pte) {
 498                 pte_t pteval = *_pte;
 499                 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
 500                         release_pte_page(pte_page(pteval));
 501         }
 502 }
 503
 504 static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
 505                                         unsigned long address,
 506                                         pte_t *pte)
 507 {
 508         struct page *page = NULL;
 509         pte_t *_pte;
 510         int none_or_zero = 0, result = 0, referenced = 0;
 511         bool writable = false;
 512
 513         for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
 514              _pte++, address += PAGE_SIZE) {
 515                 pte_t pteval = *_pte;
 516                 if (pte_none(pteval) || (pte_present(pteval) &&
 517                                 is_zero_pfn(pte_pfn(pteval)))) {
 518                         if (!userfaultfd_armed(vma) &&
 519                             ++none_or_zero <= khugepaged_max_ptes_none) {
 520                                 continue;
 521                         } else {
 522                                 result = SCAN_EXCEED_NONE_PTE;
 523                                 goto out;
 524                         }
 525                 }
 526                 if (!pte_present(pteval)) {
 527                         result = SCAN_PTE_NON_PRESENT;
 528                         goto out;
 529                 }
 530                 page = vm_normal_page(vma, address, pteval);
 531                 if (unlikely(!page)) {
 532                         result = SCAN_PAGE_NULL;
 533                         goto out;
 534                 }
 535
 536                 /* TODO: teach khugepaged to collapse THP mapped with pte */
 537                 if (PageCompound(page)) {
 538                         result = SCAN_PAGE_COMPOUND;
 539                         goto out;
 540                 }
 541
 542                 VM_BUG_ON_PAGE(!PageAnon(page), page);
 543
 544                 /*
 545                  * We can do it before isolate_lru_page because the
 546                  * page can't be freed from under us. NOTE: PG_lock
 547                  * is needed to serialize against split_huge_page
 548                  * when invoked from the VM.
 549                  */
 550                 if (!trylock_page(page)) {
 551                         result = SCAN_PAGE_LOCK;
 552                         goto out;
 553                 }
 554
 555                 /*
 556                  * cannot use mapcount: can't collapse if there's a gup pin.
 557                  * The page must only be referenced by the scanned process
 558                  * and page swap cache.
 559                  */
 560                 if (page_count(page) != 1 + PageSwapCache(page)) {
 561                         unlock_page(page);
 562                         result = SCAN_PAGE_COUNT;
 563                         goto out;
 564                 }
 565                 if (pte_write(pteval)) {
 566                         writable = true;
 567                 } else {
 568                         if (PageSwapCache(page) &&
 569                             !reuse_swap_page(page, NULL)) {
 570                                 unlock_page(page);
 571                                 result = SCAN_SWAP_CACHE_PAGE;
 572                                 goto out;
 573                         }
 574                         /*
 575                          * Page is not in the swap cache. It can be collapsed
 576                          * into a THP.
 577                          */
 578                 }
 579
 580                 /*
 581                  * Isolate the page to avoid collapsing an hugepage
 582                  * currently in use by the VM.
 583                  */
 584                 if (isolate_lru_page(page)) {
 585                         unlock_page(page);
 586                         result = SCAN_DEL_PAGE_LRU;
 587                         goto out;
 588                 }
 589                 inc_node_page_state(page,
 590                                 NR_ISOLATED_ANON + page_is_file_cache(page));
 591                 VM_BUG_ON_PAGE(!PageLocked(page), page);
 592                 VM_BUG_ON_PAGE(PageLRU(page), page);
 593
 594                 /* There should be enough young pte to collapse the page */
 595                 if (pte_young(pteval) ||
 596                     page_is_young(page) || PageReferenced(page) ||
 597                     mmu_notifier_test_young(vma->vm_mm, address))
 598                         referenced++;
 599         }
 600
 601         if (unlikely(!writable)) {
 602                 result = SCAN_PAGE_RO;
 603         } else if (unlikely(!referenced)) {
 604                 result = SCAN_LACK_REFERENCED_PAGE;
 605         } else {
 606                 result = SCAN_SUCCEED;
 607                 trace_mm_collapse_huge_page_isolate(page, none_or_zero,
 608                                                     referenced, writable, result);
 609                 return 1;
 610         }
 611 out:
 612         release_pte_pages(pte, _pte);
 613         trace_mm_collapse_huge_page_isolate(page, none_or_zero,
 614                                             referenced, writable, result);
 615         return 0;
 616 }
 617
 618 static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
 619                                       struct vm_area_struct *vma,
 620                                       unsigned long address,
 621                                       spinlock_t *ptl)
 622 {
 623         pte_t *_pte;
 624         for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
 625                                 _pte++, page++, address += PAGE_SIZE) {
 626                 pte_t pteval = *_pte;
 627                 struct page *src_page;
 628
 629                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
 630                         clear_user_highpage(page, address);
 631                         add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
 632                         if (is_zero_pfn(pte_pfn(pteval))) {
 633                                 /*
 634                                  * ptl mostly unnecessary.
 635                                  */
 636                                 spin_lock(ptl);
 637                                 /*
 638                                  * paravirt calls inside pte_clear here are
 639                                  * superfluous.
 640                                  */
 641                                 pte_clear(vma->vm_mm, address, _pte);
 642                                 spin_unlock(ptl);
 643                         }
 644                 } else {
 645                         src_page = pte_page(pteval);
 646                         copy_user_highpage(page, src_page, address, vma);
 647                         VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
 648                         release_pte_page(src_page);
 649                         /*
 650                          * ptl mostly unnecessary, but preempt has to
 651                          * be disabled to update the per-cpu stats
 652                          * inside page_remove_rmap().
 653                          */
 654                         spin_lock(ptl);
 655                         /*
 656                          * paravirt calls inside pte_clear here are
 657                          * superfluous.
 658                          */
 659                         pte_clear(vma->vm_mm, address, _pte);
 660                         page_remove_rmap(src_page, false);
 661                         spin_unlock(ptl);
 662                         free_page_and_swap_cache(src_page);
 663                 }
 664         }
 665 }
 666
 667 static void khugepaged_alloc_sleep(void)
 668 {
 669         DEFINE_WAIT(wait);
 670
 671         add_wait_queue(&khugepaged_wait, &wait);
 672         freezable_schedule_timeout_interruptible(
 673                 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
 674         remove_wait_queue(&khugepaged_wait, &wait);
 675 }
 676
 677 static int khugepaged_node_load[MAX_NUMNODES];
 678
 679 static bool khugepaged_scan_abort(int nid)
 680 {
 681         int i;
 682
 683         /*
 684          * If node_reclaim_mode is disabled, then no extra effort is made to
 685          * allocate memory locally.
 686          */
 687         if (!node_reclaim_mode)
 688                 return false;
 689
 690         /* If there is a count for this node already, it must be acceptable */
 691         if (khugepaged_node_load[nid])
 692                 return false;
 693
 694         for (i = 0; i < MAX_NUMNODES; i++) {
 695                 if (!khugepaged_node_load[i])
 696                         continue;
 697                 if (node_distance(nid, i) > RECLAIM_DISTANCE)
 698                         return true;
 699         }
 700         return false;
 701 }
 702
 703 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
 704 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
 705 {
 706         return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;
 707 }
 708
 709 #ifdef CONFIG_NUMA
 710 static int khugepaged_find_target_node(void)
 711 {
 712         static int last_khugepaged_target_node = NUMA_NO_NODE;
 713         int nid, target_node = 0, max_value = 0;
 714
 715         /* find first node with max normal pages hit */
 716         for (nid = 0; nid < MAX_NUMNODES; nid++)
 717                 if (khugepaged_node_load[nid] > max_value) {
 718                         max_value = khugepaged_node_load[nid];
 719                         target_node = nid;
 720                 }
 721
 722         /* do some balance if several nodes have the same hit record */
 723         if (target_node <= last_khugepaged_target_node)
 724                 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
 725                                 nid++)
 726                         if (max_value == khugepaged_node_load[nid]) {
 727                                 target_node = nid;
 728                                 break;
 729                         }
 730
 731         last_khugepaged_target_node = target_node;
 732         return target_node;
 733 }
 734
 735 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
 736 {
 737         if (IS_ERR(*hpage)) {
 738                 if (!*wait)
 739                         return false;
 740
 741                 *wait = false;
 742                 *hpage = NULL;
 743                 khugepaged_alloc_sleep();
 744         } else if (*hpage) {
 745                 put_page(*hpage);
 746                 *hpage = NULL;
 747         }
 748
 749         return true;
 750 }
 751
 752 static struct page *
 753 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
 754 {
 755         VM_BUG_ON_PAGE(*hpage, *hpage);
 756
 757         *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
 758         if (unlikely(!*hpage)) {
 759                 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
 760                 *hpage = ERR_PTR(-ENOMEM);
 761                 return NULL;
 762         }
 763
 764         prep_transhuge_page(*hpage);
 765         count_vm_event(THP_COLLAPSE_ALLOC);
 766         return *hpage;
 767 }
 768 #else
 769 static int khugepaged_find_target_node(void)
 770 {
 771         return 0;
 772 }
 773
 774 static inline struct page *alloc_khugepaged_hugepage(void)
 775 {
 776         struct page *page;
 777
 778         page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
 779                            HPAGE_PMD_ORDER);
 780         if (page)
 781                 prep_transhuge_page(page);
 782         return page;
 783 }
 784
 785 static struct page *khugepaged_alloc_hugepage(bool *wait)
 786 {
 787         struct page *hpage;
 788
 789         do {
 790                 hpage = alloc_khugepaged_hugepage();
 791                 if (!hpage) {
 792                         count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
 793                         if (!*wait)
 794                                 return NULL;
 795
 796                         *wait = false;
 797                         khugepaged_alloc_sleep();
 798                 } else
 799                         count_vm_event(THP_COLLAPSE_ALLOC);
 800         } while (unlikely(!hpage) && likely(khugepaged_enabled()));
 801
 802         return hpage;
 803 }
 804
 805 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
 806 {
 807         /*
 808          * If the hpage allocated earlier was briefly exposed in page cache
 809          * before collapse_file() failed, it is possible that racing lookups
 810          * have not yet completed, and would then be unpleasantly surprised by
 811          * finding the hpage reused for the same mapping at a different offset.
 812          * Just release the previous allocation if there is any danger of that.
 813          */
 814         if (*hpage && page_count(*hpage) > 1) {
 815                 put_page(*hpage);
 816                 *hpage = NULL;
 817         }
 818
 819         if (!*hpage)
 820                 *hpage = khugepaged_alloc_hugepage(wait);
 821
 822         if (unlikely(!*hpage))
 823                 return false;
 824
 825         return true;
 826 }
 827
 828 static struct page *
 829 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
 830 {
 831         VM_BUG_ON(!*hpage);
 832
 833         return  *hpage;
 834 }
 835 #endif
 836
 837 static bool hugepage_vma_check(struct vm_area_struct *vma)
 838 {
 839         if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
 840             (vma->vm_flags & VM_NOHUGEPAGE) ||
 841             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
 842                 return false;
 843         if (shmem_file(vma->vm_file)) {
 844                 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
 845                         return false;
 846                 return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
 847                                 HPAGE_PMD_NR);
 848         }
 849         if (!vma->anon_vma || vma->vm_ops)
 850                 return false;
 851         if (is_vma_temporary_stack(vma))
 852                 return false;
 853         return !(vma->vm_flags & VM_NO_KHUGEPAGED);
 854 }
 855
 856 /*
 857  * If mmap_sem temporarily dropped, revalidate vma
 858  * before taking mmap_sem.
 859  * Return 0 if succeeds, otherwise return none-zero
 860  * value (scan code).
 861  */
 862
 863 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
 864                 struct vm_area_struct **vmap)
 865 {
 866         struct vm_area_struct *vma;
 867         unsigned long hstart, hend;
 868
 869         if (unlikely(khugepaged_test_exit(mm)))
 870                 return SCAN_ANY_PROCESS;
 871
 872         *vmap = vma = find_vma(mm, address);
 873         if (!vma)
 874                 return SCAN_VMA_NULL;
 875
 876         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
 877         hend = vma->vm_end & HPAGE_PMD_MASK;
 878         if (address < hstart || address + HPAGE_PMD_SIZE > hend)
 879                 return SCAN_ADDRESS_RANGE;
 880         if (!hugepage_vma_check(vma))
 881                 return SCAN_VMA_CHECK;
 882         return 0;
 883 }
 884
 885 /*
 886  * Bring missing pages in from swap, to complete THP collapse.
 887  * Only done if khugepaged_scan_pmd believes it is worthwhile.
 888  *
 889  * Called and returns without pte mapped or spinlocks held,
 890  * but with mmap_sem held to protect against vma changes.
 891  */
 892
 893 static bool __collapse_huge_page_swapin(struct mm_struct *mm,
 894                                         struct vm_area_struct *vma,
 895                                         unsigned long address, pmd_t *pmd,
 896                                         int referenced)
 897 {
 898         int swapped_in = 0, ret = 0;
 899         struct vm_fault vmf = {
 900                 .vma = vma,
 901                 .address = address,
 902                 .flags = FAULT_FLAG_ALLOW_RETRY,
 903                 .pmd = pmd,
 904                 .pgoff = linear_page_index(vma, address),
 905         };
 906
 907         /* we only decide to swapin, if there is enough young ptes */
 908         if (referenced < HPAGE_PMD_NR/2) {
 909                 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
 910                 return false;
 911         }
 912         vmf.pte = pte_offset_map(pmd, address);
 913         for (; vmf.address < address + HPAGE_PMD_NR*PAGE_SIZE;
 914                         vmf.pte++, vmf.address += PAGE_SIZE) {
 915                 vmf.orig_pte = *vmf.pte;
 916                 if (!is_swap_pte(vmf.orig_pte))
 917                         continue;
 918                 swapped_in++;
 919                 ret = do_swap_page(&vmf);
 920
 921                 /* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
 922                 if (ret & VM_FAULT_RETRY) {
 923                         down_read(&mm->mmap_sem);
 924                         if (hugepage_vma_revalidate(mm, address, &vmf.vma)) {
 925                                 /* vma is no longer available, don't continue to swapin */
 926                                 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
 927                                 return false;
 928                         }
 929                         /* check if the pmd is still valid */
 930                         if (mm_find_pmd(mm, address) != pmd) {
 931                                 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
 932                                 return false;
 933                         }
 934                 }
 935                 if (ret & VM_FAULT_ERROR) {
 936                         trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
 937                         return false;
 938                 }
 939                 /* pte is unmapped now, we need to map it */
 940                 vmf.pte = pte_offset_map(pmd, vmf.address);
 941         }
 942         vmf.pte--;
 943         pte_unmap(vmf.pte);
 944         trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
 945         return true;
 946 }
 947
 948 static void collapse_huge_page(struct mm_struct *mm,
 949                                    unsigned long address,
 950                                    struct page **hpage,
 951                                    int node, int referenced)
 952 {
 953         pmd_t *pmd, _pmd;
 954         pte_t *pte;
 955         pgtable_t pgtable;
 956         struct page *new_page;
 957         spinlock_t *pmd_ptl, *pte_ptl;
 958         int isolated = 0, result = 0;
 959         struct mem_cgroup *memcg;
 960         struct vm_area_struct *vma;
 961         unsigned long mmun_start;       /* For mmu_notifiers */
 962         unsigned long mmun_end;         /* For mmu_notifiers */
 963         gfp_t gfp;
 964
 965         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 966
 967         /* Only allocate from the target node */
 968         gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
 969
 970         /*
 971          * Before allocating the hugepage, release the mmap_sem read lock.
 972          * The allocation can take potentially a long time if it involves
 973          * sync compaction, and we do not need to hold the mmap_sem during
 974          * that. We will recheck the vma after taking it again in write mode.
 975          */
 976         up_read(&mm->mmap_sem);
 977         new_page = khugepaged_alloc_page(hpage, gfp, node);
 978         if (!new_page) {
 979                 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
 980                 goto out_nolock;
 981         }
 982
 983         /* Do not oom kill for khugepaged charges */
 984         if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp | __GFP_NORETRY,
 985                                            &memcg, true))) {
 986                 result = SCAN_CGROUP_CHARGE_FAIL;
 987                 goto out_nolock;
 988         }
 989
 990         down_read(&mm->mmap_sem);
 991         result = hugepage_vma_revalidate(mm, address, &vma);
 992         if (result) {
 993                 mem_cgroup_cancel_charge(new_page, memcg, true);
 994                 up_read(&mm->mmap_sem);
 995                 goto out_nolock;
 996         }
 997
 998         pmd = mm_find_pmd(mm, address);
 999         if (!pmd) {
1000                 result = SCAN_PMD_NULL;
1001                 mem_cgroup_cancel_charge(new_page, memcg, true);
1002                 up_read(&mm->mmap_sem);
1003                 goto out_nolock;
1004         }
1005
1006         /*
1007          * __collapse_huge_page_swapin always returns with mmap_sem locked.
1008          * If it fails, we release mmap_sem and jump out_nolock.
1009          * Continuing to collapse causes inconsistency.
1010          */
1011         if (!__collapse_huge_page_swapin(mm, vma, address, pmd, referenced)) {
1012                 mem_cgroup_cancel_charge(new_page, memcg, true);
1013                 up_read(&mm->mmap_sem);
1014                 goto out_nolock;
1015         }
1016
1017         up_read(&mm->mmap_sem);
1018         /*
1019          * Prevent all access to pagetables with the exception of
1020          * gup_fast later handled by the ptep_clear_flush and the VM
1021          * handled by the anon_vma lock + PG_lock.
1022          */
1023         down_write(&mm->mmap_sem);
1024         result = hugepage_vma_revalidate(mm, address, &vma);
1025         if (result)
1026                 goto out;
1027         /* check if the pmd is still valid */
1028         if (mm_find_pmd(mm, address) != pmd)
1029                 goto out;
1030
1031         anon_vma_lock_write(vma->anon_vma);
1032
1033         pte = pte_offset_map(pmd, address);
1034         pte_ptl = pte_lockptr(mm, pmd);
1035
1036         mmun_start = address;
1037         mmun_end   = address + HPAGE_PMD_SIZE;
1038         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1039         pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
1040         /*
1041          * After this gup_fast can't run anymore. This also removes
1042          * any huge TLB entry from the CPU so we won't allow
1043          * huge and small TLB entries for the same virtual address
1044          * to avoid the risk of CPU bugs in that area.
1045          */
1046         _pmd = pmdp_collapse_flush(vma, address, pmd);
1047         spin_unlock(pmd_ptl);
1048         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1049
1050         spin_lock(pte_ptl);
1051         isolated = __collapse_huge_page_isolate(vma, address, pte);
1052         spin_unlock(pte_ptl);
1053
1054         if (unlikely(!isolated)) {
1055                 pte_unmap(pte);
1056                 spin_lock(pmd_ptl);
1057                 BUG_ON(!pmd_none(*pmd));
1058                 /*
1059                  * We can only use set_pmd_at when establishing
1060                  * hugepmds and never for establishing regular pmds that
1061                  * points to regular pagetables. Use pmd_populate for that
1062                  */
1063                 pmd_populate(mm, pmd, pmd_pgtable(_pmd));
1064                 spin_unlock(pmd_ptl);
1065                 anon_vma_unlock_write(vma->anon_vma);
1066                 result = SCAN_FAIL;
1067                 goto out;
1068         }
1069
1070         /*
1071          * All pages are isolated and locked so anon_vma rmap
1072          * can't run anymore.
1073          */
1074         anon_vma_unlock_write(vma->anon_vma);
1075
1076         __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
1077         pte_unmap(pte);
1078         __SetPageUptodate(new_page);
1079         pgtable = pmd_pgtable(_pmd);
1080
1081         _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
1082         _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
1083
1084         /*
1085          * spin_lock() below is not the equivalent of smp_wmb(), so
1086          * this is needed to avoid the copy_huge_page writes to become
1087          * visible after the set_pmd_at() write.
1088          */
1089         smp_wmb();
1090
1091         spin_lock(pmd_ptl);
1092         BUG_ON(!pmd_none(*pmd));
1093         page_add_new_anon_rmap(new_page, vma, address, true);
1094         mem_cgroup_commit_charge(new_page, memcg, false, true);
1095         lru_cache_add_active_or_unevictable(new_page, vma);
1096         pgtable_trans_huge_deposit(mm, pmd, pgtable);
1097         set_pmd_at(mm, address, pmd, _pmd);
1098         update_mmu_cache_pmd(vma, address, pmd);
1099         spin_unlock(pmd_ptl);
1100
1101         *hpage = NULL;
1102
1103         khugepaged_pages_collapsed++;
1104         result = SCAN_SUCCEED;
1105 out_up_write:
1106         up_write(&mm->mmap_sem);
1107 out_nolock:
1108         trace_mm_collapse_huge_page(mm, isolated, result);
1109         return;
1110 out:
1111         mem_cgroup_cancel_charge(new_page, memcg, true);
1112         goto out_up_write;
1113 }
1114
1115 static int khugepaged_scan_pmd(struct mm_struct *mm,
1116                                struct vm_area_struct *vma,
1117                                unsigned long address,
1118                                struct page **hpage)
1119 {
1120         pmd_t *pmd;
1121         pte_t *pte, *_pte;
1122         int ret = 0, none_or_zero = 0, result = 0, referenced = 0;
1123         struct page *page = NULL;
1124         unsigned long _address;
1125         spinlock_t *ptl;
1126         int node = NUMA_NO_NODE, unmapped = 0;
1127         bool writable = false;
1128
1129         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1130
1131         pmd = mm_find_pmd(mm, address);
1132         if (!pmd) {
1133                 result = SCAN_PMD_NULL;
1134                 goto out;
1135         }
1136
1137         memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
1138         pte = pte_offset_map_lock(mm, pmd, address, &ptl);
1139         for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
1140              _pte++, _address += PAGE_SIZE) {
1141                 pte_t pteval = *_pte;
1142                 if (is_swap_pte(pteval)) {
1143                         if (++unmapped <= khugepaged_max_ptes_swap) {
1144                                 continue;
1145                         } else {
1146                                 result = SCAN_EXCEED_SWAP_PTE;
1147                                 goto out_unmap;
1148                         }
1149                 }
1150                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
1151                         if (!userfaultfd_armed(vma) &&
1152                             ++none_or_zero <= khugepaged_max_ptes_none) {
1153                                 continue;
1154                         } else {
1155                                 result = SCAN_EXCEED_NONE_PTE;
1156                                 goto out_unmap;
1157                         }
1158                 }
1159                 if (!pte_present(pteval)) {
1160                         result = SCAN_PTE_NON_PRESENT;
1161                         goto out_unmap;
1162                 }
1163                 if (pte_write(pteval))
1164                         writable = true;
1165
1166                 page = vm_normal_page(vma, _address, pteval);
1167                 if (unlikely(!page)) {
1168                         result = SCAN_PAGE_NULL;
1169                         goto out_unmap;
1170                 }
1171
1172                 /* TODO: teach khugepaged to collapse THP mapped with pte */
1173                 if (PageCompound(page)) {
1174                         result = SCAN_PAGE_COMPOUND;
1175                         goto out_unmap;
1176                 }
1177
1178                 /*
1179                  * Record which node the original page is from and save this
1180                  * information to khugepaged_node_load[].
1181                  * Khupaged will allocate hugepage from the node has the max
1182                  * hit record.
1183                  */
1184                 node = page_to_nid(page);
1185                 if (khugepaged_scan_abort(node)) {
1186                         result = SCAN_SCAN_ABORT;
1187                         goto out_unmap;
1188                 }
1189                 khugepaged_node_load[node]++;
1190                 if (!PageLRU(page)) {
1191                         result = SCAN_PAGE_LRU;
1192                         goto out_unmap;
1193                 }
1194                 if (PageLocked(page)) {
1195                         result = SCAN_PAGE_LOCK;
1196                         goto out_unmap;
1197                 }
1198                 if (!PageAnon(page)) {
1199                         result = SCAN_PAGE_ANON;
1200                         goto out_unmap;
1201                 }
1202
1203                 /*
1204                  * cannot use mapcount: can't collapse if there's a gup pin.
1205                  * The page must only be referenced by the scanned process
1206                  * and page swap cache.
1207                  */
1208                 if (page_count(page) != 1 + PageSwapCache(page)) {
1209                         result = SCAN_PAGE_COUNT;
1210                         goto out_unmap;
1211                 }
1212                 if (pte_young(pteval) ||
1213                     page_is_young(page) || PageReferenced(page) ||
1214                     mmu_notifier_test_young(vma->vm_mm, address))
1215                         referenced++;
1216         }
1217         if (writable) {
1218                 if (referenced) {
1219                         result = SCAN_SUCCEED;
1220                         ret = 1;
1221                 } else {
1222                         result = SCAN_LACK_REFERENCED_PAGE;
1223                 }
1224         } else {
1225                 result = SCAN_PAGE_RO;
1226         }
1227 out_unmap:
1228         pte_unmap_unlock(pte, ptl);
1229         if (ret) {
1230                 node = khugepaged_find_target_node();
1231                 /* collapse_huge_page will return with the mmap_sem released */
1232                 collapse_huge_page(mm, address, hpage, node, referenced);
1233         }
1234 out:
1235         trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
1236                                      none_or_zero, result, unmapped);
1237         return ret;
1238 }
1239
1240 static void collect_mm_slot(struct mm_slot *mm_slot)
1241 {
1242         struct mm_struct *mm = mm_slot->mm;
1243
1244         VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
1245
1246         if (khugepaged_test_exit(mm)) {
1247                 /* free mm_slot */
1248                 hash_del(&mm_slot->hash);
1249                 list_del(&mm_slot->mm_node);
1250
1251                 /*
1252                  * Not strictly needed because the mm exited already.
1253                  *
1254                  * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
1255                  */
1256
1257                 /* khugepaged_mm_lock actually not necessary for the below */
1258                 free_mm_slot(mm_slot);
1259                 mmdrop(mm);
1260         }
1261 }
1262
1263 #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
1264 static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
1265 {
1266         struct vm_area_struct *vma;
1267         struct mm_struct *mm;
1268         unsigned long addr;
1269         pmd_t *pmd, _pmd;
1270
1271         i_mmap_lock_write(mapping);
1272         vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1273                 /* probably overkill */
1274                 if (vma->anon_vma)
1275                         continue;
1276                 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
1277                 if (addr & ~HPAGE_PMD_MASK)
1278                         continue;
1279                 if (vma->vm_end < addr + HPAGE_PMD_SIZE)
1280                         continue;
1281                 mm = vma->vm_mm;
1282                 pmd = mm_find_pmd(mm, addr);
1283                 if (!pmd)
1284                         continue;
1285                 /*
1286                  * We need exclusive mmap_sem to retract page table.
1287                  * If trylock fails we would end up with pte-mapped THP after
1288                  * re-fault. Not ideal, but it's more important to not disturb
1289                  * the system too much.
1290                  */
1291                 if (down_write_trylock(&mm->mmap_sem)) {
1292                         if (!khugepaged_test_exit(mm)) {
1293                                 spinlock_t *ptl = pmd_lock(mm, pmd);
1294                                 /* assume page table is clear */
1295                                 _pmd = pmdp_collapse_flush(vma, addr, pmd);
1296                                 spin_unlock(ptl);
1297                                 atomic_long_dec(&mm->nr_ptes);
1298                                 pte_free(mm, pmd_pgtable(_pmd));
1299                         }
1300                         up_write(&mm->mmap_sem);
1301                 }
1302         }
1303         i_mmap_unlock_write(mapping);
1304 }
1305
1306 /**
1307  * collapse_shmem - collapse small tmpfs/shmem pages into huge one.
1308  *
1309  * Basic scheme is simple, details are more complex:
1310  *  - allocate and lock a new huge page;
1311  *  - scan over radix tree replacing old pages the new one
1312  *    + swap in pages if necessary;
1313  *    + fill in gaps;
1314  *    + keep old pages around in case if rollback is required;
1315  *  - if replacing succeed:
1316  *    + copy data over;
1317  *    + free old pages;
1318  *    + unlock huge page;
1319  *  - if replacing failed;
1320  *    + put all pages back and unfreeze them;
1321  *    + restore gaps in the radix-tree;
1322  *    + unlock and free huge page;
1323  */
1324 static void collapse_shmem(struct mm_struct *mm,
1325                 struct address_space *mapping, pgoff_t start,
1326                 struct page **hpage, int node)
1327 {
1328         gfp_t gfp;
1329         struct page *page, *new_page, *tmp;
1330         struct mem_cgroup *memcg;
1331         pgoff_t index, end = start + HPAGE_PMD_NR;
1332         LIST_HEAD(pagelist);
1333         struct radix_tree_iter iter;
1334         void **slot;
1335         int nr_none = 0, result = SCAN_SUCCEED;
1336
1337         VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
1338
1339         /* Only allocate from the target node */
1340         gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
1341
1342         new_page = khugepaged_alloc_page(hpage, gfp, node);
1343         if (!new_page) {
1344                 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
1345                 goto out;
1346         }
1347
1348         /* Do not oom kill for khugepaged charges */
1349         if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp | __GFP_NORETRY,
1350                                            &memcg, true))) {
1351                 result = SCAN_CGROUP_CHARGE_FAIL;
1352                 goto out;
1353         }
1354
1355         __SetPageLocked(new_page);
1356         __SetPageSwapBacked(new_page);
1357         new_page->index = start;
1358         new_page->mapping = mapping;
1359
1360         /*
1361          * At this point the new_page is locked and not up-to-date.
1362          * It's safe to insert it into the page cache, because nobody would
1363          * be able to map it or use it in another way until we unlock it.
1364          */
1365
1366         index = start;
1367         spin_lock_irq(&mapping->tree_lock);
1368         radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1369                 int n = min(iter.index, end) - index;
1370
1371                 /*
1372                  * Stop if extent has been hole-punched, and is now completely
1373                  * empty (the more obvious i_size_read() check would take an
1374                  * irq-unsafe seqlock on 32-bit).
1375                  */
1376                 if (n >= HPAGE_PMD_NR) {
1377                         result = SCAN_TRUNCATED;
1378                         goto tree_locked;
1379                 }
1380
1381                 /*
1382                  * Handle holes in the radix tree: charge it from shmem and
1383                  * insert relevant subpage of new_page into the radix-tree.
1384                  */
1385                 if (n && !shmem_charge(mapping->host, n)) {
1386                         result = SCAN_FAIL;
1387                         goto tree_locked;
1388                 }
1389                 for (; index < min(iter.index, end); index++) {
1390                         radix_tree_insert(&mapping->page_tree, index,
1391                                         new_page + (index % HPAGE_PMD_NR));
1392                 }
1393                 nr_none += n;
1394
1395                 /* We are done. */
1396                 if (index >= end)
1397                         break;
1398
1399                 page = radix_tree_deref_slot_protected(slot,
1400                                 &mapping->tree_lock);
1401                 if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) {
1402                         spin_unlock_irq(&mapping->tree_lock);
1403                         /* swap in or instantiate fallocated page */
1404                         if (shmem_getpage(mapping->host, index, &page,
1405                                                 SGP_NOHUGE)) {
1406                                 result = SCAN_FAIL;
1407                                 goto tree_unlocked;
1408                         }
1409                 } else if (trylock_page(page)) {
1410                         get_page(page);
1411                         spin_unlock_irq(&mapping->tree_lock);
1412                 } else {
1413                         result = SCAN_PAGE_LOCK;
1414                         goto tree_locked;
1415                 }
1416
1417                 /*
1418                  * The page must be locked, so we can drop the tree_lock
1419                  * without racing with truncate.
1420                  */
1421                 VM_BUG_ON_PAGE(!PageLocked(page), page);
1422                 VM_BUG_ON_PAGE(!PageUptodate(page), page);
1423
1424                 /*
1425                  * If file was truncated then extended, or hole-punched, before
1426                  * we locked the first page, then a THP might be there already.
1427                  */
1428                 if (PageTransCompound(page)) {
1429                         result = SCAN_PAGE_COMPOUND;
1430                         goto out_unlock;
1431                 }
1432
1433                 if (page_mapping(page) != mapping) {
1434                         result = SCAN_TRUNCATED;
1435                         goto out_unlock;
1436                 }
1437
1438                 if (isolate_lru_page(page)) {
1439                         result = SCAN_DEL_PAGE_LRU;
1440                         goto out_unlock;
1441                 }
1442
1443                 if (page_mapped(page))
1444                         unmap_mapping_range(mapping, index << PAGE_SHIFT,
1445                                         PAGE_SIZE, 0);
1446
1447                 spin_lock_irq(&mapping->tree_lock);
1448
1449                 slot = radix_tree_lookup_slot(&mapping->page_tree, index);
1450                 VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot,
1451                                         &mapping->tree_lock), page);
1452                 VM_BUG_ON_PAGE(page_mapped(page), page);
1453
1454                 /*
1455                  * The page is expected to have page_count() == 3:
1456                  *  - we hold a pin on it;
1457                  *  - one reference from radix tree;
1458                  *  - one from isolate_lru_page;
1459                  */
1460                 if (!page_ref_freeze(page, 3)) {
1461                         result = SCAN_PAGE_COUNT;
1462                         spin_unlock_irq(&mapping->tree_lock);
1463                         putback_lru_page(page);
1464                         goto out_unlock;
1465                 }
1466
1467                 /*
1468                  * Add the page to the list to be able to undo the collapse if
1469                  * something go wrong.
1470                  */
1471                 list_add_tail(&page->lru, &pagelist);
1472
1473                 /* Finally, replace with the new page. */
1474                 radix_tree_replace_slot(&mapping->page_tree, slot,
1475                                 new_page + (index % HPAGE_PMD_NR));
1476
1477                 slot = radix_tree_iter_resume(slot, &iter);
1478                 index++;
1479                 continue;
1480 out_unlock:
1481                 unlock_page(page);
1482                 put_page(page);
1483                 goto tree_unlocked;
1484         }
1485
1486         /*
1487          * Handle hole in radix tree at the end of the range.
1488          * This code only triggers if there's nothing in radix tree
1489          * beyond 'end'.
1490          */
1491         if (index < end) {
1492                 int n = end - index;
1493
1494                 /* Stop if extent has been truncated, and is now empty */
1495                 if (n >= HPAGE_PMD_NR) {
1496                         result = SCAN_TRUNCATED;
1497                         goto tree_locked;
1498                 }
1499                 if (!shmem_charge(mapping->host, n)) {
1500                         result = SCAN_FAIL;
1501                         goto tree_locked;
1502                 }
1503                 for (; index < end; index++) {
1504                         radix_tree_insert(&mapping->page_tree, index,
1505                                         new_page + (index % HPAGE_PMD_NR));
1506                 }
1507                 nr_none += n;
1508         }
1509
1510         __inc_node_page_state(new_page, NR_SHMEM_THPS);
1511         if (nr_none) {
1512                 struct zone *zone = page_zone(new_page);
1513
1514                 __mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
1515                 __mod_node_page_state(zone->zone_pgdat, NR_SHMEM, nr_none);
1516         }
1517
1518 tree_locked:
1519         spin_unlock_irq(&mapping->tree_lock);
1520 tree_unlocked:
1521
1522         if (result == SCAN_SUCCEED) {
1523                 /*
1524                  * Replacing old pages with new one has succeed, now we need to
1525                  * copy the content and free old pages.
1526                  */
1527                 index = start;
1528                 list_for_each_entry_safe(page, tmp, &pagelist, lru) {
1529                         while (index < page->index) {
1530                                 clear_highpage(new_page + (index % HPAGE_PMD_NR));
1531                                 index++;
1532                         }
1533                         copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
1534                                         page);
1535                         list_del(&page->lru);
1536                         page->mapping = NULL;
1537                         page_ref_unfreeze(page, 1);
1538                         ClearPageActive(page);
1539                         ClearPageUnevictable(page);
1540                         unlock_page(page);
1541                         put_page(page);
1542                         index++;
1543                 }
1544                 while (index < end) {
1545                         clear_highpage(new_page + (index % HPAGE_PMD_NR));
1546                         index++;
1547                 }
1548
1549                 SetPageUptodate(new_page);
1550                 page_ref_add(new_page, HPAGE_PMD_NR - 1);
1551                 set_page_dirty(new_page);
1552                 mem_cgroup_commit_charge(new_page, memcg, false, true);
1553                 lru_cache_add_anon(new_page);
1554
1555                 /*
1556                  * Remove pte page tables, so we can re-fault the page as huge.
1557                  */
1558                 retract_page_tables(mapping, start);
1559                 *hpage = NULL;
1560         } else {
1561                 /* Something went wrong: rollback changes to the radix-tree */
1562                 spin_lock_irq(&mapping->tree_lock);
1563                 mapping->nrpages -= nr_none;
1564                 shmem_uncharge(mapping->host, nr_none);
1565
1566                 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter,
1567                                 start) {
1568                         if (iter.index >= end)
1569                                 break;
1570                         page = list_first_entry_or_null(&pagelist,
1571                                         struct page, lru);
1572                         if (!page || iter.index < page->index) {
1573                                 if (!nr_none)
1574                                         break;
1575                                 nr_none--;
1576                                 /* Put holes back where they were */
1577                                 radix_tree_delete(&mapping->page_tree,
1578                                                   iter.index);
1579                                 continue;
1580                         }
1581
1582                         VM_BUG_ON_PAGE(page->index != iter.index, page);
1583
1584                         /* Unfreeze the page. */
1585                         list_del(&page->lru);
1586                         page_ref_unfreeze(page, 2);
1587                         radix_tree_replace_slot(&mapping->page_tree,
1588                                                 slot, page);
1589                         slot = radix_tree_iter_resume(slot, &iter);
1590                         spin_unlock_irq(&mapping->tree_lock);
1591                         unlock_page(page);
1592                         putback_lru_page(page);
1593                         spin_lock_irq(&mapping->tree_lock);
1594                 }
1595                 VM_BUG_ON(nr_none);
1596                 spin_unlock_irq(&mapping->tree_lock);
1597
1598                 mem_cgroup_cancel_charge(new_page, memcg, true);
1599                 new_page->mapping = NULL;
1600         }
1601
1602         unlock_page(new_page);
1603 out:
1604         VM_BUG_ON(!list_empty(&pagelist));
1605         /* TODO: tracepoints */
1606 }
1607
1608 static void khugepaged_scan_shmem(struct mm_struct *mm,
1609                 struct address_space *mapping,
1610                 pgoff_t start, struct page **hpage)
1611 {
1612         struct page *page = NULL;
1613         struct radix_tree_iter iter;
1614         void **slot;
1615         int present, swap;
1616         int node = NUMA_NO_NODE;
1617         int result = SCAN_SUCCEED;
1618
1619         present = 0;
1620         swap = 0;
1621         memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
1622         rcu_read_lock();
1623         radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1624                 if (iter.index >= start + HPAGE_PMD_NR)
1625                         break;
1626
1627                 page = radix_tree_deref_slot(slot);
1628                 if (radix_tree_deref_retry(page)) {
1629                         slot = radix_tree_iter_retry(&iter);
1630                         continue;
1631                 }
1632
1633                 if (radix_tree_exception(page)) {
1634                         if (++swap > khugepaged_max_ptes_swap) {
1635                                 result = SCAN_EXCEED_SWAP_PTE;
1636                                 break;
1637                         }
1638                         continue;
1639                 }
1640
1641                 if (PageTransCompound(page)) {
1642                         result = SCAN_PAGE_COMPOUND;
1643                         break;
1644                 }
1645
1646                 node = page_to_nid(page);
1647                 if (khugepaged_scan_abort(node)) {
1648                         result = SCAN_SCAN_ABORT;
1649                         break;
1650                 }
1651                 khugepaged_node_load[node]++;
1652
1653                 if (!PageLRU(page)) {
1654                         result = SCAN_PAGE_LRU;
1655                         break;
1656                 }
1657
1658                 if (page_count(page) != 1 + page_mapcount(page)) {
1659                         result = SCAN_PAGE_COUNT;
1660                         break;
1661                 }
1662
1663                 /*
1664                  * We probably should check if the page is referenced here, but
1665                  * nobody would transfer pte_young() to PageReferenced() for us.
1666                  * And rmap walk here is just too costly...
1667                  */
1668
1669                 present++;
1670
1671                 if (need_resched()) {
1672                         slot = radix_tree_iter_resume(slot, &iter);
1673                         cond_resched_rcu();
1674                 }
1675         }
1676         rcu_read_unlock();
1677
1678         if (result == SCAN_SUCCEED) {
1679                 if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
1680                         result = SCAN_EXCEED_NONE_PTE;
1681                 } else {
1682                         node = khugepaged_find_target_node();
1683                         collapse_shmem(mm, mapping, start, hpage, node);
1684                 }
1685         }
1686
1687         /* TODO: tracepoints */
1688 }
1689 #else
1690 static void khugepaged_scan_shmem(struct mm_struct *mm,
1691                 struct address_space *mapping,
1692                 pgoff_t start, struct page **hpage)
1693 {
1694         BUILD_BUG();
1695 }
1696 #endif
1697
1698 static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
1699                                             struct page **hpage)
1700         __releases(&khugepaged_mm_lock)
1701         __acquires(&khugepaged_mm_lock)
1702 {
1703         struct mm_slot *mm_slot;
1704         struct mm_struct *mm;
1705         struct vm_area_struct *vma;
1706         int progress = 0;
1707
1708         VM_BUG_ON(!pages);
1709         VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
1710
1711         if (khugepaged_scan.mm_slot)
1712                 mm_slot = khugepaged_scan.mm_slot;
1713         else {
1714                 mm_slot = list_entry(khugepaged_scan.mm_head.next,
1715                                      struct mm_slot, mm_node);
1716                 khugepaged_scan.address = 0;
1717                 khugepaged_scan.mm_slot = mm_slot;
1718         }
1719         spin_unlock(&khugepaged_mm_lock);
1720
1721         mm = mm_slot->mm;
1722         /*
1723          * Don't wait for semaphore (to avoid long wait times).  Just move to
1724          * the next mm on the list.
1725          */
1726         vma = NULL;
1727         if (unlikely(!down_read_trylock(&mm->mmap_sem)))
1728                 goto breakouterloop_mmap_sem;
1729         if (likely(!khugepaged_test_exit(mm)))
1730                 vma = find_vma(mm, khugepaged_scan.address);
1731
1732         progress++;
1733         for (; vma; vma = vma->vm_next) {
1734                 unsigned long hstart, hend;
1735
1736                 cond_resched();
1737                 if (unlikely(khugepaged_test_exit(mm))) {
1738                         progress++;
1739                         break;
1740                 }
1741                 if (!hugepage_vma_check(vma)) {
1742 skip:
1743                         progress++;
1744                         continue;
1745                 }
1746                 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
1747                 hend = vma->vm_end & HPAGE_PMD_MASK;
1748                 if (hstart >= hend)
1749                         goto skip;
1750                 if (khugepaged_scan.address > hend)
1751                         goto skip;
1752                 if (khugepaged_scan.address < hstart)
1753                         khugepaged_scan.address = hstart;
1754                 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
1755
1756                 while (khugepaged_scan.address < hend) {
1757                         int ret;
1758                         cond_resched();
1759                         if (unlikely(khugepaged_test_exit(mm)))
1760                                 goto breakouterloop;
1761
1762                         VM_BUG_ON(khugepaged_scan.address < hstart ||
1763                                   khugepaged_scan.address + HPAGE_PMD_SIZE >
1764                                   hend);
1765                         if (shmem_file(vma->vm_file)) {
1766                                 struct file *file;
1767                                 pgoff_t pgoff = linear_page_index(vma,
1768                                                 khugepaged_scan.address);
1769                                 if (!shmem_huge_enabled(vma))
1770                                         goto skip;
1771                                 file = get_file(vma->vm_file);
1772                                 up_read(&mm->mmap_sem);
1773                                 ret = 1;
1774                                 khugepaged_scan_shmem(mm, file->f_mapping,
1775                                                 pgoff, hpage);
1776                                 fput(file);
1777                         } else {
1778                                 ret = khugepaged_scan_pmd(mm, vma,
1779                                                 khugepaged_scan.address,
1780                                                 hpage);
1781                         }
1782                         /* move to next address */
1783                         khugepaged_scan.address += HPAGE_PMD_SIZE;
1784                         progress += HPAGE_PMD_NR;
1785                         if (ret)
1786                                 /* we released mmap_sem so break loop */
1787                                 goto breakouterloop_mmap_sem;
1788                         if (progress >= pages)
1789                                 goto breakouterloop;
1790                 }
1791         }
1792 breakouterloop:
1793         up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
1794 breakouterloop_mmap_sem:
1795
1796         spin_lock(&khugepaged_mm_lock);
1797         VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
1798         /*
1799          * Release the current mm_slot if this mm is about to die, or
1800          * if we scanned all vmas of this mm.
1801          */
1802         if (khugepaged_test_exit(mm) || !vma) {
1803                 /*
1804                  * Make sure that if mm_users is reaching zero while
1805                  * khugepaged runs here, khugepaged_exit will find
1806                  * mm_slot not pointing to the exiting mm.
1807                  */
1808                 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
1809                         khugepaged_scan.mm_slot = list_entry(
1810                                 mm_slot->mm_node.next,
1811                                 struct mm_slot, mm_node);
1812                         khugepaged_scan.address = 0;
1813                 } else {
1814                         khugepaged_scan.mm_slot = NULL;
1815                         khugepaged_full_scans++;
1816                 }
1817
1818                 collect_mm_slot(mm_slot);
1819         }
1820
1821         return progress;
1822 }
1823
1824 static int khugepaged_has_work(void)
1825 {
1826         return !list_empty(&khugepaged_scan.mm_head) &&
1827                 khugepaged_enabled();
1828 }
1829
1830 static int khugepaged_wait_event(void)
1831 {
1832         return !list_empty(&khugepaged_scan.mm_head) ||
1833                 kthread_should_stop();
1834 }
1835
1836 static void khugepaged_do_scan(void)
1837 {
1838         struct page *hpage = NULL;
1839         unsigned int progress = 0, pass_through_head = 0;
1840         unsigned int pages = khugepaged_pages_to_scan;
1841         bool wait = true;
1842
1843         barrier(); /* write khugepaged_pages_to_scan to local stack */
1844
1845         while (progress < pages) {
1846                 if (!khugepaged_prealloc_page(&hpage, &wait))
1847                         break;
1848
1849                 cond_resched();
1850
1851                 if (unlikely(kthread_should_stop() || try_to_freeze()))
1852                         break;
1853
1854                 spin_lock(&khugepaged_mm_lock);
1855                 if (!khugepaged_scan.mm_slot)
1856                         pass_through_head++;
1857                 if (khugepaged_has_work() &&
1858                     pass_through_head < 2)
1859                         progress += khugepaged_scan_mm_slot(pages - progress,
1860                                                             &hpage);
1861                 else
1862                         progress = pages;
1863                 spin_unlock(&khugepaged_mm_lock);
1864         }
1865
1866         if (!IS_ERR_OR_NULL(hpage))
1867                 put_page(hpage);
1868 }
1869
1870 static bool khugepaged_should_wakeup(void)
1871 {
1872         return kthread_should_stop() ||
1873                time_after_eq(jiffies, khugepaged_sleep_expire);
1874 }
1875
1876 static void khugepaged_wait_work(void)
1877 {
1878         if (khugepaged_has_work()) {
1879                 const unsigned long scan_sleep_jiffies =
1880                         msecs_to_jiffies(khugepaged_scan_sleep_millisecs);
1881
1882                 if (!scan_sleep_jiffies)
1883                         return;
1884
1885                 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies;
1886                 wait_event_freezable_timeout(khugepaged_wait,
1887                                              khugepaged_should_wakeup(),
1888                                              scan_sleep_jiffies);
1889                 return;
1890         }
1891
1892         if (khugepaged_enabled())
1893                 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
1894 }
1895
1896 static int khugepaged(void *none)
1897 {
1898         struct mm_slot *mm_slot;
1899
1900         set_freezable();
1901         set_user_nice(current, MAX_NICE);
1902
1903         while (!kthread_should_stop()) {
1904                 khugepaged_do_scan();
1905                 khugepaged_wait_work();
1906         }
1907
1908         spin_lock(&khugepaged_mm_lock);
1909         mm_slot = khugepaged_scan.mm_slot;
1910         khugepaged_scan.mm_slot = NULL;
1911         if (mm_slot)
1912                 collect_mm_slot(mm_slot);
1913         spin_unlock(&khugepaged_mm_lock);
1914         return 0;
1915 }
1916
1917 static void set_recommended_min_free_kbytes(void)
1918 {
1919         struct zone *zone;
1920         int nr_zones = 0;
1921         unsigned long recommended_min;
1922
1923         for_each_populated_zone(zone)
1924                 nr_zones++;
1925
1926         /* Ensure 2 pageblocks are free to assist fragmentation avoidance */
1927         recommended_min = pageblock_nr_pages * nr_zones * 2;
1928
1929         /*
1930          * Make sure that on average at least two pageblocks are almost free
1931          * of another type, one for a migratetype to fall back to and a
1932          * second to avoid subsequent fallbacks of other types There are 3
1933          * MIGRATE_TYPES we care about.
1934          */
1935         recommended_min += pageblock_nr_pages * nr_zones *
1936                            MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
1937
1938         /* don't ever allow to reserve more than 5% of the lowmem */
1939         recommended_min = min(recommended_min,
1940                               (unsigned long) nr_free_buffer_pages() / 20);
1941         recommended_min <<= (PAGE_SHIFT-10);
1942
1943         if (recommended_min > min_free_kbytes) {
1944                 if (user_min_free_kbytes >= 0)
1945                         pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
1946                                 min_free_kbytes, recommended_min);
1947
1948                 min_free_kbytes = recommended_min;
1949         }
1950         setup_per_zone_wmarks();
1951 }
1952
1953 int start_stop_khugepaged(void)
1954 {
1955         int err = 0;
1956
1957         mutex_lock(&khugepaged_mutex);
1958         if (khugepaged_enabled()) {
1959                 if (!khugepaged_thread)
1960                         khugepaged_thread = kthread_run(khugepaged, NULL,
1961                                                         "khugepaged");
1962                 if (IS_ERR(khugepaged_thread)) {
1963                         pr_err("khugepaged: kthread_run(khugepaged) failed\n");
1964                         err = PTR_ERR(khugepaged_thread);
1965                         khugepaged_thread = NULL;
1966                         goto fail;
1967                 }
1968
1969                 if (!list_empty(&khugepaged_scan.mm_head))
1970                         wake_up_interruptible(&khugepaged_wait);
1971
1972                 set_recommended_min_free_kbytes();
1973         } else if (khugepaged_thread) {
1974                 kthread_stop(khugepaged_thread);
1975                 khugepaged_thread = NULL;
1976         }
1977 fail:
1978         mutex_unlock(&khugepaged_mutex);
1979         return err;
1980 }
1981
1982 void khugepaged_min_free_kbytes_update(void)
1983 {
1984         mutex_lock(&khugepaged_mutex);
1985         if (khugepaged_enabled() && khugepaged_thread)
1986                 set_recommended_min_free_kbytes();
1987         mutex_unlock(&khugepaged_mutex);
1988 }