2 * Generic hugetlb support.
3 * (C) Nadia Yvette Chambers, April 2004
5 #include <linux/list.h>
6 #include <linux/init.h>
8 #include <linux/seq_file.h>
9 #include <linux/sysctl.h>
10 #include <linux/highmem.h>
11 #include <linux/mmu_notifier.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/compiler.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24 #include <linux/page-isolation.h>
25 #include <linux/jhash.h>
28 #include <asm/pgtable.h>
32 #include <linux/hugetlb.h>
33 #include <linux/hugetlb_cgroup.h>
34 #include <linux/node.h>
37 int hugepages_treat_as_movable;
39 int hugetlb_max_hstate __read_mostly;
40 unsigned int default_hstate_idx;
41 struct hstate hstates[HUGE_MAX_HSTATE];
43 * Minimum page order among possible hugepage sizes, set to a proper value
46 static unsigned int minimum_order __read_mostly = UINT_MAX;
48 __initdata LIST_HEAD(huge_boot_pages);
50 /* for command line parsing */
51 static struct hstate * __initdata parsed_hstate;
52 static unsigned long __initdata default_hstate_max_huge_pages;
53 static unsigned long __initdata default_hstate_size;
54 static bool __initdata parsed_valid_hugepagesz = true;
57 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
58 * free_huge_pages, and surplus_huge_pages.
60 DEFINE_SPINLOCK(hugetlb_lock);
63 * Serializes faults on the same logical page. This is used to
64 * prevent spurious OOMs when the hugepage pool is fully utilized.
66 static int num_fault_mutexes;
67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
69 static inline bool PageHugeFreed(struct page *head)
71 return page_private(head + 4) == -1UL;
74 static inline void SetPageHugeFreed(struct page *head)
76 set_page_private(head + 4, -1UL);
79 static inline void ClearPageHugeFreed(struct page *head)
81 set_page_private(head + 4, 0);
84 /* Forward declaration */
85 static int hugetlb_acct_memory(struct hstate *h, long delta);
87 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
89 bool free = (spool->count == 0) && (spool->used_hpages == 0);
91 spin_unlock(&spool->lock);
93 /* If no pages are used, and no other handles to the subpool
94 * remain, give up any reservations mased on minimum size and
97 if (spool->min_hpages != -1)
98 hugetlb_acct_memory(spool->hstate,
104 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
107 struct hugepage_subpool *spool;
109 spool = kzalloc(sizeof(*spool), GFP_KERNEL);
113 spin_lock_init(&spool->lock);
115 spool->max_hpages = max_hpages;
117 spool->min_hpages = min_hpages;
119 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
123 spool->rsv_hpages = min_hpages;
128 void hugepage_put_subpool(struct hugepage_subpool *spool)
130 spin_lock(&spool->lock);
131 BUG_ON(!spool->count);
133 unlock_or_release_subpool(spool);
137 * Subpool accounting for allocating and reserving pages.
138 * Return -ENOMEM if there are not enough resources to satisfy the
139 * the request. Otherwise, return the number of pages by which the
140 * global pools must be adjusted (upward). The returned value may
141 * only be different than the passed value (delta) in the case where
142 * a subpool minimum size must be manitained.
144 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
152 spin_lock(&spool->lock);
154 if (spool->max_hpages != -1) { /* maximum size accounting */
155 if ((spool->used_hpages + delta) <= spool->max_hpages)
156 spool->used_hpages += delta;
163 /* minimum size accounting */
164 if (spool->min_hpages != -1 && spool->rsv_hpages) {
165 if (delta > spool->rsv_hpages) {
167 * Asking for more reserves than those already taken on
168 * behalf of subpool. Return difference.
170 ret = delta - spool->rsv_hpages;
171 spool->rsv_hpages = 0;
173 ret = 0; /* reserves already accounted for */
174 spool->rsv_hpages -= delta;
179 spin_unlock(&spool->lock);
184 * Subpool accounting for freeing and unreserving pages.
185 * Return the number of global page reservations that must be dropped.
186 * The return value may only be different than the passed value (delta)
187 * in the case where a subpool minimum size must be maintained.
189 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
197 spin_lock(&spool->lock);
199 if (spool->max_hpages != -1) /* maximum size accounting */
200 spool->used_hpages -= delta;
202 /* minimum size accounting */
203 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
204 if (spool->rsv_hpages + delta <= spool->min_hpages)
207 ret = spool->rsv_hpages + delta - spool->min_hpages;
209 spool->rsv_hpages += delta;
210 if (spool->rsv_hpages > spool->min_hpages)
211 spool->rsv_hpages = spool->min_hpages;
215 * If hugetlbfs_put_super couldn't free spool due to an outstanding
216 * quota reference, free it now.
218 unlock_or_release_subpool(spool);
223 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
225 return HUGETLBFS_SB(inode->i_sb)->spool;
228 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
230 return subpool_inode(file_inode(vma->vm_file));
234 * Region tracking -- allows tracking of reservations and instantiated pages
235 * across the pages in a mapping.
237 * The region data structures are embedded into a resv_map and protected
238 * by a resv_map's lock. The set of regions within the resv_map represent
239 * reservations for huge pages, or huge pages that have already been
240 * instantiated within the map. The from and to elements are huge page
241 * indicies into the associated mapping. from indicates the starting index
242 * of the region. to represents the first index past the end of the region.
244 * For example, a file region structure with from == 0 and to == 4 represents
245 * four huge pages in a mapping. It is important to note that the to element
246 * represents the first element past the end of the region. This is used in
247 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
249 * Interval notation of the form [from, to) will be used to indicate that
250 * the endpoint from is inclusive and to is exclusive.
253 struct list_head link;
259 * Add the huge page range represented by [f, t) to the reserve
260 * map. In the normal case, existing regions will be expanded
261 * to accommodate the specified range. Sufficient regions should
262 * exist for expansion due to the previous call to region_chg
263 * with the same range. However, it is possible that region_del
264 * could have been called after region_chg and modifed the map
265 * in such a way that no region exists to be expanded. In this
266 * case, pull a region descriptor from the cache associated with
267 * the map and use that for the new range.
269 * Return the number of new huge pages added to the map. This
270 * number is greater than or equal to zero.
272 static long region_add(struct resv_map *resv, long f, long t)
274 struct list_head *head = &resv->regions;
275 struct file_region *rg, *nrg, *trg;
278 spin_lock(&resv->lock);
279 /* Locate the region we are either in or before. */
280 list_for_each_entry(rg, head, link)
285 * If no region exists which can be expanded to include the
286 * specified range, the list must have been modified by an
287 * interleving call to region_del(). Pull a region descriptor
288 * from the cache and use it for this range.
290 if (&rg->link == head || t < rg->from) {
291 VM_BUG_ON(resv->region_cache_count <= 0);
293 resv->region_cache_count--;
294 nrg = list_first_entry(&resv->region_cache, struct file_region,
296 list_del(&nrg->link);
300 list_add(&nrg->link, rg->link.prev);
306 /* Round our left edge to the current segment if it encloses us. */
310 /* Check for and consume any regions we now overlap with. */
312 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
313 if (&rg->link == head)
318 /* If this area reaches higher then extend our area to
319 * include it completely. If this is not the first area
320 * which we intend to reuse, free it. */
324 /* Decrement return value by the deleted range.
325 * Another range will span this area so that by
326 * end of routine add will be >= zero
328 add -= (rg->to - rg->from);
334 add += (nrg->from - f); /* Added to beginning of region */
336 add += t - nrg->to; /* Added to end of region */
340 resv->adds_in_progress--;
341 spin_unlock(&resv->lock);
347 * Examine the existing reserve map and determine how many
348 * huge pages in the specified range [f, t) are NOT currently
349 * represented. This routine is called before a subsequent
350 * call to region_add that will actually modify the reserve
351 * map to add the specified range [f, t). region_chg does
352 * not change the number of huge pages represented by the
353 * map. However, if the existing regions in the map can not
354 * be expanded to represent the new range, a new file_region
355 * structure is added to the map as a placeholder. This is
356 * so that the subsequent region_add call will have all the
357 * regions it needs and will not fail.
359 * Upon entry, region_chg will also examine the cache of region descriptors
360 * associated with the map. If there are not enough descriptors cached, one
361 * will be allocated for the in progress add operation.
363 * Returns the number of huge pages that need to be added to the existing
364 * reservation map for the range [f, t). This number is greater or equal to
365 * zero. -ENOMEM is returned if a new file_region structure or cache entry
366 * is needed and can not be allocated.
368 static long region_chg(struct resv_map *resv, long f, long t)
370 struct list_head *head = &resv->regions;
371 struct file_region *rg, *nrg = NULL;
375 spin_lock(&resv->lock);
377 resv->adds_in_progress++;
380 * Check for sufficient descriptors in the cache to accommodate
381 * the number of in progress add operations.
383 if (resv->adds_in_progress > resv->region_cache_count) {
384 struct file_region *trg;
386 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
387 /* Must drop lock to allocate a new descriptor. */
388 resv->adds_in_progress--;
389 spin_unlock(&resv->lock);
391 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
397 spin_lock(&resv->lock);
398 list_add(&trg->link, &resv->region_cache);
399 resv->region_cache_count++;
403 /* Locate the region we are before or in. */
404 list_for_each_entry(rg, head, link)
408 /* If we are below the current region then a new region is required.
409 * Subtle, allocate a new region at the position but make it zero
410 * size such that we can guarantee to record the reservation. */
411 if (&rg->link == head || t < rg->from) {
413 resv->adds_in_progress--;
414 spin_unlock(&resv->lock);
415 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
421 INIT_LIST_HEAD(&nrg->link);
425 list_add(&nrg->link, rg->link.prev);
430 /* Round our left edge to the current segment if it encloses us. */
435 /* Check for and consume any regions we now overlap with. */
436 list_for_each_entry(rg, rg->link.prev, link) {
437 if (&rg->link == head)
442 /* We overlap with this area, if it extends further than
443 * us then we must extend ourselves. Account for its
444 * existing reservation. */
449 chg -= rg->to - rg->from;
453 spin_unlock(&resv->lock);
454 /* We already know we raced and no longer need the new region */
458 spin_unlock(&resv->lock);
463 * Abort the in progress add operation. The adds_in_progress field
464 * of the resv_map keeps track of the operations in progress between
465 * calls to region_chg and region_add. Operations are sometimes
466 * aborted after the call to region_chg. In such cases, region_abort
467 * is called to decrement the adds_in_progress counter.
469 * NOTE: The range arguments [f, t) are not needed or used in this
470 * routine. They are kept to make reading the calling code easier as
471 * arguments will match the associated region_chg call.
473 static void region_abort(struct resv_map *resv, long f, long t)
475 spin_lock(&resv->lock);
476 VM_BUG_ON(!resv->region_cache_count);
477 resv->adds_in_progress--;
478 spin_unlock(&resv->lock);
482 * Delete the specified range [f, t) from the reserve map. If the
483 * t parameter is LONG_MAX, this indicates that ALL regions after f
484 * should be deleted. Locate the regions which intersect [f, t)
485 * and either trim, delete or split the existing regions.
487 * Returns the number of huge pages deleted from the reserve map.
488 * In the normal case, the return value is zero or more. In the
489 * case where a region must be split, a new region descriptor must
490 * be allocated. If the allocation fails, -ENOMEM will be returned.
491 * NOTE: If the parameter t == LONG_MAX, then we will never split
492 * a region and possibly return -ENOMEM. Callers specifying
493 * t == LONG_MAX do not need to check for -ENOMEM error.
495 static long region_del(struct resv_map *resv, long f, long t)
497 struct list_head *head = &resv->regions;
498 struct file_region *rg, *trg;
499 struct file_region *nrg = NULL;
503 spin_lock(&resv->lock);
504 list_for_each_entry_safe(rg, trg, head, link) {
506 * Skip regions before the range to be deleted. file_region
507 * ranges are normally of the form [from, to). However, there
508 * may be a "placeholder" entry in the map which is of the form
509 * (from, to) with from == to. Check for placeholder entries
510 * at the beginning of the range to be deleted.
512 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
518 if (f > rg->from && t < rg->to) { /* Must split region */
520 * Check for an entry in the cache before dropping
521 * lock and attempting allocation.
524 resv->region_cache_count > resv->adds_in_progress) {
525 nrg = list_first_entry(&resv->region_cache,
528 list_del(&nrg->link);
529 resv->region_cache_count--;
533 spin_unlock(&resv->lock);
534 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
542 /* New entry for end of split region */
545 INIT_LIST_HEAD(&nrg->link);
547 /* Original entry is trimmed */
550 list_add(&nrg->link, &rg->link);
555 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
556 del += rg->to - rg->from;
562 if (f <= rg->from) { /* Trim beginning of region */
565 } else { /* Trim end of region */
571 spin_unlock(&resv->lock);
577 * A rare out of memory error was encountered which prevented removal of
578 * the reserve map region for a page. The huge page itself was free'ed
579 * and removed from the page cache. This routine will adjust the subpool
580 * usage count, and the global reserve count if needed. By incrementing
581 * these counts, the reserve map entry which could not be deleted will
582 * appear as a "reserved" entry instead of simply dangling with incorrect
585 void hugetlb_fix_reserve_counts(struct inode *inode)
587 struct hugepage_subpool *spool = subpool_inode(inode);
589 bool reserved = false;
591 rsv_adjust = hugepage_subpool_get_pages(spool, 1);
592 if (rsv_adjust > 0) {
593 struct hstate *h = hstate_inode(inode);
595 if (!hugetlb_acct_memory(h, 1))
597 } else if (!rsv_adjust) {
602 pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
606 * Count and return the number of huge pages in the reserve map
607 * that intersect with the range [f, t).
609 static long region_count(struct resv_map *resv, long f, long t)
611 struct list_head *head = &resv->regions;
612 struct file_region *rg;
615 spin_lock(&resv->lock);
616 /* Locate each segment we overlap with, and count that overlap. */
617 list_for_each_entry(rg, head, link) {
626 seg_from = max(rg->from, f);
627 seg_to = min(rg->to, t);
629 chg += seg_to - seg_from;
631 spin_unlock(&resv->lock);
637 * Convert the address within this vma to the page offset within
638 * the mapping, in pagecache page units; huge pages here.
640 static pgoff_t vma_hugecache_offset(struct hstate *h,
641 struct vm_area_struct *vma, unsigned long address)
643 return ((address - vma->vm_start) >> huge_page_shift(h)) +
644 (vma->vm_pgoff >> huge_page_order(h));
647 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
648 unsigned long address)
650 return vma_hugecache_offset(hstate_vma(vma), vma, address);
652 EXPORT_SYMBOL_GPL(linear_hugepage_index);
655 * Return the size of the pages allocated when backing a VMA. In the majority
656 * cases this will be same size as used by the page table entries.
658 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
660 struct hstate *hstate;
662 if (!is_vm_hugetlb_page(vma))
665 hstate = hstate_vma(vma);
667 return 1UL << huge_page_shift(hstate);
669 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
672 * Return the page size being used by the MMU to back a VMA. In the majority
673 * of cases, the page size used by the kernel matches the MMU size. On
674 * architectures where it differs, an architecture-specific version of this
675 * function is required.
677 #ifndef vma_mmu_pagesize
678 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
680 return vma_kernel_pagesize(vma);
685 * Flags for MAP_PRIVATE reservations. These are stored in the bottom
686 * bits of the reservation map pointer, which are always clear due to
689 #define HPAGE_RESV_OWNER (1UL << 0)
690 #define HPAGE_RESV_UNMAPPED (1UL << 1)
691 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
694 * These helpers are used to track how many pages are reserved for
695 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
696 * is guaranteed to have their future faults succeed.
698 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
699 * the reserve counters are updated with the hugetlb_lock held. It is safe
700 * to reset the VMA at fork() time as it is not in use yet and there is no
701 * chance of the global counters getting corrupted as a result of the values.
703 * The private mapping reservation is represented in a subtly different
704 * manner to a shared mapping. A shared mapping has a region map associated
705 * with the underlying file, this region map represents the backing file
706 * pages which have ever had a reservation assigned which this persists even
707 * after the page is instantiated. A private mapping has a region map
708 * associated with the original mmap which is attached to all VMAs which
709 * reference it, this region map represents those offsets which have consumed
710 * reservation ie. where pages have been instantiated.
712 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
714 return (unsigned long)vma->vm_private_data;
717 static void set_vma_private_data(struct vm_area_struct *vma,
720 vma->vm_private_data = (void *)value;
723 struct resv_map *resv_map_alloc(void)
725 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
726 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
728 if (!resv_map || !rg) {
734 kref_init(&resv_map->refs);
735 spin_lock_init(&resv_map->lock);
736 INIT_LIST_HEAD(&resv_map->regions);
738 resv_map->adds_in_progress = 0;
740 INIT_LIST_HEAD(&resv_map->region_cache);
741 list_add(&rg->link, &resv_map->region_cache);
742 resv_map->region_cache_count = 1;
747 void resv_map_release(struct kref *ref)
749 struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
750 struct list_head *head = &resv_map->region_cache;
751 struct file_region *rg, *trg;
753 /* Clear out any active regions before we release the map. */
754 region_del(resv_map, 0, LONG_MAX);
756 /* ... and any entries left in the cache */
757 list_for_each_entry_safe(rg, trg, head, link) {
762 VM_BUG_ON(resv_map->adds_in_progress);
767 static inline struct resv_map *inode_resv_map(struct inode *inode)
769 return inode->i_mapping->private_data;
772 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
774 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
775 if (vma->vm_flags & VM_MAYSHARE) {
776 struct address_space *mapping = vma->vm_file->f_mapping;
777 struct inode *inode = mapping->host;
779 return inode_resv_map(inode);
782 return (struct resv_map *)(get_vma_private_data(vma) &
787 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
789 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
790 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
792 set_vma_private_data(vma, (get_vma_private_data(vma) &
793 HPAGE_RESV_MASK) | (unsigned long)map);
796 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
798 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
799 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
801 set_vma_private_data(vma, get_vma_private_data(vma) | flags);
804 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
806 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
808 return (get_vma_private_data(vma) & flag) != 0;
811 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
812 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
814 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
815 if (!(vma->vm_flags & VM_MAYSHARE))
816 vma->vm_private_data = (void *)0;
819 /* Returns true if the VMA has associated reserve pages */
820 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
822 if (vma->vm_flags & VM_NORESERVE) {
824 * This address is already reserved by other process(chg == 0),
825 * so, we should decrement reserved count. Without decrementing,
826 * reserve count remains after releasing inode, because this
827 * allocated page will go into page cache and is regarded as
828 * coming from reserved pool in releasing step. Currently, we
829 * don't have any other solution to deal with this situation
830 * properly, so add work-around here.
832 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
838 /* Shared mappings always use reserves */
839 if (vma->vm_flags & VM_MAYSHARE) {
841 * We know VM_NORESERVE is not set. Therefore, there SHOULD
842 * be a region map for all pages. The only situation where
843 * there is no region map is if a hole was punched via
844 * fallocate. In this case, there really are no reverves to
845 * use. This situation is indicated if chg != 0.
854 * Only the process that called mmap() has reserves for
857 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
859 * Like the shared case above, a hole punch or truncate
860 * could have been performed on the private mapping.
861 * Examine the value of chg to determine if reserves
862 * actually exist or were previously consumed.
863 * Very Subtle - The value of chg comes from a previous
864 * call to vma_needs_reserves(). The reserve map for
865 * private mappings has different (opposite) semantics
866 * than that of shared mappings. vma_needs_reserves()
867 * has already taken this difference in semantics into
868 * account. Therefore, the meaning of chg is the same
869 * as in the shared case above. Code could easily be
870 * combined, but keeping it separate draws attention to
871 * subtle differences.
882 static void enqueue_huge_page(struct hstate *h, struct page *page)
884 int nid = page_to_nid(page);
885 list_move(&page->lru, &h->hugepage_freelists[nid]);
886 h->free_huge_pages++;
887 h->free_huge_pages_node[nid]++;
888 SetPageHugeFreed(page);
891 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
895 list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
896 if (!is_migrate_isolate_page(page))
899 * if 'non-isolated free hugepage' not found on the list,
900 * the allocation fails.
902 if (&h->hugepage_freelists[nid] == &page->lru)
904 list_move(&page->lru, &h->hugepage_activelist);
905 set_page_refcounted(page);
906 ClearPageHugeFreed(page);
907 h->free_huge_pages--;
908 h->free_huge_pages_node[nid]--;
912 /* Movability of hugepages depends on migration support. */
913 static inline gfp_t htlb_alloc_mask(struct hstate *h)
915 if (hugepages_treat_as_movable || hugepage_migration_supported(h))
916 return GFP_HIGHUSER_MOVABLE;
921 static struct page *dequeue_huge_page_vma(struct hstate *h,
922 struct vm_area_struct *vma,
923 unsigned long address, int avoid_reserve,
926 struct page *page = NULL;
927 struct mempolicy *mpol;
928 nodemask_t *nodemask;
929 struct zonelist *zonelist;
932 unsigned int cpuset_mems_cookie;
935 * A child process with MAP_PRIVATE mappings created by their parent
936 * have no page reserves. This check ensures that reservations are
937 * not "stolen". The child may still get SIGKILLed
939 if (!vma_has_reserves(vma, chg) &&
940 h->free_huge_pages - h->resv_huge_pages == 0)
943 /* If reserves cannot be used, ensure enough pages are in the pool */
944 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
948 cpuset_mems_cookie = read_mems_allowed_begin();
949 zonelist = huge_zonelist(vma, address,
950 htlb_alloc_mask(h), &mpol, &nodemask);
952 for_each_zone_zonelist_nodemask(zone, z, zonelist,
953 MAX_NR_ZONES - 1, nodemask) {
954 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
955 page = dequeue_huge_page_node(h, zone_to_nid(zone));
959 if (!vma_has_reserves(vma, chg))
962 SetPagePrivate(page);
963 h->resv_huge_pages--;
970 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
979 * common helper functions for hstate_next_node_to_{alloc|free}.
980 * We may have allocated or freed a huge page based on a different
981 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
982 * be outside of *nodes_allowed. Ensure that we use an allowed
983 * node for alloc or free.
985 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
987 nid = next_node_in(nid, *nodes_allowed);
988 VM_BUG_ON(nid >= MAX_NUMNODES);
993 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
995 if (!node_isset(nid, *nodes_allowed))
996 nid = next_node_allowed(nid, nodes_allowed);
1001 * returns the previously saved node ["this node"] from which to
1002 * allocate a persistent huge page for the pool and advance the
1003 * next node from which to allocate, handling wrap at end of node
1006 static int hstate_next_node_to_alloc(struct hstate *h,
1007 nodemask_t *nodes_allowed)
1011 VM_BUG_ON(!nodes_allowed);
1013 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1014 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1020 * helper for free_pool_huge_page() - return the previously saved
1021 * node ["this node"] from which to free a huge page. Advance the
1022 * next node id whether or not we find a free huge page to free so
1023 * that the next attempt to free addresses the next node.
1025 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1029 VM_BUG_ON(!nodes_allowed);
1031 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1032 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1037 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
1038 for (nr_nodes = nodes_weight(*mask); \
1040 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
1043 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
1044 for (nr_nodes = nodes_weight(*mask); \
1046 ((node = hstate_next_node_to_free(hs, mask)) || 1); \
1049 #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \
1050 ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \
1051 defined(CONFIG_CMA))
1052 static void destroy_compound_gigantic_page(struct page *page,
1056 int nr_pages = 1 << order;
1057 struct page *p = page + 1;
1059 atomic_set(compound_mapcount_ptr(page), 0);
1060 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1061 clear_compound_head(p);
1062 set_page_refcounted(p);
1065 set_compound_order(page, 0);
1066 __ClearPageHead(page);
1069 static void free_gigantic_page(struct page *page, unsigned int order)
1071 free_contig_range(page_to_pfn(page), 1 << order);
1074 static int __alloc_gigantic_page(unsigned long start_pfn,
1075 unsigned long nr_pages)
1077 unsigned long end_pfn = start_pfn + nr_pages;
1078 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1081 static bool pfn_range_valid_gigantic(struct zone *z,
1082 unsigned long start_pfn, unsigned long nr_pages)
1084 unsigned long i, end_pfn = start_pfn + nr_pages;
1087 for (i = start_pfn; i < end_pfn; i++) {
1091 page = pfn_to_page(i);
1093 if (page_zone(page) != z)
1096 if (PageReserved(page))
1099 if (page_count(page) > 0)
1109 static bool zone_spans_last_pfn(const struct zone *zone,
1110 unsigned long start_pfn, unsigned long nr_pages)
1112 unsigned long last_pfn = start_pfn + nr_pages - 1;
1113 return zone_spans_pfn(zone, last_pfn);
1116 static struct page *alloc_gigantic_page(int nid, unsigned int order)
1118 unsigned long nr_pages = 1 << order;
1119 unsigned long ret, pfn, flags;
1122 z = NODE_DATA(nid)->node_zones;
1123 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1124 spin_lock_irqsave(&z->lock, flags);
1126 pfn = ALIGN(z->zone_start_pfn, nr_pages);
1127 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1128 if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1130 * We release the zone lock here because
1131 * alloc_contig_range() will also lock the zone
1132 * at some point. If there's an allocation
1133 * spinning on this lock, it may win the race
1134 * and cause alloc_contig_range() to fail...
1136 spin_unlock_irqrestore(&z->lock, flags);
1137 ret = __alloc_gigantic_page(pfn, nr_pages);
1139 return pfn_to_page(pfn);
1140 spin_lock_irqsave(&z->lock, flags);
1145 spin_unlock_irqrestore(&z->lock, flags);
1151 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1152 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1154 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1158 page = alloc_gigantic_page(nid, huge_page_order(h));
1160 prep_compound_gigantic_page(page, huge_page_order(h));
1161 prep_new_huge_page(h, page, nid);
1167 static int alloc_fresh_gigantic_page(struct hstate *h,
1168 nodemask_t *nodes_allowed)
1170 struct page *page = NULL;
1173 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1174 page = alloc_fresh_gigantic_page_node(h, node);
1182 static inline bool gigantic_page_supported(void) { return true; }
1184 static inline bool gigantic_page_supported(void) { return false; }
1185 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1186 static inline void destroy_compound_gigantic_page(struct page *page,
1187 unsigned int order) { }
1188 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1189 nodemask_t *nodes_allowed) { return 0; }
1192 static void update_and_free_page(struct hstate *h, struct page *page)
1195 struct page *subpage = page;
1197 if (hstate_is_gigantic(h) && !gigantic_page_supported())
1201 h->nr_huge_pages_node[page_to_nid(page)]--;
1202 for (i = 0; i < pages_per_huge_page(h);
1203 i++, subpage = mem_map_next(subpage, page, i)) {
1204 subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1205 1 << PG_referenced | 1 << PG_dirty |
1206 1 << PG_active | 1 << PG_private |
1209 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1210 set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1211 set_page_refcounted(page);
1212 if (hstate_is_gigantic(h)) {
1213 destroy_compound_gigantic_page(page, huge_page_order(h));
1214 free_gigantic_page(page, huge_page_order(h));
1216 __free_pages(page, huge_page_order(h));
1220 struct hstate *size_to_hstate(unsigned long size)
1224 for_each_hstate(h) {
1225 if (huge_page_size(h) == size)
1232 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1233 * to hstate->hugepage_activelist.)
1235 * This function can be called for tail pages, but never returns true for them.
1237 bool page_huge_active(struct page *page)
1239 return PageHeadHuge(page) && PagePrivate(&page[1]);
1242 /* never called for tail page */
1243 void set_page_huge_active(struct page *page)
1245 VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1246 SetPagePrivate(&page[1]);
1249 static void clear_page_huge_active(struct page *page)
1251 VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1252 ClearPagePrivate(&page[1]);
1255 void free_huge_page(struct page *page)
1258 * Can't pass hstate in here because it is called from the
1259 * compound page destructor.
1261 struct hstate *h = page_hstate(page);
1262 int nid = page_to_nid(page);
1263 struct hugepage_subpool *spool =
1264 (struct hugepage_subpool *)page_private(page);
1265 bool restore_reserve;
1267 set_page_private(page, 0);
1268 page->mapping = NULL;
1269 VM_BUG_ON_PAGE(page_count(page), page);
1270 VM_BUG_ON_PAGE(page_mapcount(page), page);
1271 restore_reserve = PagePrivate(page);
1272 ClearPagePrivate(page);
1275 * If PagePrivate() was set on page, page allocation consumed a
1276 * reservation. If the page was associated with a subpool, there
1277 * would have been a page reserved in the subpool before allocation
1278 * via hugepage_subpool_get_pages(). Since we are 'restoring' the
1279 * reservtion, do not call hugepage_subpool_put_pages() as this will
1280 * remove the reserved page from the subpool.
1282 if (!restore_reserve) {
1284 * A return code of zero implies that the subpool will be
1285 * under its minimum size if the reservation is not restored
1286 * after page is free. Therefore, force restore_reserve
1289 if (hugepage_subpool_put_pages(spool, 1) == 0)
1290 restore_reserve = true;
1293 spin_lock(&hugetlb_lock);
1294 clear_page_huge_active(page);
1295 hugetlb_cgroup_uncharge_page(hstate_index(h),
1296 pages_per_huge_page(h), page);
1297 if (restore_reserve)
1298 h->resv_huge_pages++;
1300 if (h->surplus_huge_pages_node[nid]) {
1301 /* remove the page from active list */
1302 list_del(&page->lru);
1303 update_and_free_page(h, page);
1304 h->surplus_huge_pages--;
1305 h->surplus_huge_pages_node[nid]--;
1307 arch_clear_hugepage_flags(page);
1308 enqueue_huge_page(h, page);
1310 spin_unlock(&hugetlb_lock);
1313 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1315 INIT_LIST_HEAD(&page->lru);
1316 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1317 spin_lock(&hugetlb_lock);
1318 set_hugetlb_cgroup(page, NULL);
1320 h->nr_huge_pages_node[nid]++;
1321 ClearPageHugeFreed(page);
1322 spin_unlock(&hugetlb_lock);
1323 put_page(page); /* free it into the hugepage allocator */
1326 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1329 int nr_pages = 1 << order;
1330 struct page *p = page + 1;
1332 /* we rely on prep_new_huge_page to set the destructor */
1333 set_compound_order(page, order);
1334 __ClearPageReserved(page);
1335 __SetPageHead(page);
1336 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1338 * For gigantic hugepages allocated through bootmem at
1339 * boot, it's safer to be consistent with the not-gigantic
1340 * hugepages and clear the PG_reserved bit from all tail pages
1341 * too. Otherwse drivers using get_user_pages() to access tail
1342 * pages may get the reference counting wrong if they see
1343 * PG_reserved set on a tail page (despite the head page not
1344 * having PG_reserved set). Enforcing this consistency between
1345 * head and tail pages allows drivers to optimize away a check
1346 * on the head page when they need know if put_page() is needed
1347 * after get_user_pages().
1349 __ClearPageReserved(p);
1350 set_page_count(p, 0);
1351 set_compound_head(p, page);
1353 atomic_set(compound_mapcount_ptr(page), -1);
1357 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1358 * transparent huge pages. See the PageTransHuge() documentation for more
1361 int PageHuge(struct page *page)
1363 if (!PageCompound(page))
1366 page = compound_head(page);
1367 return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1369 EXPORT_SYMBOL_GPL(PageHuge);
1372 * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1373 * normal or transparent huge pages.
1375 int PageHeadHuge(struct page *page_head)
1377 if (!PageHead(page_head))
1380 return get_compound_page_dtor(page_head) == free_huge_page;
1383 pgoff_t hugetlb_basepage_index(struct page *page)
1385 struct page *page_head = compound_head(page);
1386 pgoff_t index = page_index(page_head);
1387 unsigned long compound_idx;
1389 if (compound_order(page_head) >= MAX_ORDER)
1390 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1392 compound_idx = page - page_head;
1394 return (index << compound_order(page_head)) + compound_idx;
1397 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1401 page = __alloc_pages_node(nid,
1402 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1403 __GFP_REPEAT|__GFP_NOWARN,
1404 huge_page_order(h));
1406 prep_new_huge_page(h, page, nid);
1412 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1418 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1419 page = alloc_fresh_huge_page_node(h, node);
1427 count_vm_event(HTLB_BUDDY_PGALLOC);
1429 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1435 * Free huge page from pool from next node to free.
1436 * Attempt to keep persistent huge pages more or less
1437 * balanced over allowed nodes.
1438 * Called with hugetlb_lock locked.
1440 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1446 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1448 * If we're returning unused surplus pages, only examine
1449 * nodes with surplus pages.
1451 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1452 !list_empty(&h->hugepage_freelists[node])) {
1454 list_entry(h->hugepage_freelists[node].next,
1456 list_del(&page->lru);
1457 h->free_huge_pages--;
1458 h->free_huge_pages_node[node]--;
1460 h->surplus_huge_pages--;
1461 h->surplus_huge_pages_node[node]--;
1463 update_and_free_page(h, page);
1473 * Dissolve a given free hugepage into free buddy pages. This function does
1474 * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the
1475 * number of free hugepages would be reduced below the number of reserved
1478 static int dissolve_free_huge_page(struct page *page)
1483 spin_lock(&hugetlb_lock);
1484 if (PageHuge(page) && !page_count(page)) {
1485 struct page *head = compound_head(page);
1486 struct hstate *h = page_hstate(head);
1487 int nid = page_to_nid(head);
1488 if (h->free_huge_pages - h->resv_huge_pages == 0) {
1494 * We should make sure that the page is already on the free list
1495 * when it is dissolved.
1497 if (unlikely(!PageHugeFreed(head))) {
1498 spin_unlock(&hugetlb_lock);
1502 * Theoretically, we should return -EBUSY when we
1503 * encounter this race. In fact, we have a chance
1504 * to successfully dissolve the page if we do a
1505 * retry. Because the race window is quite small.
1506 * If we seize this opportunity, it is an optimization
1507 * for increasing the success rate of dissolving page.
1512 list_del(&head->lru);
1513 h->free_huge_pages--;
1514 h->free_huge_pages_node[nid]--;
1515 h->max_huge_pages--;
1516 update_and_free_page(h, head);
1519 spin_unlock(&hugetlb_lock);
1524 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1525 * make specified memory blocks removable from the system.
1526 * Note that this will dissolve a free gigantic hugepage completely, if any
1527 * part of it lies within the given range.
1528 * Also note that if dissolve_free_huge_page() returns with an error, all
1529 * free hugepages that were dissolved before that error are lost.
1531 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1537 if (!hugepages_supported())
1540 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1541 page = pfn_to_page(pfn);
1542 if (PageHuge(page) && !page_count(page)) {
1543 rc = dissolve_free_huge_page(page);
1553 * There are 3 ways this can get called:
1554 * 1. With vma+addr: we use the VMA's memory policy
1555 * 2. With !vma, but nid=NUMA_NO_NODE: We try to allocate a huge
1556 * page from any node, and let the buddy allocator itself figure
1558 * 3. With !vma, but nid!=NUMA_NO_NODE. We allocate a huge page
1559 * strictly from 'nid'
1561 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1562 struct vm_area_struct *vma, unsigned long addr, int nid)
1564 int order = huge_page_order(h);
1565 gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1566 unsigned int cpuset_mems_cookie;
1569 * We need a VMA to get a memory policy. If we do not
1570 * have one, we use the 'nid' argument.
1572 * The mempolicy stuff below has some non-inlined bits
1573 * and calls ->vm_ops. That makes it hard to optimize at
1574 * compile-time, even when NUMA is off and it does
1575 * nothing. This helps the compiler optimize it out.
1577 if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1579 * If a specific node is requested, make sure to
1580 * get memory from there, but only when a node
1581 * is explicitly specified.
1583 if (nid != NUMA_NO_NODE)
1584 gfp |= __GFP_THISNODE;
1586 * Make sure to call something that can handle
1589 return alloc_pages_node(nid, gfp, order);
1593 * OK, so we have a VMA. Fetch the mempolicy and try to
1594 * allocate a huge page with it. We will only reach this
1595 * when CONFIG_NUMA=y.
1599 struct mempolicy *mpol;
1600 struct zonelist *zl;
1601 nodemask_t *nodemask;
1603 cpuset_mems_cookie = read_mems_allowed_begin();
1604 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
1605 mpol_cond_put(mpol);
1606 page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
1609 } while (read_mems_allowed_retry(cpuset_mems_cookie));
1615 * There are two ways to allocate a huge page:
1616 * 1. When you have a VMA and an address (like a fault)
1617 * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1619 * 'vma' and 'addr' are only for (1). 'nid' is always NUMA_NO_NODE in
1620 * this case which signifies that the allocation should be done with
1621 * respect for the VMA's memory policy.
1623 * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1624 * implies that memory policies will not be taken in to account.
1626 static struct page *__alloc_buddy_huge_page(struct hstate *h,
1627 struct vm_area_struct *vma, unsigned long addr, int nid)
1632 if (hstate_is_gigantic(h))
1636 * Make sure that anyone specifying 'nid' is not also specifying a VMA.
1637 * This makes sure the caller is picking _one_ of the modes with which
1638 * we can call this function, not both.
1640 if (vma || (addr != -1)) {
1641 VM_WARN_ON_ONCE(addr == -1);
1642 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1645 * Assume we will successfully allocate the surplus page to
1646 * prevent racing processes from causing the surplus to exceed
1649 * This however introduces a different race, where a process B
1650 * tries to grow the static hugepage pool while alloc_pages() is
1651 * called by process A. B will only examine the per-node
1652 * counters in determining if surplus huge pages can be
1653 * converted to normal huge pages in adjust_pool_surplus(). A
1654 * won't be able to increment the per-node counter, until the
1655 * lock is dropped by B, but B doesn't drop hugetlb_lock until
1656 * no more huge pages can be converted from surplus to normal
1657 * state (and doesn't try to convert again). Thus, we have a
1658 * case where a surplus huge page exists, the pool is grown, and
1659 * the surplus huge page still exists after, even though it
1660 * should just have been converted to a normal huge page. This
1661 * does not leak memory, though, as the hugepage will be freed
1662 * once it is out of use. It also does not allow the counters to
1663 * go out of whack in adjust_pool_surplus() as we don't modify
1664 * the node values until we've gotten the hugepage and only the
1665 * per-node value is checked there.
1667 spin_lock(&hugetlb_lock);
1668 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1669 spin_unlock(&hugetlb_lock);
1673 h->surplus_huge_pages++;
1675 spin_unlock(&hugetlb_lock);
1677 page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1679 spin_lock(&hugetlb_lock);
1681 INIT_LIST_HEAD(&page->lru);
1682 r_nid = page_to_nid(page);
1683 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1684 set_hugetlb_cgroup(page, NULL);
1686 * We incremented the global counters already
1688 h->nr_huge_pages_node[r_nid]++;
1689 h->surplus_huge_pages_node[r_nid]++;
1690 __count_vm_event(HTLB_BUDDY_PGALLOC);
1693 h->surplus_huge_pages--;
1694 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1696 spin_unlock(&hugetlb_lock);
1702 * Allocate a huge page from 'nid'. Note, 'nid' may be
1703 * NUMA_NO_NODE, which means that it may be allocated
1707 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1709 unsigned long addr = -1;
1711 return __alloc_buddy_huge_page(h, NULL, addr, nid);
1715 * Use the VMA's mpolicy to allocate a huge page from the buddy.
1718 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1719 struct vm_area_struct *vma, unsigned long addr)
1721 return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1725 * This allocation function is useful in the context where vma is irrelevant.
1726 * E.g. soft-offlining uses this function because it only cares physical
1727 * address of error page.
1729 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1731 struct page *page = NULL;
1733 spin_lock(&hugetlb_lock);
1734 if (h->free_huge_pages - h->resv_huge_pages > 0)
1735 page = dequeue_huge_page_node(h, nid);
1736 spin_unlock(&hugetlb_lock);
1739 page = __alloc_buddy_huge_page_no_mpol(h, nid);
1745 * Increase the hugetlb pool such that it can accommodate a reservation
1748 static int gather_surplus_pages(struct hstate *h, int delta)
1750 struct list_head surplus_list;
1751 struct page *page, *tmp;
1753 int needed, allocated;
1754 bool alloc_ok = true;
1756 needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1758 h->resv_huge_pages += delta;
1763 INIT_LIST_HEAD(&surplus_list);
1767 spin_unlock(&hugetlb_lock);
1768 for (i = 0; i < needed; i++) {
1769 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1774 list_add(&page->lru, &surplus_list);
1779 * After retaking hugetlb_lock, we need to recalculate 'needed'
1780 * because either resv_huge_pages or free_huge_pages may have changed.
1782 spin_lock(&hugetlb_lock);
1783 needed = (h->resv_huge_pages + delta) -
1784 (h->free_huge_pages + allocated);
1789 * We were not able to allocate enough pages to
1790 * satisfy the entire reservation so we free what
1791 * we've allocated so far.
1796 * The surplus_list now contains _at_least_ the number of extra pages
1797 * needed to accommodate the reservation. Add the appropriate number
1798 * of pages to the hugetlb pool and free the extras back to the buddy
1799 * allocator. Commit the entire reservation here to prevent another
1800 * process from stealing the pages as they are added to the pool but
1801 * before they are reserved.
1803 needed += allocated;
1804 h->resv_huge_pages += delta;
1807 /* Free the needed pages to the hugetlb pool */
1808 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1812 * This page is now managed by the hugetlb allocator and has
1813 * no users -- drop the buddy allocator's reference.
1815 put_page_testzero(page);
1816 VM_BUG_ON_PAGE(page_count(page), page);
1817 enqueue_huge_page(h, page);
1820 spin_unlock(&hugetlb_lock);
1822 /* Free unnecessary surplus pages to the buddy allocator */
1823 list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1825 spin_lock(&hugetlb_lock);
1831 * This routine has two main purposes:
1832 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
1833 * in unused_resv_pages. This corresponds to the prior adjustments made
1834 * to the associated reservation map.
1835 * 2) Free any unused surplus pages that may have been allocated to satisfy
1836 * the reservation. As many as unused_resv_pages may be freed.
1838 * Called with hugetlb_lock held. However, the lock could be dropped (and
1839 * reacquired) during calls to cond_resched_lock. Whenever dropping the lock,
1840 * we must make sure nobody else can claim pages we are in the process of
1841 * freeing. Do this by ensuring resv_huge_page always is greater than the
1842 * number of huge pages we plan to free when dropping the lock.
1844 static void return_unused_surplus_pages(struct hstate *h,
1845 unsigned long unused_resv_pages)
1847 unsigned long nr_pages;
1849 /* Cannot return gigantic pages currently */
1850 if (hstate_is_gigantic(h))
1854 * Part (or even all) of the reservation could have been backed
1855 * by pre-allocated pages. Only free surplus pages.
1857 nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1860 * We want to release as many surplus pages as possible, spread
1861 * evenly across all nodes with memory. Iterate across these nodes
1862 * until we can no longer free unreserved surplus pages. This occurs
1863 * when the nodes with surplus pages have no free pages.
1864 * free_pool_huge_page() will balance the the freed pages across the
1865 * on-line nodes with memory and will handle the hstate accounting.
1867 * Note that we decrement resv_huge_pages as we free the pages. If
1868 * we drop the lock, resv_huge_pages will still be sufficiently large
1869 * to cover subsequent pages we may free.
1871 while (nr_pages--) {
1872 h->resv_huge_pages--;
1873 unused_resv_pages--;
1874 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1876 cond_resched_lock(&hugetlb_lock);
1880 /* Fully uncommit the reservation */
1881 h->resv_huge_pages -= unused_resv_pages;
1886 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1887 * are used by the huge page allocation routines to manage reservations.
1889 * vma_needs_reservation is called to determine if the huge page at addr
1890 * within the vma has an associated reservation. If a reservation is
1891 * needed, the value 1 is returned. The caller is then responsible for
1892 * managing the global reservation and subpool usage counts. After
1893 * the huge page has been allocated, vma_commit_reservation is called
1894 * to add the page to the reservation map. If the page allocation fails,
1895 * the reservation must be ended instead of committed. vma_end_reservation
1896 * is called in such cases.
1898 * In the normal case, vma_commit_reservation returns the same value
1899 * as the preceding vma_needs_reservation call. The only time this
1900 * is not the case is if a reserve map was changed between calls. It
1901 * is the responsibility of the caller to notice the difference and
1902 * take appropriate action.
1904 * vma_add_reservation is used in error paths where a reservation must
1905 * be restored when a newly allocated huge page must be freed. It is
1906 * to be called after calling vma_needs_reservation to determine if a
1907 * reservation exists.
1909 enum vma_resv_mode {
1915 static long __vma_reservation_common(struct hstate *h,
1916 struct vm_area_struct *vma, unsigned long addr,
1917 enum vma_resv_mode mode)
1919 struct resv_map *resv;
1923 resv = vma_resv_map(vma);
1927 idx = vma_hugecache_offset(h, vma, addr);
1929 case VMA_NEEDS_RESV:
1930 ret = region_chg(resv, idx, idx + 1);
1932 case VMA_COMMIT_RESV:
1933 ret = region_add(resv, idx, idx + 1);
1936 region_abort(resv, idx, idx + 1);
1940 if (vma->vm_flags & VM_MAYSHARE)
1941 ret = region_add(resv, idx, idx + 1);
1943 region_abort(resv, idx, idx + 1);
1944 ret = region_del(resv, idx, idx + 1);
1951 if (vma->vm_flags & VM_MAYSHARE)
1953 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1955 * In most cases, reserves always exist for private mappings.
1956 * However, a file associated with mapping could have been
1957 * hole punched or truncated after reserves were consumed.
1958 * As subsequent fault on such a range will not use reserves.
1959 * Subtle - The reserve map for private mappings has the
1960 * opposite meaning than that of shared mappings. If NO
1961 * entry is in the reserve map, it means a reservation exists.
1962 * If an entry exists in the reserve map, it means the
1963 * reservation has already been consumed. As a result, the
1964 * return value of this routine is the opposite of the
1965 * value returned from reserve map manipulation routines above.
1973 return ret < 0 ? ret : 0;
1976 static long vma_needs_reservation(struct hstate *h,
1977 struct vm_area_struct *vma, unsigned long addr)
1979 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1982 static long vma_commit_reservation(struct hstate *h,
1983 struct vm_area_struct *vma, unsigned long addr)
1985 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1988 static void vma_end_reservation(struct hstate *h,
1989 struct vm_area_struct *vma, unsigned long addr)
1991 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1994 static long vma_add_reservation(struct hstate *h,
1995 struct vm_area_struct *vma, unsigned long addr)
1997 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2001 * This routine is called to restore a reservation on error paths. In the
2002 * specific error paths, a huge page was allocated (via alloc_huge_page)
2003 * and is about to be freed. If a reservation for the page existed,
2004 * alloc_huge_page would have consumed the reservation and set PagePrivate
2005 * in the newly allocated page. When the page is freed via free_huge_page,
2006 * the global reservation count will be incremented if PagePrivate is set.
2007 * However, free_huge_page can not adjust the reserve map. Adjust the
2008 * reserve map here to be consistent with global reserve count adjustments
2009 * to be made by free_huge_page.
2011 static void restore_reserve_on_error(struct hstate *h,
2012 struct vm_area_struct *vma, unsigned long address,
2015 if (unlikely(PagePrivate(page))) {
2016 long rc = vma_needs_reservation(h, vma, address);
2018 if (unlikely(rc < 0)) {
2020 * Rare out of memory condition in reserve map
2021 * manipulation. Clear PagePrivate so that
2022 * global reserve count will not be incremented
2023 * by free_huge_page. This will make it appear
2024 * as though the reservation for this page was
2025 * consumed. This may prevent the task from
2026 * faulting in the page at a later time. This
2027 * is better than inconsistent global huge page
2028 * accounting of reserve counts.
2030 ClearPagePrivate(page);
2032 rc = vma_add_reservation(h, vma, address);
2033 if (unlikely(rc < 0))
2035 * See above comment about rare out of
2038 ClearPagePrivate(page);
2040 vma_end_reservation(h, vma, address);
2044 struct page *alloc_huge_page(struct vm_area_struct *vma,
2045 unsigned long addr, int avoid_reserve)
2047 struct hugepage_subpool *spool = subpool_vma(vma);
2048 struct hstate *h = hstate_vma(vma);
2050 long map_chg, map_commit;
2053 struct hugetlb_cgroup *h_cg;
2055 idx = hstate_index(h);
2057 * Examine the region/reserve map to determine if the process
2058 * has a reservation for the page to be allocated. A return
2059 * code of zero indicates a reservation exists (no change).
2061 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2063 return ERR_PTR(-ENOMEM);
2066 * Processes that did not create the mapping will have no
2067 * reserves as indicated by the region/reserve map. Check
2068 * that the allocation will not exceed the subpool limit.
2069 * Allocations for MAP_NORESERVE mappings also need to be
2070 * checked against any subpool limit.
2072 if (map_chg || avoid_reserve) {
2073 gbl_chg = hugepage_subpool_get_pages(spool, 1);
2075 vma_end_reservation(h, vma, addr);
2076 return ERR_PTR(-ENOSPC);
2080 * Even though there was no reservation in the region/reserve
2081 * map, there could be reservations associated with the
2082 * subpool that can be used. This would be indicated if the
2083 * return value of hugepage_subpool_get_pages() is zero.
2084 * However, if avoid_reserve is specified we still avoid even
2085 * the subpool reservations.
2091 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2093 goto out_subpool_put;
2095 spin_lock(&hugetlb_lock);
2097 * glb_chg is passed to indicate whether or not a page must be taken
2098 * from the global free pool (global change). gbl_chg == 0 indicates
2099 * a reservation exists for the allocation.
2101 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2103 spin_unlock(&hugetlb_lock);
2104 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
2106 goto out_uncharge_cgroup;
2107 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2108 SetPagePrivate(page);
2109 h->resv_huge_pages--;
2111 spin_lock(&hugetlb_lock);
2112 list_move(&page->lru, &h->hugepage_activelist);
2115 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2116 spin_unlock(&hugetlb_lock);
2118 set_page_private(page, (unsigned long)spool);
2120 map_commit = vma_commit_reservation(h, vma, addr);
2121 if (unlikely(map_chg > map_commit)) {
2123 * The page was added to the reservation map between
2124 * vma_needs_reservation and vma_commit_reservation.
2125 * This indicates a race with hugetlb_reserve_pages.
2126 * Adjust for the subpool count incremented above AND
2127 * in hugetlb_reserve_pages for the same page. Also,
2128 * the reservation count added in hugetlb_reserve_pages
2129 * no longer applies.
2133 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2134 hugetlb_acct_memory(h, -rsv_adjust);
2138 out_uncharge_cgroup:
2139 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2141 if (map_chg || avoid_reserve)
2142 hugepage_subpool_put_pages(spool, 1);
2143 vma_end_reservation(h, vma, addr);
2144 return ERR_PTR(-ENOSPC);
2148 * alloc_huge_page()'s wrapper which simply returns the page if allocation
2149 * succeeds, otherwise NULL. This function is called from new_vma_page(),
2150 * where no ERR_VALUE is expected to be returned.
2152 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
2153 unsigned long addr, int avoid_reserve)
2155 struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
2161 int __weak alloc_bootmem_huge_page(struct hstate *h)
2163 struct huge_bootmem_page *m;
2166 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2169 addr = memblock_virt_alloc_try_nid_nopanic(
2170 huge_page_size(h), huge_page_size(h),
2171 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2174 * Use the beginning of the huge page to store the
2175 * huge_bootmem_page struct (until gather_bootmem
2176 * puts them into the mem_map).
2185 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2186 /* Put them into a private list first because mem_map is not up yet */
2187 list_add(&m->list, &huge_boot_pages);
2192 static void __init prep_compound_huge_page(struct page *page,
2195 if (unlikely(order > (MAX_ORDER - 1)))
2196 prep_compound_gigantic_page(page, order);
2198 prep_compound_page(page, order);
2201 /* Put bootmem huge pages into the standard lists after mem_map is up */
2202 static void __init gather_bootmem_prealloc(void)
2204 struct huge_bootmem_page *m;
2206 list_for_each_entry(m, &huge_boot_pages, list) {
2207 struct hstate *h = m->hstate;
2210 #ifdef CONFIG_HIGHMEM
2211 page = pfn_to_page(m->phys >> PAGE_SHIFT);
2212 memblock_free_late(__pa(m),
2213 sizeof(struct huge_bootmem_page));
2215 page = virt_to_page(m);
2217 WARN_ON(page_count(page) != 1);
2218 prep_compound_huge_page(page, h->order);
2219 WARN_ON(PageReserved(page));
2220 prep_new_huge_page(h, page, page_to_nid(page));
2222 * If we had gigantic hugepages allocated at boot time, we need
2223 * to restore the 'stolen' pages to totalram_pages in order to
2224 * fix confusing memory reports from free(1) and another
2225 * side-effects, like CommitLimit going negative.
2227 if (hstate_is_gigantic(h))
2228 adjust_managed_page_count(page, 1 << h->order);
2233 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2237 for (i = 0; i < h->max_huge_pages; ++i) {
2238 if (hstate_is_gigantic(h)) {
2239 if (!alloc_bootmem_huge_page(h))
2241 } else if (!alloc_fresh_huge_page(h,
2242 &node_states[N_MEMORY]))
2245 h->max_huge_pages = i;
2248 static void __init hugetlb_init_hstates(void)
2252 for_each_hstate(h) {
2253 if (minimum_order > huge_page_order(h))
2254 minimum_order = huge_page_order(h);
2256 /* oversize hugepages were init'ed in early boot */
2257 if (!hstate_is_gigantic(h))
2258 hugetlb_hstate_alloc_pages(h);
2260 VM_BUG_ON(minimum_order == UINT_MAX);
2263 static char * __init memfmt(char *buf, unsigned long n)
2265 if (n >= (1UL << 30))
2266 sprintf(buf, "%lu GB", n >> 30);
2267 else if (n >= (1UL << 20))
2268 sprintf(buf, "%lu MB", n >> 20);
2270 sprintf(buf, "%lu KB", n >> 10);
2274 static void __init report_hugepages(void)
2278 for_each_hstate(h) {
2280 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2281 memfmt(buf, huge_page_size(h)),
2282 h->free_huge_pages);
2286 #ifdef CONFIG_HIGHMEM
2287 static void try_to_free_low(struct hstate *h, unsigned long count,
2288 nodemask_t *nodes_allowed)
2292 if (hstate_is_gigantic(h))
2295 for_each_node_mask(i, *nodes_allowed) {
2296 struct page *page, *next;
2297 struct list_head *freel = &h->hugepage_freelists[i];
2298 list_for_each_entry_safe(page, next, freel, lru) {
2299 if (count >= h->nr_huge_pages)
2301 if (PageHighMem(page))
2303 list_del(&page->lru);
2304 update_and_free_page(h, page);
2305 h->free_huge_pages--;
2306 h->free_huge_pages_node[page_to_nid(page)]--;
2311 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2312 nodemask_t *nodes_allowed)
2318 * Increment or decrement surplus_huge_pages. Keep node-specific counters
2319 * balanced by operating on them in a round-robin fashion.
2320 * Returns 1 if an adjustment was made.
2322 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2327 VM_BUG_ON(delta != -1 && delta != 1);
2330 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2331 if (h->surplus_huge_pages_node[node])
2335 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2336 if (h->surplus_huge_pages_node[node] <
2337 h->nr_huge_pages_node[node])
2344 h->surplus_huge_pages += delta;
2345 h->surplus_huge_pages_node[node] += delta;
2349 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2350 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2351 nodemask_t *nodes_allowed)
2353 unsigned long min_count, ret;
2355 if (hstate_is_gigantic(h) && !gigantic_page_supported())
2356 return h->max_huge_pages;
2359 * Increase the pool size
2360 * First take pages out of surplus state. Then make up the
2361 * remaining difference by allocating fresh huge pages.
2363 * We might race with __alloc_buddy_huge_page() here and be unable
2364 * to convert a surplus huge page to a normal huge page. That is
2365 * not critical, though, it just means the overall size of the
2366 * pool might be one hugepage larger than it needs to be, but
2367 * within all the constraints specified by the sysctls.
2369 spin_lock(&hugetlb_lock);
2370 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2371 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2375 while (count > persistent_huge_pages(h)) {
2377 * If this allocation races such that we no longer need the
2378 * page, free_huge_page will handle it by freeing the page
2379 * and reducing the surplus.
2381 spin_unlock(&hugetlb_lock);
2383 /* yield cpu to avoid soft lockup */
2386 if (hstate_is_gigantic(h))
2387 ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2389 ret = alloc_fresh_huge_page(h, nodes_allowed);
2390 spin_lock(&hugetlb_lock);
2394 /* Bail for signals. Probably ctrl-c from user */
2395 if (signal_pending(current))
2400 * Decrease the pool size
2401 * First return free pages to the buddy allocator (being careful
2402 * to keep enough around to satisfy reservations). Then place
2403 * pages into surplus state as needed so the pool will shrink
2404 * to the desired size as pages become free.
2406 * By placing pages into the surplus state independent of the
2407 * overcommit value, we are allowing the surplus pool size to
2408 * exceed overcommit. There are few sane options here. Since
2409 * __alloc_buddy_huge_page() is checking the global counter,
2410 * though, we'll note that we're not allowed to exceed surplus
2411 * and won't grow the pool anywhere else. Not until one of the
2412 * sysctls are changed, or the surplus pages go out of use.
2414 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2415 min_count = max(count, min_count);
2416 try_to_free_low(h, min_count, nodes_allowed);
2417 while (min_count < persistent_huge_pages(h)) {
2418 if (!free_pool_huge_page(h, nodes_allowed, 0))
2420 cond_resched_lock(&hugetlb_lock);
2422 while (count < persistent_huge_pages(h)) {
2423 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2427 ret = persistent_huge_pages(h);
2428 spin_unlock(&hugetlb_lock);
2432 #define HSTATE_ATTR_RO(_name) \
2433 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2435 #define HSTATE_ATTR(_name) \
2436 static struct kobj_attribute _name##_attr = \
2437 __ATTR(_name, 0644, _name##_show, _name##_store)
2439 static struct kobject *hugepages_kobj;
2440 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2442 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2444 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2448 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2449 if (hstate_kobjs[i] == kobj) {
2451 *nidp = NUMA_NO_NODE;
2455 return kobj_to_node_hstate(kobj, nidp);
2458 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2459 struct kobj_attribute *attr, char *buf)
2462 unsigned long nr_huge_pages;
2465 h = kobj_to_hstate(kobj, &nid);
2466 if (nid == NUMA_NO_NODE)
2467 nr_huge_pages = h->nr_huge_pages;
2469 nr_huge_pages = h->nr_huge_pages_node[nid];
2471 return sprintf(buf, "%lu\n", nr_huge_pages);
2474 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2475 struct hstate *h, int nid,
2476 unsigned long count, size_t len)
2479 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2481 if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2486 if (nid == NUMA_NO_NODE) {
2488 * global hstate attribute
2490 if (!(obey_mempolicy &&
2491 init_nodemask_of_mempolicy(nodes_allowed))) {
2492 NODEMASK_FREE(nodes_allowed);
2493 nodes_allowed = &node_states[N_MEMORY];
2495 } else if (nodes_allowed) {
2497 * per node hstate attribute: adjust count to global,
2498 * but restrict alloc/free to the specified node.
2500 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2501 init_nodemask_of_node(nodes_allowed, nid);
2503 nodes_allowed = &node_states[N_MEMORY];
2505 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2507 if (nodes_allowed != &node_states[N_MEMORY])
2508 NODEMASK_FREE(nodes_allowed);
2512 NODEMASK_FREE(nodes_allowed);
2516 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2517 struct kobject *kobj, const char *buf,
2521 unsigned long count;
2525 err = kstrtoul(buf, 10, &count);
2529 h = kobj_to_hstate(kobj, &nid);
2530 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2533 static ssize_t nr_hugepages_show(struct kobject *kobj,
2534 struct kobj_attribute *attr, char *buf)
2536 return nr_hugepages_show_common(kobj, attr, buf);
2539 static ssize_t nr_hugepages_store(struct kobject *kobj,
2540 struct kobj_attribute *attr, const char *buf, size_t len)
2542 return nr_hugepages_store_common(false, kobj, buf, len);
2544 HSTATE_ATTR(nr_hugepages);
2549 * hstate attribute for optionally mempolicy-based constraint on persistent
2550 * huge page alloc/free.
2552 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2553 struct kobj_attribute *attr, char *buf)
2555 return nr_hugepages_show_common(kobj, attr, buf);
2558 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2559 struct kobj_attribute *attr, const char *buf, size_t len)
2561 return nr_hugepages_store_common(true, kobj, buf, len);
2563 HSTATE_ATTR(nr_hugepages_mempolicy);
2567 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2568 struct kobj_attribute *attr, char *buf)
2570 struct hstate *h = kobj_to_hstate(kobj, NULL);
2571 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2574 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2575 struct kobj_attribute *attr, const char *buf, size_t count)
2578 unsigned long input;
2579 struct hstate *h = kobj_to_hstate(kobj, NULL);
2581 if (hstate_is_gigantic(h))
2584 err = kstrtoul(buf, 10, &input);
2588 spin_lock(&hugetlb_lock);
2589 h->nr_overcommit_huge_pages = input;
2590 spin_unlock(&hugetlb_lock);
2594 HSTATE_ATTR(nr_overcommit_hugepages);
2596 static ssize_t free_hugepages_show(struct kobject *kobj,
2597 struct kobj_attribute *attr, char *buf)
2600 unsigned long free_huge_pages;
2603 h = kobj_to_hstate(kobj, &nid);
2604 if (nid == NUMA_NO_NODE)
2605 free_huge_pages = h->free_huge_pages;
2607 free_huge_pages = h->free_huge_pages_node[nid];
2609 return sprintf(buf, "%lu\n", free_huge_pages);
2611 HSTATE_ATTR_RO(free_hugepages);
2613 static ssize_t resv_hugepages_show(struct kobject *kobj,
2614 struct kobj_attribute *attr, char *buf)
2616 struct hstate *h = kobj_to_hstate(kobj, NULL);
2617 return sprintf(buf, "%lu\n", h->resv_huge_pages);
2619 HSTATE_ATTR_RO(resv_hugepages);
2621 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2622 struct kobj_attribute *attr, char *buf)
2625 unsigned long surplus_huge_pages;
2628 h = kobj_to_hstate(kobj, &nid);
2629 if (nid == NUMA_NO_NODE)
2630 surplus_huge_pages = h->surplus_huge_pages;
2632 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2634 return sprintf(buf, "%lu\n", surplus_huge_pages);
2636 HSTATE_ATTR_RO(surplus_hugepages);
2638 static struct attribute *hstate_attrs[] = {
2639 &nr_hugepages_attr.attr,
2640 &nr_overcommit_hugepages_attr.attr,
2641 &free_hugepages_attr.attr,
2642 &resv_hugepages_attr.attr,
2643 &surplus_hugepages_attr.attr,
2645 &nr_hugepages_mempolicy_attr.attr,
2650 static struct attribute_group hstate_attr_group = {
2651 .attrs = hstate_attrs,
2654 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2655 struct kobject **hstate_kobjs,
2656 struct attribute_group *hstate_attr_group)
2659 int hi = hstate_index(h);
2661 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2662 if (!hstate_kobjs[hi])
2665 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2667 kobject_put(hstate_kobjs[hi]);
2668 hstate_kobjs[hi] = NULL;
2674 static void __init hugetlb_sysfs_init(void)
2679 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2680 if (!hugepages_kobj)
2683 for_each_hstate(h) {
2684 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2685 hstate_kobjs, &hstate_attr_group);
2687 pr_err("Hugetlb: Unable to add hstate %s", h->name);
2694 * node_hstate/s - associate per node hstate attributes, via their kobjects,
2695 * with node devices in node_devices[] using a parallel array. The array
2696 * index of a node device or _hstate == node id.
2697 * This is here to avoid any static dependency of the node device driver, in
2698 * the base kernel, on the hugetlb module.
2700 struct node_hstate {
2701 struct kobject *hugepages_kobj;
2702 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2704 static struct node_hstate node_hstates[MAX_NUMNODES];
2707 * A subset of global hstate attributes for node devices
2709 static struct attribute *per_node_hstate_attrs[] = {
2710 &nr_hugepages_attr.attr,
2711 &free_hugepages_attr.attr,
2712 &surplus_hugepages_attr.attr,
2716 static struct attribute_group per_node_hstate_attr_group = {
2717 .attrs = per_node_hstate_attrs,
2721 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2722 * Returns node id via non-NULL nidp.
2724 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2728 for (nid = 0; nid < nr_node_ids; nid++) {
2729 struct node_hstate *nhs = &node_hstates[nid];
2731 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2732 if (nhs->hstate_kobjs[i] == kobj) {
2744 * Unregister hstate attributes from a single node device.
2745 * No-op if no hstate attributes attached.
2747 static void hugetlb_unregister_node(struct node *node)
2750 struct node_hstate *nhs = &node_hstates[node->dev.id];
2752 if (!nhs->hugepages_kobj)
2753 return; /* no hstate attributes */
2755 for_each_hstate(h) {
2756 int idx = hstate_index(h);
2757 if (nhs->hstate_kobjs[idx]) {
2758 kobject_put(nhs->hstate_kobjs[idx]);
2759 nhs->hstate_kobjs[idx] = NULL;
2763 kobject_put(nhs->hugepages_kobj);
2764 nhs->hugepages_kobj = NULL;
2769 * Register hstate attributes for a single node device.
2770 * No-op if attributes already registered.
2772 static void hugetlb_register_node(struct node *node)
2775 struct node_hstate *nhs = &node_hstates[node->dev.id];
2778 if (nhs->hugepages_kobj)
2779 return; /* already allocated */
2781 nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2783 if (!nhs->hugepages_kobj)
2786 for_each_hstate(h) {
2787 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2789 &per_node_hstate_attr_group);
2791 pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2792 h->name, node->dev.id);
2793 hugetlb_unregister_node(node);
2800 * hugetlb init time: register hstate attributes for all registered node
2801 * devices of nodes that have memory. All on-line nodes should have
2802 * registered their associated device by this time.
2804 static void __init hugetlb_register_all_nodes(void)
2808 for_each_node_state(nid, N_MEMORY) {
2809 struct node *node = node_devices[nid];
2810 if (node->dev.id == nid)
2811 hugetlb_register_node(node);
2815 * Let the node device driver know we're here so it can
2816 * [un]register hstate attributes on node hotplug.
2818 register_hugetlbfs_with_node(hugetlb_register_node,
2819 hugetlb_unregister_node);
2821 #else /* !CONFIG_NUMA */
2823 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2831 static void hugetlb_register_all_nodes(void) { }
2835 static int __init hugetlb_init(void)
2839 if (!hugepages_supported())
2842 if (!size_to_hstate(default_hstate_size)) {
2843 default_hstate_size = HPAGE_SIZE;
2844 if (!size_to_hstate(default_hstate_size))
2845 hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2847 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2848 if (default_hstate_max_huge_pages) {
2849 if (!default_hstate.max_huge_pages)
2850 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2853 hugetlb_init_hstates();
2854 gather_bootmem_prealloc();
2857 hugetlb_sysfs_init();
2858 hugetlb_register_all_nodes();
2859 hugetlb_cgroup_file_init();
2862 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2864 num_fault_mutexes = 1;
2866 hugetlb_fault_mutex_table =
2867 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2868 BUG_ON(!hugetlb_fault_mutex_table);
2870 for (i = 0; i < num_fault_mutexes; i++)
2871 mutex_init(&hugetlb_fault_mutex_table[i]);
2874 subsys_initcall(hugetlb_init);
2876 /* Should be called on processing a hugepagesz=... option */
2877 void __init hugetlb_bad_size(void)
2879 parsed_valid_hugepagesz = false;
2882 void __init hugetlb_add_hstate(unsigned int order)
2887 if (size_to_hstate(PAGE_SIZE << order)) {
2888 pr_warn("hugepagesz= specified twice, ignoring\n");
2891 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2893 h = &hstates[hugetlb_max_hstate++];
2895 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2896 h->nr_huge_pages = 0;
2897 h->free_huge_pages = 0;
2898 for (i = 0; i < MAX_NUMNODES; ++i)
2899 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2900 INIT_LIST_HEAD(&h->hugepage_activelist);
2901 h->next_nid_to_alloc = first_memory_node;
2902 h->next_nid_to_free = first_memory_node;
2903 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2904 huge_page_size(h)/1024);
2909 static int __init hugetlb_nrpages_setup(char *s)
2912 static unsigned long *last_mhp;
2914 if (!parsed_valid_hugepagesz) {
2915 pr_warn("hugepages = %s preceded by "
2916 "an unsupported hugepagesz, ignoring\n", s);
2917 parsed_valid_hugepagesz = true;
2921 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2922 * so this hugepages= parameter goes to the "default hstate".
2924 else if (!hugetlb_max_hstate)
2925 mhp = &default_hstate_max_huge_pages;
2927 mhp = &parsed_hstate->max_huge_pages;
2929 if (mhp == last_mhp) {
2930 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2934 if (sscanf(s, "%lu", mhp) <= 0)
2938 * Global state is always initialized later in hugetlb_init.
2939 * But we need to allocate >= MAX_ORDER hstates here early to still
2940 * use the bootmem allocator.
2942 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2943 hugetlb_hstate_alloc_pages(parsed_hstate);
2949 __setup("hugepages=", hugetlb_nrpages_setup);
2951 static int __init hugetlb_default_setup(char *s)
2953 default_hstate_size = memparse(s, &s);
2956 __setup("default_hugepagesz=", hugetlb_default_setup);
2958 static unsigned int cpuset_mems_nr(unsigned int *array)
2961 unsigned int nr = 0;
2963 for_each_node_mask(node, cpuset_current_mems_allowed)
2969 #ifdef CONFIG_SYSCTL
2970 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
2971 void *buffer, size_t *length,
2972 loff_t *ppos, unsigned long *out)
2974 struct ctl_table dup_table;
2977 * In order to avoid races with __do_proc_doulongvec_minmax(), we
2978 * can duplicate the @table and alter the duplicate of it.
2981 dup_table.data = out;
2983 return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
2986 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2987 struct ctl_table *table, int write,
2988 void __user *buffer, size_t *length, loff_t *ppos)
2990 struct hstate *h = &default_hstate;
2991 unsigned long tmp = h->max_huge_pages;
2994 if (!hugepages_supported())
2997 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3003 ret = __nr_hugepages_store_common(obey_mempolicy, h,
3004 NUMA_NO_NODE, tmp, *length);
3009 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
3010 void __user *buffer, size_t *length, loff_t *ppos)
3013 return hugetlb_sysctl_handler_common(false, table, write,
3014 buffer, length, ppos);
3018 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
3019 void __user *buffer, size_t *length, loff_t *ppos)
3021 return hugetlb_sysctl_handler_common(true, table, write,
3022 buffer, length, ppos);
3024 #endif /* CONFIG_NUMA */
3026 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
3027 void __user *buffer,
3028 size_t *length, loff_t *ppos)
3030 struct hstate *h = &default_hstate;
3034 if (!hugepages_supported())
3037 tmp = h->nr_overcommit_huge_pages;
3039 if (write && hstate_is_gigantic(h))
3042 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3048 spin_lock(&hugetlb_lock);
3049 h->nr_overcommit_huge_pages = tmp;
3050 spin_unlock(&hugetlb_lock);
3056 #endif /* CONFIG_SYSCTL */
3058 void hugetlb_report_meminfo(struct seq_file *m)
3060 struct hstate *h = &default_hstate;
3061 if (!hugepages_supported())
3064 "HugePages_Total: %5lu\n"
3065 "HugePages_Free: %5lu\n"
3066 "HugePages_Rsvd: %5lu\n"
3067 "HugePages_Surp: %5lu\n"
3068 "Hugepagesize: %8lu kB\n",
3072 h->surplus_huge_pages,
3073 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3076 int hugetlb_report_node_meminfo(int nid, char *buf)
3078 struct hstate *h = &default_hstate;
3079 if (!hugepages_supported())
3082 "Node %d HugePages_Total: %5u\n"
3083 "Node %d HugePages_Free: %5u\n"
3084 "Node %d HugePages_Surp: %5u\n",
3085 nid, h->nr_huge_pages_node[nid],
3086 nid, h->free_huge_pages_node[nid],
3087 nid, h->surplus_huge_pages_node[nid]);
3090 void hugetlb_show_meminfo(void)
3095 if (!hugepages_supported())
3098 for_each_node_state(nid, N_MEMORY)
3100 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3102 h->nr_huge_pages_node[nid],
3103 h->free_huge_pages_node[nid],
3104 h->surplus_huge_pages_node[nid],
3105 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3108 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3110 seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3111 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3114 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3115 unsigned long hugetlb_total_pages(void)
3118 unsigned long nr_total_pages = 0;
3121 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3122 return nr_total_pages;
3125 static int hugetlb_acct_memory(struct hstate *h, long delta)
3129 spin_lock(&hugetlb_lock);
3131 * When cpuset is configured, it breaks the strict hugetlb page
3132 * reservation as the accounting is done on a global variable. Such
3133 * reservation is completely rubbish in the presence of cpuset because
3134 * the reservation is not checked against page availability for the
3135 * current cpuset. Application can still potentially OOM'ed by kernel
3136 * with lack of free htlb page in cpuset that the task is in.
3137 * Attempt to enforce strict accounting with cpuset is almost
3138 * impossible (or too ugly) because cpuset is too fluid that
3139 * task or memory node can be dynamically moved between cpusets.
3141 * The change of semantics for shared hugetlb mapping with cpuset is
3142 * undesirable. However, in order to preserve some of the semantics,
3143 * we fall back to check against current free page availability as
3144 * a best attempt and hopefully to minimize the impact of changing
3145 * semantics that cpuset has.
3148 if (gather_surplus_pages(h, delta) < 0)
3151 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
3152 return_unused_surplus_pages(h, delta);
3159 return_unused_surplus_pages(h, (unsigned long) -delta);
3162 spin_unlock(&hugetlb_lock);
3166 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3168 struct resv_map *resv = vma_resv_map(vma);
3171 * This new VMA should share its siblings reservation map if present.
3172 * The VMA will only ever have a valid reservation map pointer where
3173 * it is being copied for another still existing VMA. As that VMA
3174 * has a reference to the reservation map it cannot disappear until
3175 * after this open call completes. It is therefore safe to take a
3176 * new reference here without additional locking.
3178 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3179 kref_get(&resv->refs);
3182 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3184 struct hstate *h = hstate_vma(vma);
3185 struct resv_map *resv = vma_resv_map(vma);
3186 struct hugepage_subpool *spool = subpool_vma(vma);
3187 unsigned long reserve, start, end;
3190 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3193 start = vma_hugecache_offset(h, vma, vma->vm_start);
3194 end = vma_hugecache_offset(h, vma, vma->vm_end);
3196 reserve = (end - start) - region_count(resv, start, end);
3198 kref_put(&resv->refs, resv_map_release);
3202 * Decrement reserve counts. The global reserve count may be
3203 * adjusted if the subpool has a minimum size.
3205 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3206 hugetlb_acct_memory(h, -gbl_reserve);
3210 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
3212 if (addr & ~(huge_page_mask(hstate_vma(vma))))
3218 * We cannot handle pagefaults against hugetlb pages at all. They cause
3219 * handle_mm_fault() to try to instantiate regular-sized pages in the
3220 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
3223 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3229 const struct vm_operations_struct hugetlb_vm_ops = {
3230 .fault = hugetlb_vm_op_fault,
3231 .open = hugetlb_vm_op_open,
3232 .close = hugetlb_vm_op_close,
3233 .split = hugetlb_vm_op_split,
3236 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3242 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3243 vma->vm_page_prot)));
3245 entry = huge_pte_wrprotect(mk_huge_pte(page,
3246 vma->vm_page_prot));
3248 entry = pte_mkyoung(entry);
3249 entry = pte_mkhuge(entry);
3250 entry = arch_make_huge_pte(entry, vma, page, writable);
3255 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3256 unsigned long address, pte_t *ptep)
3260 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3261 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3262 update_mmu_cache(vma, address, ptep);
3265 static int is_hugetlb_entry_migration(pte_t pte)
3269 if (huge_pte_none(pte) || pte_present(pte))
3271 swp = pte_to_swp_entry(pte);
3272 if (non_swap_entry(swp) && is_migration_entry(swp))
3278 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3282 if (huge_pte_none(pte) || pte_present(pte))
3284 swp = pte_to_swp_entry(pte);
3285 if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3291 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3292 struct vm_area_struct *vma)
3294 pte_t *src_pte, *dst_pte, entry, dst_entry;
3295 struct page *ptepage;
3298 struct hstate *h = hstate_vma(vma);
3299 unsigned long sz = huge_page_size(h);
3300 unsigned long mmun_start; /* For mmu_notifiers */
3301 unsigned long mmun_end; /* For mmu_notifiers */
3304 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3306 mmun_start = vma->vm_start;
3307 mmun_end = vma->vm_end;
3309 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3311 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3312 spinlock_t *src_ptl, *dst_ptl;
3313 src_pte = huge_pte_offset(src, addr);
3316 dst_pte = huge_pte_alloc(dst, addr, sz);
3323 * If the pagetables are shared don't copy or take references.
3324 * dst_pte == src_pte is the common case of src/dest sharing.
3326 * However, src could have 'unshared' and dst shares with
3327 * another vma. If dst_pte !none, this implies sharing.
3328 * Check here before taking page table lock, and once again
3329 * after taking the lock below.
3331 dst_entry = huge_ptep_get(dst_pte);
3332 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
3335 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3336 src_ptl = huge_pte_lockptr(h, src, src_pte);
3337 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3338 entry = huge_ptep_get(src_pte);
3339 dst_entry = huge_ptep_get(dst_pte);
3340 if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
3342 * Skip if src entry none. Also, skip in the
3343 * unlikely case dst entry !none as this implies
3344 * sharing with another vma.
3347 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3348 is_hugetlb_entry_hwpoisoned(entry))) {
3349 swp_entry_t swp_entry = pte_to_swp_entry(entry);
3351 if (is_write_migration_entry(swp_entry) && cow) {
3353 * COW mappings require pages in both
3354 * parent and child to be set to read.
3356 make_migration_entry_read(&swp_entry);
3357 entry = swp_entry_to_pte(swp_entry);
3358 set_huge_pte_at(src, addr, src_pte, entry);
3360 set_huge_pte_at(dst, addr, dst_pte, entry);
3363 huge_ptep_set_wrprotect(src, addr, src_pte);
3364 mmu_notifier_invalidate_range(src, mmun_start,
3367 entry = huge_ptep_get(src_pte);
3368 ptepage = pte_page(entry);
3370 page_dup_rmap(ptepage, true);
3371 set_huge_pte_at(dst, addr, dst_pte, entry);
3372 hugetlb_count_add(pages_per_huge_page(h), dst);
3374 spin_unlock(src_ptl);
3375 spin_unlock(dst_ptl);
3379 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3384 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3385 unsigned long start, unsigned long end,
3386 struct page *ref_page)
3388 struct mm_struct *mm = vma->vm_mm;
3389 unsigned long address;
3394 struct hstate *h = hstate_vma(vma);
3395 unsigned long sz = huge_page_size(h);
3396 const unsigned long mmun_start = start; /* For mmu_notifiers */
3397 const unsigned long mmun_end = end; /* For mmu_notifiers */
3399 WARN_ON(!is_vm_hugetlb_page(vma));
3400 BUG_ON(start & ~huge_page_mask(h));
3401 BUG_ON(end & ~huge_page_mask(h));
3403 tlb_start_vma(tlb, vma);
3404 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3406 for (; address < end; address += sz) {
3407 ptep = huge_pte_offset(mm, address);
3411 ptl = huge_pte_lock(h, mm, ptep);
3412 if (huge_pmd_unshare(mm, &address, ptep)) {
3417 pte = huge_ptep_get(ptep);
3418 if (huge_pte_none(pte)) {
3424 * Migrating hugepage or HWPoisoned hugepage is already
3425 * unmapped and its refcount is dropped, so just clear pte here.
3427 if (unlikely(!pte_present(pte))) {
3428 huge_pte_clear(mm, address, ptep);
3433 page = pte_page(pte);
3435 * If a reference page is supplied, it is because a specific
3436 * page is being unmapped, not a range. Ensure the page we
3437 * are about to unmap is the actual page of interest.
3440 if (page != ref_page) {
3445 * Mark the VMA as having unmapped its page so that
3446 * future faults in this VMA will fail rather than
3447 * looking like data was lost
3449 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3452 pte = huge_ptep_get_and_clear(mm, address, ptep);
3453 tlb_remove_tlb_entry(tlb, ptep, address);
3454 if (huge_pte_dirty(pte))
3455 set_page_dirty(page);
3457 hugetlb_count_sub(pages_per_huge_page(h), mm);
3458 page_remove_rmap(page, true);
3461 tlb_remove_page_size(tlb, page, huge_page_size(h));
3463 * Bail out after unmapping reference page if supplied
3468 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3469 tlb_end_vma(tlb, vma);
3472 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3473 struct vm_area_struct *vma, unsigned long start,
3474 unsigned long end, struct page *ref_page)
3476 __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3479 * Clear this flag so that x86's huge_pmd_share page_table_shareable
3480 * test will fail on a vma being torn down, and not grab a page table
3481 * on its way out. We're lucky that the flag has such an appropriate
3482 * name, and can in fact be safely cleared here. We could clear it
3483 * before the __unmap_hugepage_range above, but all that's necessary
3484 * is to clear it before releasing the i_mmap_rwsem. This works
3485 * because in the context this is called, the VMA is about to be
3486 * destroyed and the i_mmap_rwsem is held.
3488 vma->vm_flags &= ~VM_MAYSHARE;
3491 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3492 unsigned long end, struct page *ref_page)
3494 struct mm_struct *mm;
3495 struct mmu_gather tlb;
3499 tlb_gather_mmu(&tlb, mm, start, end);
3500 __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3501 tlb_finish_mmu(&tlb, start, end);
3505 * This is called when the original mapper is failing to COW a MAP_PRIVATE
3506 * mappping it owns the reserve page for. The intention is to unmap the page
3507 * from other VMAs and let the children be SIGKILLed if they are faulting the
3510 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3511 struct page *page, unsigned long address)
3513 struct hstate *h = hstate_vma(vma);
3514 struct vm_area_struct *iter_vma;
3515 struct address_space *mapping;
3519 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3520 * from page cache lookup which is in HPAGE_SIZE units.
3522 address = address & huge_page_mask(h);
3523 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3525 mapping = vma->vm_file->f_mapping;
3528 * Take the mapping lock for the duration of the table walk. As
3529 * this mapping should be shared between all the VMAs,
3530 * __unmap_hugepage_range() is called as the lock is already held
3532 i_mmap_lock_write(mapping);
3533 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3534 /* Do not unmap the current VMA */
3535 if (iter_vma == vma)
3539 * Shared VMAs have their own reserves and do not affect
3540 * MAP_PRIVATE accounting but it is possible that a shared
3541 * VMA is using the same page so check and skip such VMAs.
3543 if (iter_vma->vm_flags & VM_MAYSHARE)
3547 * Unmap the page from other VMAs without their own reserves.
3548 * They get marked to be SIGKILLed if they fault in these
3549 * areas. This is because a future no-page fault on this VMA
3550 * could insert a zeroed page instead of the data existing
3551 * from the time of fork. This would look like data corruption
3553 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3554 unmap_hugepage_range(iter_vma, address,
3555 address + huge_page_size(h), page);
3557 i_mmap_unlock_write(mapping);
3561 * Hugetlb_cow() should be called with page lock of the original hugepage held.
3562 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3563 * cannot race with other handlers or page migration.
3564 * Keep the pte_same checks anyway to make transition from the mutex easier.
3566 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3567 unsigned long address, pte_t *ptep,
3568 struct page *pagecache_page, spinlock_t *ptl)
3571 struct hstate *h = hstate_vma(vma);
3572 struct page *old_page, *new_page;
3573 int ret = 0, outside_reserve = 0;
3574 unsigned long mmun_start; /* For mmu_notifiers */
3575 unsigned long mmun_end; /* For mmu_notifiers */
3577 pte = huge_ptep_get(ptep);
3578 old_page = pte_page(pte);
3581 /* If no-one else is actually using this page, avoid the copy
3582 * and just make the page writable */
3583 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3584 page_move_anon_rmap(old_page, vma);
3585 set_huge_ptep_writable(vma, address, ptep);
3590 * If the process that created a MAP_PRIVATE mapping is about to
3591 * perform a COW due to a shared page count, attempt to satisfy
3592 * the allocation without using the existing reserves. The pagecache
3593 * page is used to determine if the reserve at this address was
3594 * consumed or not. If reserves were used, a partial faulted mapping
3595 * at the time of fork() could consume its reserves on COW instead
3596 * of the full address range.
3598 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3599 old_page != pagecache_page)
3600 outside_reserve = 1;
3605 * Drop page table lock as buddy allocator may be called. It will
3606 * be acquired again before returning to the caller, as expected.
3609 new_page = alloc_huge_page(vma, address, outside_reserve);
3611 if (IS_ERR(new_page)) {
3613 * If a process owning a MAP_PRIVATE mapping fails to COW,
3614 * it is due to references held by a child and an insufficient
3615 * huge page pool. To guarantee the original mappers
3616 * reliability, unmap the page from child processes. The child
3617 * may get SIGKILLed if it later faults.
3619 if (outside_reserve) {
3621 BUG_ON(huge_pte_none(pte));
3622 unmap_ref_private(mm, vma, old_page, address);
3623 BUG_ON(huge_pte_none(pte));
3625 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3627 pte_same(huge_ptep_get(ptep), pte)))
3628 goto retry_avoidcopy;
3630 * race occurs while re-acquiring page table
3631 * lock, and our job is done.
3636 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3637 VM_FAULT_OOM : VM_FAULT_SIGBUS;
3638 goto out_release_old;
3642 * When the original hugepage is shared one, it does not have
3643 * anon_vma prepared.
3645 if (unlikely(anon_vma_prepare(vma))) {
3647 goto out_release_all;
3650 copy_user_huge_page(new_page, old_page, address, vma,
3651 pages_per_huge_page(h));
3652 __SetPageUptodate(new_page);
3654 mmun_start = address & huge_page_mask(h);
3655 mmun_end = mmun_start + huge_page_size(h);
3656 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3659 * Retake the page table lock to check for racing updates
3660 * before the page tables are altered
3663 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3664 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3665 ClearPagePrivate(new_page);
3668 huge_ptep_clear_flush(vma, address, ptep);
3669 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3670 set_huge_pte_at(mm, address, ptep,
3671 make_huge_pte(vma, new_page, 1));
3672 page_remove_rmap(old_page, true);
3673 hugepage_add_new_anon_rmap(new_page, vma, address);
3674 set_page_huge_active(new_page);
3675 /* Make the old page be freed below */
3676 new_page = old_page;
3679 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3681 restore_reserve_on_error(h, vma, address, new_page);
3686 spin_lock(ptl); /* Caller expects lock to be held */
3690 /* Return the pagecache page at a given address within a VMA */
3691 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3692 struct vm_area_struct *vma, unsigned long address)
3694 struct address_space *mapping;
3697 mapping = vma->vm_file->f_mapping;
3698 idx = vma_hugecache_offset(h, vma, address);
3700 return find_lock_page(mapping, idx);
3704 * Return whether there is a pagecache page to back given address within VMA.
3705 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3707 static bool hugetlbfs_pagecache_present(struct hstate *h,
3708 struct vm_area_struct *vma, unsigned long address)
3710 struct address_space *mapping;
3714 mapping = vma->vm_file->f_mapping;
3715 idx = vma_hugecache_offset(h, vma, address);
3717 page = find_get_page(mapping, idx);
3720 return page != NULL;
3723 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3726 struct inode *inode = mapping->host;
3727 struct hstate *h = hstate_inode(inode);
3728 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3732 ClearPagePrivate(page);
3735 * set page dirty so that it will not be removed from cache/file
3736 * by non-hugetlbfs specific code paths.
3738 set_page_dirty(page);
3740 spin_lock(&inode->i_lock);
3741 inode->i_blocks += blocks_per_huge_page(h);
3742 spin_unlock(&inode->i_lock);
3746 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3747 struct address_space *mapping, pgoff_t idx,
3748 unsigned long address, pte_t *ptep, unsigned int flags)
3750 struct hstate *h = hstate_vma(vma);
3751 int ret = VM_FAULT_SIGBUS;
3757 bool new_page = false;
3760 * Currently, we are forced to kill the process in the event the
3761 * original mapper has unmapped pages from the child due to a failed
3762 * COW. Warn that such a situation has occurred as it may not be obvious
3764 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3765 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3771 * Use page lock to guard against racing truncation
3772 * before we get page_table_lock.
3775 page = find_lock_page(mapping, idx);
3777 size = i_size_read(mapping->host) >> huge_page_shift(h);
3780 page = alloc_huge_page(vma, address, 0);
3782 ret = PTR_ERR(page);
3786 ret = VM_FAULT_SIGBUS;
3789 clear_huge_page(page, address, pages_per_huge_page(h));
3790 __SetPageUptodate(page);
3793 if (vma->vm_flags & VM_MAYSHARE) {
3794 int err = huge_add_to_page_cache(page, mapping, idx);
3803 if (unlikely(anon_vma_prepare(vma))) {
3805 goto backout_unlocked;
3811 * If memory error occurs between mmap() and fault, some process
3812 * don't have hwpoisoned swap entry for errored virtual address.
3813 * So we need to block hugepage fault by PG_hwpoison bit check.
3815 if (unlikely(PageHWPoison(page))) {
3816 ret = VM_FAULT_HWPOISON_LARGE |
3817 VM_FAULT_SET_HINDEX(hstate_index(h));
3818 goto backout_unlocked;
3823 * If we are going to COW a private mapping later, we examine the
3824 * pending reservations for this page now. This will ensure that
3825 * any allocations necessary to record that reservation occur outside
3828 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3829 if (vma_needs_reservation(h, vma, address) < 0) {
3831 goto backout_unlocked;
3833 /* Just decrements count, does not deallocate */
3834 vma_end_reservation(h, vma, address);
3837 ptl = huge_pte_lockptr(h, mm, ptep);
3839 size = i_size_read(mapping->host) >> huge_page_shift(h);
3844 if (!huge_pte_none(huge_ptep_get(ptep)))
3848 ClearPagePrivate(page);
3849 hugepage_add_new_anon_rmap(page, vma, address);
3851 page_dup_rmap(page, true);
3852 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3853 && (vma->vm_flags & VM_SHARED)));
3854 set_huge_pte_at(mm, address, ptep, new_pte);
3856 hugetlb_count_add(pages_per_huge_page(h), mm);
3857 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3858 /* Optimization, do the COW without a second fault */
3859 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
3865 * Only make newly allocated pages active. Existing pages found
3866 * in the pagecache could be !page_huge_active() if they have been
3867 * isolated for migration.
3870 set_page_huge_active(page);
3880 restore_reserve_on_error(h, vma, address, page);
3886 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
3889 unsigned long key[2];
3892 key[0] = (unsigned long) mapping;
3895 hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
3897 return hash & (num_fault_mutexes - 1);
3901 * For uniprocesor systems we always use a single mutex, so just
3902 * return 0 and avoid the hashing overhead.
3904 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
3911 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3912 unsigned long address, unsigned int flags)
3919 struct page *page = NULL;
3920 struct page *pagecache_page = NULL;
3921 struct hstate *h = hstate_vma(vma);
3922 struct address_space *mapping;
3923 int need_wait_lock = 0;
3925 address &= huge_page_mask(h);
3927 ptep = huge_pte_offset(mm, address);
3929 entry = huge_ptep_get(ptep);
3930 if (unlikely(is_hugetlb_entry_migration(entry))) {
3931 migration_entry_wait_huge(vma, mm, ptep);
3933 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3934 return VM_FAULT_HWPOISON_LARGE |
3935 VM_FAULT_SET_HINDEX(hstate_index(h));
3937 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3939 return VM_FAULT_OOM;
3942 mapping = vma->vm_file->f_mapping;
3943 idx = vma_hugecache_offset(h, vma, address);
3946 * Serialize hugepage allocation and instantiation, so that we don't
3947 * get spurious allocation failures if two CPUs race to instantiate
3948 * the same page in the page cache.
3950 hash = hugetlb_fault_mutex_hash(h, mapping, idx);
3951 mutex_lock(&hugetlb_fault_mutex_table[hash]);
3953 entry = huge_ptep_get(ptep);
3954 if (huge_pte_none(entry)) {
3955 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3962 * entry could be a migration/hwpoison entry at this point, so this
3963 * check prevents the kernel from going below assuming that we have
3964 * a active hugepage in pagecache. This goto expects the 2nd page fault,
3965 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3968 if (!pte_present(entry))
3972 * If we are going to COW the mapping later, we examine the pending
3973 * reservations for this page now. This will ensure that any
3974 * allocations necessary to record that reservation occur outside the
3975 * spinlock. For private mappings, we also lookup the pagecache
3976 * page now as it is used to determine if a reservation has been
3979 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3980 if (vma_needs_reservation(h, vma, address) < 0) {
3984 /* Just decrements count, does not deallocate */
3985 vma_end_reservation(h, vma, address);
3987 if (!(vma->vm_flags & VM_MAYSHARE))
3988 pagecache_page = hugetlbfs_pagecache_page(h,
3992 ptl = huge_pte_lock(h, mm, ptep);
3994 /* Check for a racing update before calling hugetlb_cow */
3995 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3999 * hugetlb_cow() requires page locks of pte_page(entry) and
4000 * pagecache_page, so here we need take the former one
4001 * when page != pagecache_page or !pagecache_page.
4003 page = pte_page(entry);
4004 if (page != pagecache_page)
4005 if (!trylock_page(page)) {
4012 if (flags & FAULT_FLAG_WRITE) {
4013 if (!huge_pte_write(entry)) {
4014 ret = hugetlb_cow(mm, vma, address, ptep,
4015 pagecache_page, ptl);
4018 entry = huge_pte_mkdirty(entry);
4020 entry = pte_mkyoung(entry);
4021 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
4022 flags & FAULT_FLAG_WRITE))
4023 update_mmu_cache(vma, address, ptep);
4025 if (page != pagecache_page)
4031 if (pagecache_page) {
4032 unlock_page(pagecache_page);
4033 put_page(pagecache_page);
4036 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4038 * Generally it's safe to hold refcount during waiting page lock. But
4039 * here we just wait to defer the next page fault to avoid busy loop and
4040 * the page is not used after unlocked before returning from the current
4041 * page fault. So we are safe from accessing freed page, even if we wait
4042 * here without taking refcount.
4045 wait_on_page_locked(page);
4049 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
4050 struct page **pages, struct vm_area_struct **vmas,
4051 unsigned long *position, unsigned long *nr_pages,
4052 long i, unsigned int flags)
4054 unsigned long pfn_offset;
4055 unsigned long vaddr = *position;
4056 unsigned long remainder = *nr_pages;
4057 struct hstate *h = hstate_vma(vma);
4060 while (vaddr < vma->vm_end && remainder) {
4062 spinlock_t *ptl = NULL;
4067 * If we have a pending SIGKILL, don't keep faulting pages and
4068 * potentially allocating memory.
4070 if (unlikely(fatal_signal_pending(current))) {
4076 * Some archs (sparc64, sh*) have multiple pte_ts to
4077 * each hugepage. We have to make sure we get the
4078 * first, for the page indexing below to work.
4080 * Note that page table lock is not held when pte is null.
4082 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
4084 ptl = huge_pte_lock(h, mm, pte);
4085 absent = !pte || huge_pte_none(huge_ptep_get(pte));
4088 * When coredumping, it suits get_dump_page if we just return
4089 * an error where there's an empty slot with no huge pagecache
4090 * to back it. This way, we avoid allocating a hugepage, and
4091 * the sparse dumpfile avoids allocating disk blocks, but its
4092 * huge holes still show up with zeroes where they need to be.
4094 if (absent && (flags & FOLL_DUMP) &&
4095 !hugetlbfs_pagecache_present(h, vma, vaddr)) {
4103 * We need call hugetlb_fault for both hugepages under migration
4104 * (in which case hugetlb_fault waits for the migration,) and
4105 * hwpoisoned hugepages (in which case we need to prevent the
4106 * caller from accessing to them.) In order to do this, we use
4107 * here is_swap_pte instead of is_hugetlb_entry_migration and
4108 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
4109 * both cases, and because we can't follow correct pages
4110 * directly from any kind of swap entries.
4112 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
4113 ((flags & FOLL_WRITE) &&
4114 !huge_pte_write(huge_ptep_get(pte)))) {
4119 ret = hugetlb_fault(mm, vma, vaddr,
4120 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
4121 if (!(ret & VM_FAULT_ERROR))
4128 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
4129 page = pte_page(huge_ptep_get(pte));
4132 * Instead of doing 'try_get_page()' below in the same_page
4133 * loop, just check the count once here.
4135 if (unlikely(page_count(page) <= 0)) {
4145 pages[i] = mem_map_offset(page, pfn_offset);
4156 if (vaddr < vma->vm_end && remainder &&
4157 pfn_offset < pages_per_huge_page(h)) {
4159 * We use pfn_offset to avoid touching the pageframes
4160 * of this compound page.
4166 *nr_pages = remainder;
4172 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
4174 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
4177 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
4180 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
4181 unsigned long address, unsigned long end, pgprot_t newprot)
4183 struct mm_struct *mm = vma->vm_mm;
4184 unsigned long start = address;
4187 struct hstate *h = hstate_vma(vma);
4188 unsigned long pages = 0;
4190 BUG_ON(address >= end);
4191 flush_cache_range(vma, address, end);
4193 mmu_notifier_invalidate_range_start(mm, start, end);
4194 i_mmap_lock_write(vma->vm_file->f_mapping);
4195 for (; address < end; address += huge_page_size(h)) {
4197 ptep = huge_pte_offset(mm, address);
4200 ptl = huge_pte_lock(h, mm, ptep);
4201 if (huge_pmd_unshare(mm, &address, ptep)) {
4206 pte = huge_ptep_get(ptep);
4207 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
4211 if (unlikely(is_hugetlb_entry_migration(pte))) {
4212 swp_entry_t entry = pte_to_swp_entry(pte);
4214 if (is_write_migration_entry(entry)) {
4217 make_migration_entry_read(&entry);
4218 newpte = swp_entry_to_pte(entry);
4219 set_huge_pte_at(mm, address, ptep, newpte);
4225 if (!huge_pte_none(pte)) {
4226 pte = huge_ptep_get_and_clear(mm, address, ptep);
4227 pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4228 pte = arch_make_huge_pte(pte, vma, NULL, 0);
4229 set_huge_pte_at(mm, address, ptep, pte);
4235 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4236 * may have cleared our pud entry and done put_page on the page table:
4237 * once we release i_mmap_rwsem, another task can do the final put_page
4238 * and that page table be reused and filled with junk.
4240 flush_hugetlb_tlb_range(vma, start, end);
4241 mmu_notifier_invalidate_range(mm, start, end);
4242 i_mmap_unlock_write(vma->vm_file->f_mapping);
4243 mmu_notifier_invalidate_range_end(mm, start, end);
4245 return pages << h->order;
4248 int hugetlb_reserve_pages(struct inode *inode,
4250 struct vm_area_struct *vma,
4251 vm_flags_t vm_flags)
4254 struct hstate *h = hstate_inode(inode);
4255 struct hugepage_subpool *spool = subpool_inode(inode);
4256 struct resv_map *resv_map;
4259 /* This should never happen */
4261 VM_WARN(1, "%s called with a negative range\n", __func__);
4266 * Only apply hugepage reservation if asked. At fault time, an
4267 * attempt will be made for VM_NORESERVE to allocate a page
4268 * without using reserves
4270 if (vm_flags & VM_NORESERVE)
4274 * Shared mappings base their reservation on the number of pages that
4275 * are already allocated on behalf of the file. Private mappings need
4276 * to reserve the full area even if read-only as mprotect() may be
4277 * called to make the mapping read-write. Assume !vma is a shm mapping
4279 if (!vma || vma->vm_flags & VM_MAYSHARE) {
4280 resv_map = inode_resv_map(inode);
4282 chg = region_chg(resv_map, from, to);
4285 resv_map = resv_map_alloc();
4291 set_vma_resv_map(vma, resv_map);
4292 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4301 * There must be enough pages in the subpool for the mapping. If
4302 * the subpool has a minimum size, there may be some global
4303 * reservations already in place (gbl_reserve).
4305 gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4306 if (gbl_reserve < 0) {
4312 * Check enough hugepages are available for the reservation.
4313 * Hand the pages back to the subpool if there are not
4315 ret = hugetlb_acct_memory(h, gbl_reserve);
4317 /* put back original number of pages, chg */
4318 (void)hugepage_subpool_put_pages(spool, chg);
4323 * Account for the reservations made. Shared mappings record regions
4324 * that have reservations as they are shared by multiple VMAs.
4325 * When the last VMA disappears, the region map says how much
4326 * the reservation was and the page cache tells how much of
4327 * the reservation was consumed. Private mappings are per-VMA and
4328 * only the consumed reservations are tracked. When the VMA
4329 * disappears, the original reservation is the VMA size and the
4330 * consumed reservations are stored in the map. Hence, nothing
4331 * else has to be done for private mappings here
4333 if (!vma || vma->vm_flags & VM_MAYSHARE) {
4334 long add = region_add(resv_map, from, to);
4336 if (unlikely(chg > add)) {
4338 * pages in this range were added to the reserve
4339 * map between region_chg and region_add. This
4340 * indicates a race with alloc_huge_page. Adjust
4341 * the subpool and reserve counts modified above
4342 * based on the difference.
4346 rsv_adjust = hugepage_subpool_put_pages(spool,
4348 hugetlb_acct_memory(h, -rsv_adjust);
4353 if (!vma || vma->vm_flags & VM_MAYSHARE)
4354 /* Don't call region_abort if region_chg failed */
4356 region_abort(resv_map, from, to);
4357 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4358 kref_put(&resv_map->refs, resv_map_release);
4362 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4365 struct hstate *h = hstate_inode(inode);
4366 struct resv_map *resv_map = inode_resv_map(inode);
4368 struct hugepage_subpool *spool = subpool_inode(inode);
4372 chg = region_del(resv_map, start, end);
4374 * region_del() can fail in the rare case where a region
4375 * must be split and another region descriptor can not be
4376 * allocated. If end == LONG_MAX, it will not fail.
4382 spin_lock(&inode->i_lock);
4383 inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4384 spin_unlock(&inode->i_lock);
4387 * If the subpool has a minimum size, the number of global
4388 * reservations to be released may be adjusted.
4390 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4391 hugetlb_acct_memory(h, -gbl_reserve);
4396 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4397 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4398 struct vm_area_struct *vma,
4399 unsigned long addr, pgoff_t idx)
4401 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4403 unsigned long sbase = saddr & PUD_MASK;
4404 unsigned long s_end = sbase + PUD_SIZE;
4406 /* Allow segments to share if only one is marked locked */
4407 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4408 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4411 * match the virtual addresses, permission and the alignment of the
4414 if (pmd_index(addr) != pmd_index(saddr) ||
4415 vm_flags != svm_flags ||
4416 sbase < svma->vm_start || svma->vm_end < s_end)
4422 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4424 unsigned long base = addr & PUD_MASK;
4425 unsigned long end = base + PUD_SIZE;
4428 * check on proper vm_flags and page table alignment
4430 if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
4436 * Determine if start,end range within vma could be mapped by shared pmd.
4437 * If yes, adjust start and end to cover range associated with possible
4438 * shared pmd mappings.
4440 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4441 unsigned long *start, unsigned long *end)
4443 unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
4444 v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
4447 * vma need span at least one aligned PUD size and the start,end range
4448 * must at least partialy within it.
4450 if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
4451 (*end <= v_start) || (*start >= v_end))
4454 /* Extend the range to be PUD aligned for a worst case scenario */
4455 if (*start > v_start)
4456 *start = ALIGN_DOWN(*start, PUD_SIZE);
4459 *end = ALIGN(*end, PUD_SIZE);
4463 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4464 * and returns the corresponding pte. While this is not necessary for the
4465 * !shared pmd case because we can allocate the pmd later as well, it makes the
4466 * code much cleaner. pmd allocation is essential for the shared case because
4467 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4468 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4469 * bad pmd for sharing.
4471 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4473 struct vm_area_struct *vma = find_vma(mm, addr);
4474 struct address_space *mapping = vma->vm_file->f_mapping;
4475 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4477 struct vm_area_struct *svma;
4478 unsigned long saddr;
4483 if (!vma_shareable(vma, addr))
4484 return (pte_t *)pmd_alloc(mm, pud, addr);
4486 i_mmap_lock_write(mapping);
4487 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4491 saddr = page_table_shareable(svma, vma, addr, idx);
4493 spte = huge_pte_offset(svma->vm_mm, saddr);
4495 get_page(virt_to_page(spte));
4504 ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
4506 if (pud_none(*pud)) {
4507 pud_populate(mm, pud,
4508 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4511 put_page(virt_to_page(spte));
4515 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4516 i_mmap_unlock_write(mapping);
4521 * unmap huge page backed by shared pte.
4523 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared
4524 * indicated by page_count > 1, unmap is achieved by clearing pud and
4525 * decrementing the ref count. If count == 1, the pte page is not shared.
4527 * called with page table lock held.
4529 * returns: 1 successfully unmapped a shared pte page
4530 * 0 the underlying pte page is not shared, or it is the last user
4532 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4534 pgd_t *pgd = pgd_offset(mm, *addr);
4535 pud_t *pud = pud_offset(pgd, *addr);
4537 BUG_ON(page_count(virt_to_page(ptep)) == 0);
4538 if (page_count(virt_to_page(ptep)) == 1)
4542 put_page(virt_to_page(ptep));
4544 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4547 #define want_pmd_share() (1)
4548 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4549 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4554 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4559 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4560 unsigned long *start, unsigned long *end)
4563 #define want_pmd_share() (0)
4564 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4566 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4567 pte_t *huge_pte_alloc(struct mm_struct *mm,
4568 unsigned long addr, unsigned long sz)
4574 pgd = pgd_offset(mm, addr);
4575 pud = pud_alloc(mm, pgd, addr);
4577 if (sz == PUD_SIZE) {
4580 BUG_ON(sz != PMD_SIZE);
4581 if (want_pmd_share() && pud_none(*pud))
4582 pte = huge_pmd_share(mm, addr, pud);
4584 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4587 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
4592 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4598 pgd = pgd_offset(mm, addr);
4599 if (pgd_present(*pgd)) {
4600 pud = pud_offset(pgd, addr);
4601 if (pud_present(*pud)) {
4603 return (pte_t *)pud;
4604 pmd = pmd_offset(pud, addr);
4607 return (pte_t *) pmd;
4610 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4613 * These functions are overwritable if your architecture needs its own
4616 struct page * __weak
4617 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4620 return ERR_PTR(-EINVAL);
4623 struct page * __weak
4624 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4625 pmd_t *pmd, int flags)
4627 struct page *page = NULL;
4631 ptl = pmd_lockptr(mm, pmd);
4634 * make sure that the address range covered by this pmd is not
4635 * unmapped from other threads.
4637 if (!pmd_huge(*pmd))
4639 pte = huge_ptep_get((pte_t *)pmd);
4640 if (pte_present(pte)) {
4641 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4642 if (flags & FOLL_GET)
4645 if (is_hugetlb_entry_migration(pte)) {
4647 __migration_entry_wait(mm, (pte_t *)pmd, ptl);
4651 * hwpoisoned entry is treated as no_page_table in
4652 * follow_page_mask().
4660 struct page * __weak
4661 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4662 pud_t *pud, int flags)
4664 if (flags & FOLL_GET)
4667 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4670 #ifdef CONFIG_MEMORY_FAILURE
4673 * This function is called from memory failure code.
4675 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4677 struct hstate *h = page_hstate(hpage);
4678 int nid = page_to_nid(hpage);
4681 spin_lock(&hugetlb_lock);
4683 * Just checking !page_huge_active is not enough, because that could be
4684 * an isolated/hwpoisoned hugepage (which have >0 refcount).
4686 if (!page_huge_active(hpage) && !page_count(hpage)) {
4688 * Hwpoisoned hugepage isn't linked to activelist or freelist,
4689 * but dangling hpage->lru can trigger list-debug warnings
4690 * (this happens when we call unpoison_memory() on it),
4691 * so let it point to itself with list_del_init().
4693 list_del_init(&hpage->lru);
4694 set_page_refcounted(hpage);
4695 h->free_huge_pages--;
4696 h->free_huge_pages_node[nid]--;
4699 spin_unlock(&hugetlb_lock);
4704 bool isolate_huge_page(struct page *page, struct list_head *list)
4708 spin_lock(&hugetlb_lock);
4709 if (!PageHeadHuge(page) || !page_huge_active(page) ||
4710 !get_page_unless_zero(page)) {
4714 clear_page_huge_active(page);
4715 list_move_tail(&page->lru, list);
4717 spin_unlock(&hugetlb_lock);
4721 void putback_active_hugepage(struct page *page)
4723 VM_BUG_ON_PAGE(!PageHead(page), page);
4724 spin_lock(&hugetlb_lock);
4725 set_page_huge_active(page);
4726 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4727 spin_unlock(&hugetlb_lock);