1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_MMZONE_H
3 #define _LINUX_MMZONE_H
6 #ifndef __GENERATING_BOUNDS_H
8 #include <linux/spinlock.h>
9 #include <linux/list.h>
10 #include <linux/wait.h>
11 #include <linux/bitops.h>
12 #include <linux/cache.h>
13 #include <linux/threads.h>
14 #include <linux/numa.h>
15 #include <linux/init.h>
16 #include <linux/seqlock.h>
17 #include <linux/nodemask.h>
18 #include <linux/pageblock-flags.h>
19 #include <linux/page-flags-layout.h>
20 #include <linux/atomic.h>
21 #include <linux/mm_types.h>
22 #include <linux/page-flags.h>
23 #include <linux/local_lock.h>
26 /* Free memory management - zoned buddy allocator. */
27 #ifndef CONFIG_FORCE_MAX_ZONEORDER
30 #define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
32 #define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
35 * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
36 * costly to service. That is between allocation orders which should
37 * coalesce naturally under reasonable reclaim pressure and those which
40 #define PAGE_ALLOC_COSTLY_ORDER 3
46 MIGRATE_PCPTYPES, /* the number of types on the pcp lists */
47 MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,
50 * MIGRATE_CMA migration type is designed to mimic the way
51 * ZONE_MOVABLE works. Only movable pages can be allocated
52 * from MIGRATE_CMA pageblocks and page allocator never
53 * implicitly change migration type of MIGRATE_CMA pageblock.
55 * The way to use it is to change migratetype of a range of
56 * pageblocks to MIGRATE_CMA which can be done by
57 * __free_pageblock_cma() function.
61 #ifdef CONFIG_MEMORY_ISOLATION
62 MIGRATE_ISOLATE, /* can't allocate from here */
67 /* In mm/page_alloc.c; keep in sync also with show_migration_types() there */
68 extern const char * const migratetype_names[MIGRATE_TYPES];
71 # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
72 # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)
74 # define is_migrate_cma(migratetype) false
75 # define is_migrate_cma_page(_page) false
78 static inline bool is_migrate_movable(int mt)
80 return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
84 * Check whether a migratetype can be merged with another migratetype.
86 * It is only mergeable when it can fall back to other migratetypes for
87 * allocation. See fallbacks[MIGRATE_TYPES][3] in page_alloc.c.
89 static inline bool migratetype_is_mergeable(int mt)
91 return mt < MIGRATE_PCPTYPES;
94 #define for_each_migratetype_order(order, type) \
95 for (order = 0; order < MAX_ORDER; order++) \
96 for (type = 0; type < MIGRATE_TYPES; type++)
98 extern int page_group_by_mobility_disabled;
100 #define MIGRATETYPE_MASK ((1UL << PB_migratetype_bits) - 1)
102 #define get_pageblock_migratetype(page) \
103 get_pfnblock_flags_mask(page, page_to_pfn(page), MIGRATETYPE_MASK)
106 struct list_head free_list[MIGRATE_TYPES];
107 unsigned long nr_free;
110 static inline struct page *get_page_from_free_area(struct free_area *area,
113 return list_first_entry_or_null(&area->free_list[migratetype],
117 static inline bool free_area_empty(struct free_area *area, int migratetype)
119 return list_empty(&area->free_list[migratetype]);
125 * Add a wild amount of padding here to ensure data fall into separate
126 * cachelines. There are very few zone structures in the machine, so space
127 * consumption is not a concern here.
129 #if defined(CONFIG_SMP)
130 struct zone_padding {
132 } ____cacheline_internodealigned_in_smp;
133 #define ZONE_PADDING(name) struct zone_padding name;
135 #define ZONE_PADDING(name)
139 enum numa_stat_item {
140 NUMA_HIT, /* allocated in intended node */
141 NUMA_MISS, /* allocated in non intended node */
142 NUMA_FOREIGN, /* was intended here, hit elsewhere */
143 NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */
144 NUMA_LOCAL, /* allocation from local node */
145 NUMA_OTHER, /* allocation from other node */
146 NR_VM_NUMA_EVENT_ITEMS
149 #define NR_VM_NUMA_EVENT_ITEMS 0
152 enum zone_stat_item {
153 /* First 128 byte cacheline (assuming 64 bit words) */
155 NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */
156 NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,
158 NR_ZONE_INACTIVE_FILE,
161 NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages */
162 NR_MLOCK, /* mlock()ed pages found and moved off LRU */
163 /* Second 128 byte cacheline */
165 #if IS_ENABLED(CONFIG_ZSMALLOC)
166 NR_ZSPAGES, /* allocated in zsmalloc */
169 NR_VM_ZONE_STAT_ITEMS };
171 enum node_stat_item {
173 NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
174 NR_ACTIVE_ANON, /* " " " " " */
175 NR_INACTIVE_FILE, /* " " " " " */
176 NR_ACTIVE_FILE, /* " " " " " */
177 NR_UNEVICTABLE, /* " " " " " */
178 NR_SLAB_RECLAIMABLE_B,
179 NR_SLAB_UNRECLAIMABLE_B,
180 NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */
181 NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */
183 WORKINGSET_REFAULT_BASE,
184 WORKINGSET_REFAULT_ANON = WORKINGSET_REFAULT_BASE,
185 WORKINGSET_REFAULT_FILE,
186 WORKINGSET_ACTIVATE_BASE,
187 WORKINGSET_ACTIVATE_ANON = WORKINGSET_ACTIVATE_BASE,
188 WORKINGSET_ACTIVATE_FILE,
189 WORKINGSET_RESTORE_BASE,
190 WORKINGSET_RESTORE_ANON = WORKINGSET_RESTORE_BASE,
191 WORKINGSET_RESTORE_FILE,
192 WORKINGSET_NODERECLAIM,
193 NR_ANON_MAPPED, /* Mapped anonymous pages */
194 NR_FILE_MAPPED, /* pagecache pages mapped into pagetables.
195 only modified from process context */
199 NR_WRITEBACK_TEMP, /* Writeback using temporary buffers */
200 NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */
207 NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */
208 NR_DIRTIED, /* page dirtyings since bootup */
209 NR_WRITTEN, /* page writings since bootup */
210 NR_THROTTLED_WRITTEN, /* NR_WRITTEN while reclaim throttled */
211 NR_KERNEL_MISC_RECLAIMABLE, /* reclaimable non-slab kernel pages */
212 NR_FOLL_PIN_ACQUIRED, /* via: pin_user_page(), gup flag: FOLL_PIN */
213 NR_FOLL_PIN_RELEASED, /* pages returned via unpin_user_page() */
214 NR_KERNEL_STACK_KB, /* measured in KiB */
215 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
216 NR_KERNEL_SCS_KB, /* measured in KiB */
218 NR_PAGETABLE, /* used for pagetables */
222 #ifdef CONFIG_NUMA_BALANCING
223 PGPROMOTE_SUCCESS, /* promote successfully */
225 NR_VM_NODE_STAT_ITEMS
229 * Returns true if the item should be printed in THPs (/proc/vmstat
230 * currently prints number of anon, file and shmem THPs. But the item
231 * is charged in pages).
233 static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item)
235 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
238 return item == NR_ANON_THPS ||
239 item == NR_FILE_THPS ||
240 item == NR_SHMEM_THPS ||
241 item == NR_SHMEM_PMDMAPPED ||
242 item == NR_FILE_PMDMAPPED;
246 * Returns true if the value is measured in bytes (most vmstat values are
247 * measured in pages). This defines the API part, the internal representation
248 * might be different.
250 static __always_inline bool vmstat_item_in_bytes(int idx)
253 * Global and per-node slab counters track slab pages.
254 * It's expected that changes are multiples of PAGE_SIZE.
255 * Internally values are stored in pages.
257 * Per-memcg and per-lruvec counters track memory, consumed
258 * by individual slab objects. These counters are actually
261 return (idx == NR_SLAB_RECLAIMABLE_B ||
262 idx == NR_SLAB_UNRECLAIMABLE_B);
266 * We do arithmetic on the LRU lists in various places in the code,
267 * so it is important to keep the active lists LRU_ACTIVE higher in
268 * the array than the corresponding inactive lists, and to keep
269 * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
271 * This has to be kept in sync with the statistics in zone_stat_item
272 * above and the descriptions in vmstat_text in mm/vmstat.c
279 LRU_INACTIVE_ANON = LRU_BASE,
280 LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
281 LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
282 LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
287 enum vmscan_throttle_state {
288 VMSCAN_THROTTLE_WRITEBACK,
289 VMSCAN_THROTTLE_ISOLATED,
290 VMSCAN_THROTTLE_NOPROGRESS,
291 VMSCAN_THROTTLE_CONGESTED,
295 #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
297 #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
299 static inline bool is_file_lru(enum lru_list lru)
301 return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
304 static inline bool is_active_lru(enum lru_list lru)
306 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
309 #define ANON_AND_FILE 2
312 LRUVEC_CONGESTED, /* lruvec has many dirty pages
313 * backed by a congested BDI
318 struct list_head lists[NR_LRU_LISTS];
319 /* per lruvec lru_lock for memcg */
322 * These track the cost of reclaiming one LRU - file or anon -
323 * over the other. As the observed cost of reclaiming one LRU
324 * increases, the reclaim scan balance tips toward the other.
326 unsigned long anon_cost;
327 unsigned long file_cost;
328 /* Non-resident age, driven by LRU movement */
329 atomic_long_t nonresident_age;
330 /* Refaults at the time of last reclaim cycle */
331 unsigned long refaults[ANON_AND_FILE];
332 /* Various lruvec state flags (enum lruvec_flags) */
335 struct pglist_data *pgdat;
339 /* Isolate unmapped pages */
340 #define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)
341 /* Isolate for asynchronous migration */
342 #define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
343 /* Isolate unevictable pages */
344 #define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
346 /* LRU Isolation modes. */
347 typedef unsigned __bitwise isolate_mode_t;
349 enum zone_watermarks {
358 * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER plus one additional
359 * for pageblock size for THP if configured.
361 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
366 #define NR_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1 + NR_PCP_THP))
369 * Shift to encode migratetype and order in the same integer, with order
370 * in the least significant bits.
372 #define NR_PCP_ORDER_WIDTH 8
373 #define NR_PCP_ORDER_MASK ((1<<NR_PCP_ORDER_WIDTH) - 1)
375 #define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost)
376 #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost)
377 #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
378 #define wmark_pages(z, i) (z->_watermark[i] + z->watermark_boost)
380 /* Fields and list protected by pagesets local_lock in page_alloc.c */
381 struct per_cpu_pages {
382 int count; /* number of pages in the list */
383 int high; /* high watermark, emptying needed */
384 int batch; /* chunk size for buddy add/remove */
385 short free_factor; /* batch scaling factor during free */
387 short expire; /* When 0, remote pagesets are drained */
390 /* Lists of pages, one per migrate type stored on the pcp-lists */
391 struct list_head lists[NR_PCP_LISTS];
394 struct per_cpu_zonestat {
396 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
401 * Low priority inaccurate counters that are only folded
402 * on demand. Use a large type to avoid the overhead of
403 * folding during refresh_cpu_vm_stats.
405 unsigned long vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
409 struct per_cpu_nodestat {
411 s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];
414 #endif /* !__GENERATING_BOUNDS.H */
418 * ZONE_DMA and ZONE_DMA32 are used when there are peripherals not able
419 * to DMA to all of the addressable memory (ZONE_NORMAL).
420 * On architectures where this area covers the whole 32 bit address
421 * space ZONE_DMA32 is used. ZONE_DMA is left for the ones with smaller
422 * DMA addressing constraints. This distinction is important as a 32bit
423 * DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit
424 * platforms may need both zones as they support peripherals with
425 * different DMA addressing limitations.
427 #ifdef CONFIG_ZONE_DMA
430 #ifdef CONFIG_ZONE_DMA32
434 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
435 * performed on pages in ZONE_NORMAL if the DMA devices support
436 * transfers to all addressable memory.
439 #ifdef CONFIG_HIGHMEM
441 * A memory area that is only addressable by the kernel through
442 * mapping portions into its own address space. This is for example
443 * used by i386 to allow the kernel to address the memory beyond
444 * 900MB. The kernel will set up special mappings (page
445 * table entries on i386) for each page that the kernel needs to
451 * ZONE_MOVABLE is similar to ZONE_NORMAL, except that it contains
452 * movable pages with few exceptional cases described below. Main use
453 * cases for ZONE_MOVABLE are to make memory offlining/unplug more
454 * likely to succeed, and to locally limit unmovable allocations - e.g.,
455 * to increase the number of THP/huge pages. Notable special cases are:
457 * 1. Pinned pages: (long-term) pinning of movable pages might
458 * essentially turn such pages unmovable. Therefore, we do not allow
459 * pinning long-term pages in ZONE_MOVABLE. When pages are pinned and
460 * faulted, they come from the right zone right away. However, it is
461 * still possible that address space already has pages in
462 * ZONE_MOVABLE at the time when pages are pinned (i.e. user has
463 * touches that memory before pinning). In such case we migrate them
464 * to a different zone. When migration fails - pinning fails.
465 * 2. memblock allocations: kernelcore/movablecore setups might create
466 * situations where ZONE_MOVABLE contains unmovable allocations
467 * after boot. Memory offlining and allocations fail early.
468 * 3. Memory holes: kernelcore/movablecore setups might create very rare
469 * situations where ZONE_MOVABLE contains memory holes after boot,
470 * for example, if we have sections that are only partially
471 * populated. Memory offlining and allocations fail early.
472 * 4. PG_hwpoison pages: while poisoned pages can be skipped during
473 * memory offlining, such pages cannot be allocated.
474 * 5. Unmovable PG_offline pages: in paravirtualized environments,
475 * hotplugged memory blocks might only partially be managed by the
476 * buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The
477 * parts not manged by the buddy are unmovable PG_offline pages. In
478 * some cases (virtio-mem), such pages can be skipped during
479 * memory offlining, however, cannot be moved/allocated. These
480 * techniques might use alloc_contig_range() to hide previously
481 * exposed pages from the buddy again (e.g., to implement some sort
482 * of memory unplug in virtio-mem).
483 * 6. ZERO_PAGE(0), kernelcore/movablecore setups might create
484 * situations where ZERO_PAGE(0) which is allocated differently
485 * on different platforms may end up in a movable zone. ZERO_PAGE(0)
486 * cannot be migrated.
487 * 7. Memory-hotplug: when using memmap_on_memory and onlining the
488 * memory to the MOVABLE zone, the vmemmap pages are also placed in
489 * such zone. Such pages cannot be really moved around as they are
490 * self-stored in the range, but they are treated as movable when
491 * the range they describe is about to be offlined.
493 * In general, no unmovable allocations that degrade memory offlining
494 * should end up in ZONE_MOVABLE. Allocators (like alloc_contig_range())
495 * have to expect that migrating pages in ZONE_MOVABLE can fail (even
496 * if has_unmovable_pages() states that there are no unmovable pages,
497 * there can be false negatives).
500 #ifdef CONFIG_ZONE_DEVICE
507 #ifndef __GENERATING_BOUNDS_H
509 #define ASYNC_AND_SYNC 2
512 /* Read-mostly fields */
514 /* zone watermarks, access with *_wmark_pages(zone) macros */
515 unsigned long _watermark[NR_WMARK];
516 unsigned long watermark_boost;
518 unsigned long nr_reserved_highatomic;
521 * We don't know if the memory that we're going to allocate will be
522 * freeable or/and it will be released eventually, so to avoid totally
523 * wasting several GB of ram we must reserve some of the lower zone
524 * memory (otherwise we risk to run OOM on the lower zones despite
525 * there being tons of freeable ram on the higher zones). This array is
526 * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl
529 long lowmem_reserve[MAX_NR_ZONES];
534 struct pglist_data *zone_pgdat;
535 struct per_cpu_pages __percpu *per_cpu_pageset;
536 struct per_cpu_zonestat __percpu *per_cpu_zonestats;
538 * the high and batch values are copied to individual pagesets for
544 #ifndef CONFIG_SPARSEMEM
546 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
547 * In SPARSEMEM, this map is stored in struct mem_section
549 unsigned long *pageblock_flags;
550 #endif /* CONFIG_SPARSEMEM */
552 /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
553 unsigned long zone_start_pfn;
556 * spanned_pages is the total pages spanned by the zone, including
557 * holes, which is calculated as:
558 * spanned_pages = zone_end_pfn - zone_start_pfn;
560 * present_pages is physical pages existing within the zone, which
562 * present_pages = spanned_pages - absent_pages(pages in holes);
564 * present_early_pages is present pages existing within the zone
565 * located on memory available since early boot, excluding hotplugged
568 * managed_pages is present pages managed by the buddy system, which
569 * is calculated as (reserved_pages includes pages allocated by the
570 * bootmem allocator):
571 * managed_pages = present_pages - reserved_pages;
573 * cma pages is present pages that are assigned for CMA use
576 * So present_pages may be used by memory hotplug or memory power
577 * management logic to figure out unmanaged pages by checking
578 * (present_pages - managed_pages). And managed_pages should be used
579 * by page allocator and vm scanner to calculate all kinds of watermarks
584 * zone_start_pfn and spanned_pages are protected by span_seqlock.
585 * It is a seqlock because it has to be read outside of zone->lock,
586 * and it is done in the main allocator path. But, it is written
587 * quite infrequently.
589 * The span_seq lock is declared along with zone->lock because it is
590 * frequently read in proximity to zone->lock. It's good to
591 * give them a chance of being in the same cacheline.
593 * Write access to present_pages at runtime should be protected by
594 * mem_hotplug_begin/end(). Any reader who can't tolerant drift of
595 * present_pages should get_online_mems() to get a stable value.
597 atomic_long_t managed_pages;
598 unsigned long spanned_pages;
599 unsigned long present_pages;
600 #if defined(CONFIG_MEMORY_HOTPLUG)
601 unsigned long present_early_pages;
604 unsigned long cma_pages;
609 #ifdef CONFIG_MEMORY_ISOLATION
611 * Number of isolated pageblock. It is used to solve incorrect
612 * freepage counting problem due to racy retrieving migratetype
613 * of pageblock. Protected by zone->lock.
615 unsigned long nr_isolate_pageblock;
618 #ifdef CONFIG_MEMORY_HOTPLUG
619 /* see spanned/present_pages for more description */
620 seqlock_t span_seqlock;
625 /* Write-intensive fields used from the page allocator */
628 /* free areas of different sizes */
629 struct free_area free_area[MAX_ORDER];
631 /* zone flags, see below */
634 /* Primarily protects free_area */
637 /* Write-intensive fields used by compaction and vmstats. */
641 * When free pages are below this point, additional steps are taken
642 * when reading the number of free pages to avoid per-cpu counter
643 * drift allowing watermarks to be breached
645 unsigned long percpu_drift_mark;
647 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
648 /* pfn where compaction free scanner should start */
649 unsigned long compact_cached_free_pfn;
650 /* pfn where compaction migration scanner should start */
651 unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC];
652 unsigned long compact_init_migrate_pfn;
653 unsigned long compact_init_free_pfn;
656 #ifdef CONFIG_COMPACTION
658 * On compaction failure, 1<<compact_defer_shift compactions
659 * are skipped before trying again. The number attempted since
660 * last failure is tracked with compact_considered.
661 * compact_order_failed is the minimum compaction failed order.
663 unsigned int compact_considered;
664 unsigned int compact_defer_shift;
665 int compact_order_failed;
668 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
669 /* Set to true when the PG_migrate_skip bits should be cleared */
670 bool compact_blockskip_flush;
676 /* Zone statistics */
677 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
678 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
679 } ____cacheline_internodealigned_in_smp;
682 PGDAT_DIRTY, /* reclaim scanning has recently found
683 * many dirty file pages at the tail
686 PGDAT_WRITEBACK, /* reclaim scanning has recently found
687 * many pages under writeback
689 PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */
693 ZONE_BOOSTED_WATERMARK, /* zone recently boosted watermarks.
694 * Cleared when kswapd is woken.
696 ZONE_RECLAIM_ACTIVE, /* kswapd may be scanning the zone. */
699 static inline unsigned long zone_managed_pages(struct zone *zone)
701 return (unsigned long)atomic_long_read(&zone->managed_pages);
704 static inline unsigned long zone_cma_pages(struct zone *zone)
707 return zone->cma_pages;
713 static inline unsigned long zone_end_pfn(const struct zone *zone)
715 return zone->zone_start_pfn + zone->spanned_pages;
718 static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
720 return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
723 static inline bool zone_is_initialized(struct zone *zone)
725 return zone->initialized;
728 static inline bool zone_is_empty(struct zone *zone)
730 return zone->spanned_pages == 0;
734 * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty
735 * intersection with the given zone
737 static inline bool zone_intersects(struct zone *zone,
738 unsigned long start_pfn, unsigned long nr_pages)
740 if (zone_is_empty(zone))
742 if (start_pfn >= zone_end_pfn(zone) ||
743 start_pfn + nr_pages <= zone->zone_start_pfn)
750 * The "priority" of VM scanning is how much of the queues we will scan in one
751 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
752 * queues ("queue_length >> 12") during an aging round.
754 #define DEF_PRIORITY 12
756 /* Maximum number of zones on a zonelist */
757 #define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
760 ZONELIST_FALLBACK, /* zonelist with fallback */
763 * The NUMA zonelists are doubled because we need zonelists that
764 * restrict the allocations to a single node for __GFP_THISNODE.
766 ZONELIST_NOFALLBACK, /* zonelist without fallback (__GFP_THISNODE) */
772 * This struct contains information about a zone in a zonelist. It is stored
773 * here to avoid dereferences into large structures and lookups of tables
776 struct zone *zone; /* Pointer to actual zone */
777 int zone_idx; /* zone_idx(zoneref->zone) */
781 * One allocation request operates on a zonelist. A zonelist
782 * is a list of zones, the first one is the 'goal' of the
783 * allocation, the other zones are fallback zones, in decreasing
786 * To speed the reading of the zonelist, the zonerefs contain the zone index
787 * of the entry being read. Helper functions to access information given
788 * a struct zoneref are
790 * zonelist_zone() - Return the struct zone * for an entry in _zonerefs
791 * zonelist_zone_idx() - Return the index of the zone for an entry
792 * zonelist_node_idx() - Return the index of the node for an entry
795 struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
799 * The array of struct pages for flatmem.
800 * It must be declared for SPARSEMEM as well because there are configurations
803 extern struct page *mem_map;
805 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
806 struct deferred_split {
807 spinlock_t split_queue_lock;
808 struct list_head split_queue;
809 unsigned long split_queue_len;
814 * On NUMA machines, each NUMA node would have a pg_data_t to describe
815 * it's memory layout. On UMA machines there is a single pglist_data which
816 * describes the whole memory.
818 * Memory statistics and page replacement data structures are maintained on a
821 typedef struct pglist_data {
823 * node_zones contains just the zones for THIS node. Not all of the
824 * zones may be populated, but it is the full list. It is referenced by
825 * this node's node_zonelists as well as other node's node_zonelists.
827 struct zone node_zones[MAX_NR_ZONES];
830 * node_zonelists contains references to all zones in all nodes.
831 * Generally the first zones will be references to this node's
834 struct zonelist node_zonelists[MAX_ZONELISTS];
836 int nr_zones; /* number of populated zones in this node */
837 #ifdef CONFIG_FLATMEM /* means !SPARSEMEM */
838 struct page *node_mem_map;
839 #ifdef CONFIG_PAGE_EXTENSION
840 struct page_ext *node_page_ext;
843 #if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
845 * Must be held any time you expect node_start_pfn,
846 * node_present_pages, node_spanned_pages or nr_zones to stay constant.
847 * Also synchronizes pgdat->first_deferred_pfn during deferred page
850 * pgdat_resize_lock() and pgdat_resize_unlock() are provided to
851 * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG
852 * or CONFIG_DEFERRED_STRUCT_PAGE_INIT.
854 * Nests above zone->lock and zone->span_seqlock
856 spinlock_t node_size_lock;
858 unsigned long node_start_pfn;
859 unsigned long node_present_pages; /* total number of physical pages */
860 unsigned long node_spanned_pages; /* total size of physical page
861 range, including holes */
863 wait_queue_head_t kswapd_wait;
864 wait_queue_head_t pfmemalloc_wait;
866 /* workqueues for throttling reclaim for different reasons. */
867 wait_queue_head_t reclaim_wait[NR_VMSCAN_THROTTLE];
869 atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */
870 unsigned long nr_reclaim_start; /* nr pages written while throttled
871 * when throttling started. */
872 struct task_struct *kswapd; /* Protected by
873 mem_hotplug_begin/end() */
875 enum zone_type kswapd_highest_zoneidx;
877 int kswapd_failures; /* Number of 'reclaimed == 0' runs */
879 #ifdef CONFIG_COMPACTION
880 int kcompactd_max_order;
881 enum zone_type kcompactd_highest_zoneidx;
882 wait_queue_head_t kcompactd_wait;
883 struct task_struct *kcompactd;
884 bool proactive_compact_trigger;
887 * This is a per-node reserve of pages that are not available
888 * to userspace allocations.
890 unsigned long totalreserve_pages;
894 * node reclaim becomes active if more unmapped pages exist.
896 unsigned long min_unmapped_pages;
897 unsigned long min_slab_pages;
898 #endif /* CONFIG_NUMA */
900 /* Write-intensive fields used by page reclaim */
903 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
905 * If memory initialisation on large machines is deferred then this
906 * is the first PFN that needs to be initialised.
908 unsigned long first_deferred_pfn;
909 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
911 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
912 struct deferred_split deferred_split_queue;
915 /* Fields commonly accessed by the page reclaim scanner */
918 * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
920 * Use mem_cgroup_lruvec() to look up lruvecs.
922 struct lruvec __lruvec;
928 /* Per-node vmstats */
929 struct per_cpu_nodestat __percpu *per_cpu_nodestats;
930 atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
933 #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
934 #define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
936 #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
937 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
939 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
941 return pgdat->node_start_pfn + pgdat->node_spanned_pages;
944 static inline bool pgdat_is_empty(pg_data_t *pgdat)
946 return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
949 #include <linux/memory_hotplug.h>
951 void build_all_zonelists(pg_data_t *pgdat);
952 void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order,
953 enum zone_type highest_zoneidx);
954 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
955 int highest_zoneidx, unsigned int alloc_flags,
957 bool zone_watermark_ok(struct zone *z, unsigned int order,
958 unsigned long mark, int highest_zoneidx,
959 unsigned int alloc_flags);
960 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
961 unsigned long mark, int highest_zoneidx);
963 * Memory initialization context, use to differentiate memory added by
964 * the platform statically or via memory hotplug interface.
966 enum meminit_context {
971 extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
974 extern void lruvec_init(struct lruvec *lruvec);
976 static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
979 return lruvec->pgdat;
981 return container_of(lruvec, struct pglist_data, __lruvec);
985 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
986 int local_memory_node(int node_id);
988 static inline int local_memory_node(int node_id) { return node_id; };
992 * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
994 #define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
996 #ifdef CONFIG_ZONE_DEVICE
997 static inline bool zone_is_zone_device(struct zone *zone)
999 return zone_idx(zone) == ZONE_DEVICE;
1002 static inline bool zone_is_zone_device(struct zone *zone)
1009 * Returns true if a zone has pages managed by the buddy allocator.
1010 * All the reclaim decisions have to use this function rather than
1011 * populated_zone(). If the whole zone is reserved then we can easily
1012 * end up with populated_zone() && !managed_zone().
1014 static inline bool managed_zone(struct zone *zone)
1016 return zone_managed_pages(zone);
1019 /* Returns true if a zone has memory */
1020 static inline bool populated_zone(struct zone *zone)
1022 return zone->present_pages;
1026 static inline int zone_to_nid(struct zone *zone)
1031 static inline void zone_set_nid(struct zone *zone, int nid)
1036 static inline int zone_to_nid(struct zone *zone)
1041 static inline void zone_set_nid(struct zone *zone, int nid) {}
1044 extern int movable_zone;
1046 static inline int is_highmem_idx(enum zone_type idx)
1048 #ifdef CONFIG_HIGHMEM
1049 return (idx == ZONE_HIGHMEM ||
1050 (idx == ZONE_MOVABLE && movable_zone == ZONE_HIGHMEM));
1056 #ifdef CONFIG_ZONE_DMA
1057 bool has_managed_dma(void);
1059 static inline bool has_managed_dma(void)
1066 * is_highmem - helper function to quickly check if a struct zone is a
1067 * highmem zone or not. This is an attempt to keep references
1068 * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
1069 * @zone: pointer to struct zone variable
1070 * Return: 1 for a highmem zone, 0 otherwise
1072 static inline int is_highmem(struct zone *zone)
1074 #ifdef CONFIG_HIGHMEM
1075 return is_highmem_idx(zone_idx(zone));
1081 /* These two functions are used to setup the per zone pages min values */
1084 int min_free_kbytes_sysctl_handler(struct ctl_table *, int, void *, size_t *,
1086 int watermark_scale_factor_sysctl_handler(struct ctl_table *, int, void *,
1087 size_t *, loff_t *);
1088 extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES];
1089 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, void *,
1090 size_t *, loff_t *);
1091 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *, int,
1092 void *, size_t *, loff_t *);
1093 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
1094 void *, size_t *, loff_t *);
1095 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
1096 void *, size_t *, loff_t *);
1097 int numa_zonelist_order_handler(struct ctl_table *, int,
1098 void *, size_t *, loff_t *);
1099 extern int percpu_pagelist_high_fraction;
1100 extern char numa_zonelist_order[];
1101 #define NUMA_ZONELIST_ORDER_LEN 16
1105 extern struct pglist_data contig_page_data;
1106 static inline struct pglist_data *NODE_DATA(int nid)
1108 return &contig_page_data;
1111 #else /* CONFIG_NUMA */
1113 #include <asm/mmzone.h>
1115 #endif /* !CONFIG_NUMA */
1117 extern struct pglist_data *first_online_pgdat(void);
1118 extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
1119 extern struct zone *next_zone(struct zone *zone);
1122 * for_each_online_pgdat - helper macro to iterate over all online nodes
1123 * @pgdat: pointer to a pg_data_t variable
1125 #define for_each_online_pgdat(pgdat) \
1126 for (pgdat = first_online_pgdat(); \
1128 pgdat = next_online_pgdat(pgdat))
1130 * for_each_zone - helper macro to iterate over all memory zones
1131 * @zone: pointer to struct zone variable
1133 * The user only needs to declare the zone variable, for_each_zone
1136 #define for_each_zone(zone) \
1137 for (zone = (first_online_pgdat())->node_zones; \
1139 zone = next_zone(zone))
1141 #define for_each_populated_zone(zone) \
1142 for (zone = (first_online_pgdat())->node_zones; \
1144 zone = next_zone(zone)) \
1145 if (!populated_zone(zone)) \
1146 ; /* do nothing */ \
1149 static inline struct zone *zonelist_zone(struct zoneref *zoneref)
1151 return zoneref->zone;
1154 static inline int zonelist_zone_idx(struct zoneref *zoneref)
1156 return zoneref->zone_idx;
1159 static inline int zonelist_node_idx(struct zoneref *zoneref)
1161 return zone_to_nid(zoneref->zone);
1164 struct zoneref *__next_zones_zonelist(struct zoneref *z,
1165 enum zone_type highest_zoneidx,
1169 * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
1170 * @z: The cursor used as a starting point for the search
1171 * @highest_zoneidx: The zone index of the highest zone to return
1172 * @nodes: An optional nodemask to filter the zonelist with
1174 * This function returns the next zone at or below a given zone index that is
1175 * within the allowed nodemask using a cursor as the starting point for the
1176 * search. The zoneref returned is a cursor that represents the current zone
1177 * being examined. It should be advanced by one before calling
1178 * next_zones_zonelist again.
1180 * Return: the next zone at or below highest_zoneidx within the allowed
1181 * nodemask using a cursor within a zonelist as a starting point
1183 static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
1184 enum zone_type highest_zoneidx,
1187 if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx))
1189 return __next_zones_zonelist(z, highest_zoneidx, nodes);
1193 * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
1194 * @zonelist: The zonelist to search for a suitable zone
1195 * @highest_zoneidx: The zone index of the highest zone to return
1196 * @nodes: An optional nodemask to filter the zonelist with
1198 * This function returns the first zone at or below a given zone index that is
1199 * within the allowed nodemask. The zoneref returned is a cursor that can be
1200 * used to iterate the zonelist with next_zones_zonelist by advancing it by
1201 * one before calling.
1203 * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is
1204 * never NULL). This may happen either genuinely, or due to concurrent nodemask
1205 * update due to cpuset modification.
1207 * Return: Zoneref pointer for the first suitable zone found
1209 static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
1210 enum zone_type highest_zoneidx,
1213 return next_zones_zonelist(zonelist->_zonerefs,
1214 highest_zoneidx, nodes);
1218 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
1219 * @zone: The current zone in the iterator
1220 * @z: The current pointer within zonelist->_zonerefs being iterated
1221 * @zlist: The zonelist being iterated
1222 * @highidx: The zone index of the highest zone to return
1223 * @nodemask: Nodemask allowed by the allocator
1225 * This iterator iterates though all zones at or below a given zone index and
1226 * within a given nodemask
1228 #define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
1229 for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \
1231 z = next_zones_zonelist(++z, highidx, nodemask), \
1232 zone = zonelist_zone(z))
1234 #define for_next_zone_zonelist_nodemask(zone, z, highidx, nodemask) \
1235 for (zone = z->zone; \
1237 z = next_zones_zonelist(++z, highidx, nodemask), \
1238 zone = zonelist_zone(z))
1242 * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
1243 * @zone: The current zone in the iterator
1244 * @z: The current pointer within zonelist->zones being iterated
1245 * @zlist: The zonelist being iterated
1246 * @highidx: The zone index of the highest zone to return
1248 * This iterator iterates though all zones at or below a given zone index.
1250 #define for_each_zone_zonelist(zone, z, zlist, highidx) \
1251 for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
1253 /* Whether the 'nodes' are all movable nodes */
1254 static inline bool movable_only_nodes(nodemask_t *nodes)
1256 struct zonelist *zonelist;
1260 if (nodes_empty(*nodes))
1264 * We can chose arbitrary node from the nodemask to get a
1265 * zonelist as they are interlinked. We just need to find
1266 * at least one zone that can satisfy kernel allocations.
1268 nid = first_node(*nodes);
1269 zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
1270 z = first_zones_zonelist(zonelist, ZONE_NORMAL, nodes);
1271 return (!z->zone) ? true : false;
1275 #ifdef CONFIG_SPARSEMEM
1276 #include <asm/sparsemem.h>
1279 #ifdef CONFIG_FLATMEM
1280 #define pfn_to_nid(pfn) (0)
1283 #ifdef CONFIG_SPARSEMEM
1286 * PA_SECTION_SHIFT physical address to/from section number
1287 * PFN_SECTION_SHIFT pfn to/from section number
1289 #define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
1290 #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
1292 #define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
1294 #define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)
1295 #define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
1297 #define SECTION_BLOCKFLAGS_BITS \
1298 ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
1300 #if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
1301 #error Allocator MAX_ORDER exceeds SECTION_SIZE
1304 static inline unsigned long pfn_to_section_nr(unsigned long pfn)
1306 return pfn >> PFN_SECTION_SHIFT;
1308 static inline unsigned long section_nr_to_pfn(unsigned long sec)
1310 return sec << PFN_SECTION_SHIFT;
1313 #define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
1314 #define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
1316 #define SUBSECTION_SHIFT 21
1317 #define SUBSECTION_SIZE (1UL << SUBSECTION_SHIFT)
1319 #define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)
1320 #define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT)
1321 #define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
1323 #if SUBSECTION_SHIFT > SECTION_SIZE_BITS
1324 #error Subsection size exceeds section size
1326 #define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))
1329 #define SUBSECTION_ALIGN_UP(pfn) ALIGN((pfn), PAGES_PER_SUBSECTION)
1330 #define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
1332 struct mem_section_usage {
1333 #ifdef CONFIG_SPARSEMEM_VMEMMAP
1334 DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);
1336 /* See declaration of similar field in struct zone */
1337 unsigned long pageblock_flags[0];
1340 void subsection_map_init(unsigned long pfn, unsigned long nr_pages);
1344 struct mem_section {
1346 * This is, logically, a pointer to an array of struct
1347 * pages. However, it is stored with some other magic.
1348 * (see sparse.c::sparse_init_one_section())
1350 * Additionally during early boot we encode node id of
1351 * the location of the section here to guide allocation.
1352 * (see sparse.c::memory_present())
1354 * Making it a UL at least makes someone do a cast
1355 * before using it wrong.
1357 unsigned long section_mem_map;
1359 struct mem_section_usage *usage;
1360 #ifdef CONFIG_PAGE_EXTENSION
1362 * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use
1363 * section. (see page_ext.h about this.)
1365 struct page_ext *page_ext;
1369 * WARNING: mem_section must be a power-of-2 in size for the
1370 * calculation and use of SECTION_ROOT_MASK to make sense.
1374 #ifdef CONFIG_SPARSEMEM_EXTREME
1375 #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
1377 #define SECTIONS_PER_ROOT 1
1380 #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
1381 #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
1382 #define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
1384 #ifdef CONFIG_SPARSEMEM_EXTREME
1385 extern struct mem_section **mem_section;
1387 extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
1390 static inline unsigned long *section_to_usemap(struct mem_section *ms)
1392 return ms->usage->pageblock_flags;
1395 static inline struct mem_section *__nr_to_section(unsigned long nr)
1397 unsigned long root = SECTION_NR_TO_ROOT(nr);
1399 if (unlikely(root >= NR_SECTION_ROOTS))
1402 #ifdef CONFIG_SPARSEMEM_EXTREME
1403 if (!mem_section || !mem_section[root])
1406 return &mem_section[root][nr & SECTION_ROOT_MASK];
1408 extern size_t mem_section_usage_size(void);
1411 * We use the lower bits of the mem_map pointer to store
1412 * a little bit of information. The pointer is calculated
1413 * as mem_map - section_nr_to_pfn(pnum). The result is
1414 * aligned to the minimum alignment of the two values:
1415 * 1. All mem_map arrays are page-aligned.
1416 * 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT
1417 * lowest bits. PFN_SECTION_SHIFT is arch-specific
1418 * (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the
1419 * worst combination is powerpc with 256k pages,
1420 * which results in PFN_SECTION_SHIFT equal 6.
1421 * To sum it up, at least 6 bits are available.
1423 #define SECTION_MARKED_PRESENT (1UL<<0)
1424 #define SECTION_HAS_MEM_MAP (1UL<<1)
1425 #define SECTION_IS_ONLINE (1UL<<2)
1426 #define SECTION_IS_EARLY (1UL<<3)
1427 #define SECTION_TAINT_ZONE_DEVICE (1UL<<4)
1428 #define SECTION_MAP_LAST_BIT (1UL<<5)
1429 #define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1))
1430 #define SECTION_NID_SHIFT 6
1432 static inline struct page *__section_mem_map_addr(struct mem_section *section)
1434 unsigned long map = section->section_mem_map;
1435 map &= SECTION_MAP_MASK;
1436 return (struct page *)map;
1439 static inline int present_section(struct mem_section *section)
1441 return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
1444 static inline int present_section_nr(unsigned long nr)
1446 return present_section(__nr_to_section(nr));
1449 static inline int valid_section(struct mem_section *section)
1451 return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
1454 static inline int early_section(struct mem_section *section)
1456 return (section && (section->section_mem_map & SECTION_IS_EARLY));
1459 static inline int valid_section_nr(unsigned long nr)
1461 return valid_section(__nr_to_section(nr));
1464 static inline int online_section(struct mem_section *section)
1466 return (section && (section->section_mem_map & SECTION_IS_ONLINE));
1469 static inline int online_device_section(struct mem_section *section)
1471 unsigned long flags = SECTION_IS_ONLINE | SECTION_TAINT_ZONE_DEVICE;
1473 return section && ((section->section_mem_map & flags) == flags);
1476 static inline int online_section_nr(unsigned long nr)
1478 return online_section(__nr_to_section(nr));
1481 #ifdef CONFIG_MEMORY_HOTPLUG
1482 void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
1483 void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
1486 static inline struct mem_section *__pfn_to_section(unsigned long pfn)
1488 return __nr_to_section(pfn_to_section_nr(pfn));
1491 extern unsigned long __highest_present_section_nr;
1493 static inline int subsection_map_index(unsigned long pfn)
1495 return (pfn & ~(PAGE_SECTION_MASK)) / PAGES_PER_SUBSECTION;
1498 #ifdef CONFIG_SPARSEMEM_VMEMMAP
1499 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
1501 int idx = subsection_map_index(pfn);
1503 return test_bit(idx, ms->usage->subsection_map);
1506 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
1512 #ifndef CONFIG_HAVE_ARCH_PFN_VALID
1514 * pfn_valid - check if there is a valid memory map entry for a PFN
1515 * @pfn: the page frame number to check
1517 * Check if there is a valid memory map entry aka struct page for the @pfn.
1518 * Note, that availability of the memory map entry does not imply that
1519 * there is actual usable memory at that @pfn. The struct page may
1520 * represent a hole or an unusable page frame.
1522 * Return: 1 for PFNs that have memory map entries and 0 otherwise
1524 static inline int pfn_valid(unsigned long pfn)
1526 struct mem_section *ms;
1529 * Ensure the upper PAGE_SHIFT bits are clear in the
1530 * pfn. Else it might lead to false positives when
1531 * some of the upper bits are set, but the lower bits
1532 * match a valid pfn.
1534 if (PHYS_PFN(PFN_PHYS(pfn)) != pfn)
1537 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1539 ms = __pfn_to_section(pfn);
1540 if (!valid_section(ms))
1543 * Traditionally early sections always returned pfn_valid() for
1544 * the entire section-sized span.
1546 return early_section(ms) || pfn_section_valid(ms, pfn);
1550 static inline int pfn_in_present_section(unsigned long pfn)
1552 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1554 return present_section(__pfn_to_section(pfn));
1557 static inline unsigned long next_present_section_nr(unsigned long section_nr)
1559 while (++section_nr <= __highest_present_section_nr) {
1560 if (present_section_nr(section_nr))
1568 * These are _only_ used during initialisation, therefore they
1569 * can use __initdata ... They could have names to indicate
1573 #define pfn_to_nid(pfn) \
1575 unsigned long __pfn_to_nid_pfn = (pfn); \
1576 page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \
1579 #define pfn_to_nid(pfn) (0)
1582 void sparse_init(void);
1584 #define sparse_init() do {} while (0)
1585 #define sparse_index_init(_sec, _nid) do {} while (0)
1586 #define pfn_in_present_section pfn_valid
1587 #define subsection_map_init(_pfn, _nr_pages) do {} while (0)
1588 #endif /* CONFIG_SPARSEMEM */
1590 #endif /* !__GENERATING_BOUNDS.H */
1591 #endif /* !__ASSEMBLY__ */
1592 #endif /* _LINUX_MMZONE_H */