@@ -11,7 +11,6 @@
#undef CONFIG_PGTABLE_LEVELS
#undef CONFIG_ILLEGAL_POINTER_VALUE
#undef CONFIG_SPARSEMEM_VMEMMAP
-#undef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION
#undef CONFIG_NR_CPUS
#undef CONFIG_PARAVIRT_XXL
@@ -8,7 +8,7 @@ if DAX
config DEV_DAX
tristate "Device DAX: direct access mapping device"
depends on TRANSPARENT_HUGEPAGE
- select SPARSEMEM_VMEMMAP_OPTIMIZATION if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP && SPARSEMEM_VMEMMAP
+ select SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP && SPARSEMEM_VMEMMAP
help
Support raw access to differentiated (persistence, bandwidth,
latency...) memory via an mmap(2) capable character
@@ -278,7 +278,7 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP
def_bool HUGETLB_PAGE
depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
depends on SPARSEMEM_VMEMMAP
- select SPARSEMEM_VMEMMAP_OPTIMIZATION
+ select SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE
config HUGETLB_PMD_PAGE_TABLE_SHARING
def_bool HUGETLB_PAGE
@@ -546,6 +546,7 @@ FOLIO_MATCH(flags, _flags_3);
FOLIO_MATCH(compound_info, _head_3);
#undef FOLIO_MATCH
+#ifndef __GENERATING_BOUNDS_H
/**
* struct ptdesc - Memory descriptor for page tables.
* @pt_flags: enum pt_flags plus zone/node/section.
@@ -1990,5 +1991,5 @@ static inline unsigned long mmf_init_legacy_flags(unsigned long flags)
(1UL << MMF_HAS_MDWE_NO_INHERIT));
return flags & MMF_INIT_LEGACY_MASK;
}
-
+#endif /* __GENERATING_BOUNDS_H */
#endif /* _LINUX_MM_TYPES_H */
@@ -3,8 +3,6 @@
#define _LINUX_MMZONE_H
#ifndef __ASSEMBLY__
-#ifndef __GENERATING_BOUNDS_H
-
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/list_nulls.h>
@@ -96,33 +94,32 @@
#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER)
-/*
- * Hugepage Vmemmap Optimization (HVO) requires struct pages of the head page to
- * be naturally aligned with regard to the folio size.
- *
- * HVO which is only active if the size of struct page is a power of 2.
- */
-#define MAX_FOLIO_VMEMMAP_ALIGN \
- (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION) && \
- is_power_of_2(sizeof(struct page)) ? \
- MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
-
/* The number of vmemmap pages required by a vmemmap-optimized folio. */
#define OPTIMIZED_FOLIO_VMEMMAP_PAGES 1
#define OPTIMIZED_FOLIO_VMEMMAP_SIZE (OPTIMIZED_FOLIO_VMEMMAP_PAGES * PAGE_SIZE)
#define OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES (OPTIMIZED_FOLIO_VMEMMAP_SIZE / sizeof(struct page))
#define OPTIMIZABLE_FOLIO_MIN_ORDER (ilog2(OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES) + 1)
-#define __NR_OPTIMIZABLE_FOLIO_ORDERS (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1)
-#define NR_OPTIMIZABLE_FOLIO_ORDERS \
- ((__NR_OPTIMIZABLE_FOLIO_ORDERS > 0 && \
- IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION)) ? __NR_OPTIMIZABLE_FOLIO_ORDERS : 0)
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION
+/*
+ * Hugepage Vmemmap Optimization (HVO) requires the struct page of the head page
+ * to be naturally aligned with regard to the vmemmap size of the maximal folio.
+ */
+#define MAX_FOLIO_VMEMMAP_ALIGN (MAX_FOLIO_NR_PAGES * sizeof(struct page))
+#define NR_OPTIMIZABLE_FOLIO_ORDERS (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1)
+#else
+#define MAX_FOLIO_VMEMMAP_ALIGN 0
+#define NR_OPTIMIZABLE_FOLIO_ORDERS 0
+#endif
static inline bool order_vmemmap_optimizable(unsigned int order)
{
+ if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION))
+ return false;
return order >= OPTIMIZABLE_FOLIO_MIN_ORDER;
}
+#ifndef __GENERATING_BOUNDS_H
enum migratetype {
MIGRATE_UNMOVABLE,
MIGRATE_MOVABLE,
@@ -2044,7 +2041,7 @@ struct mem_section {
*/
struct page_ext *page_ext;
#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION
/*
* The order of compound pages in this section. Typically, the section
* holds compound pages of this order; a larger compound page will span
@@ -2236,7 +2233,7 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long
}
#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION
static inline void section_set_order(struct mem_section *section, unsigned int order)
{
VM_WARN_ON(section->order && order && section->order != order);
@@ -2277,9 +2274,6 @@ static inline unsigned int pfn_to_section_order(unsigned long pfn)
static inline bool section_vmemmap_optimizable(const struct mem_section *section)
{
- if (!is_power_of_2(sizeof(struct page)))
- return false;
-
return order_vmemmap_optimizable(section_order(section));
}
@@ -2,6 +2,7 @@
#ifndef PAGE_FLAGS_LAYOUT_H
#define PAGE_FLAGS_LAYOUT_H
+#ifndef __GENERATING_BOUNDS_H
#include <linux/numa.h>
#include <generated/bounds.h>
@@ -121,4 +122,5 @@
(NR_NON_PAGEFLAG_BITS + NR_PAGEFLAGS))
#endif
+#endif /* __GENERATING_BOUNDS_H */
#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
@@ -198,32 +198,12 @@ enum pageflags {
#ifndef __GENERATING_BOUNDS_H
-/*
- * For tail pages, if the size of struct page is power-of-2 ->compound_info
- * encodes the mask that converts the address of the tail page address to
- * the head page address.
- *
- * Otherwise, ->compound_info has direct pointer to head pages.
- */
-static __always_inline bool compound_info_has_mask(void)
-{
- /*
- * The approach with mask would work in the wider set of conditions,
- * but it requires validating that struct pages are naturally aligned
- * for all orders up to the MAX_FOLIO_ORDER, which can be tricky.
- */
- if (!IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION))
- return false;
-
- return is_power_of_2(sizeof(struct page));
-}
-
static __always_inline unsigned long _compound_head(const struct page *page)
{
unsigned long info = READ_ONCE(page->compound_info);
unsigned long mask;
- if (!compound_info_has_mask()) {
+ if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) {
/* Bit 0 encodes PageTail() */
if (info & 1)
return info - 1;
@@ -232,8 +212,8 @@ static __always_inline unsigned long _compound_head(const struct page *page)
}
/*
- * If compound_info_has_mask() is true the rest of the info encodes
- * the mask that converts the address of the tail page to the head page.
+ * If HVO is enabled the rest of the info encodes the mask that converts
+ * the address of the tail page to the head page.
*
* No need to clear bit 0 in the mask as 'page' always has it clear.
*
@@ -257,7 +237,7 @@ static __always_inline void set_compound_head(struct page *tail,
unsigned int shift;
unsigned long mask;
- if (!compound_info_has_mask()) {
+ if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) {
WRITE_ONCE(tail->compound_info, (unsigned long)head | 1);
return;
}
@@ -8,6 +8,7 @@
#define __GENERATING_BOUNDS_H
#define COMPILE_OFFSETS
/* Include headers that define the enum constants of interest */
+#include <linux/mm_types.h>
#include <linux/page-flags.h>
#include <linux/mmzone.h>
#include <linux/kbuild.h>
@@ -30,6 +31,10 @@ int main(void)
DEFINE(LRU_GEN_WIDTH, 0);
DEFINE(__LRU_REFS_WIDTH, 0);
#endif
+ if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE) &&
+ is_power_of_2(sizeof(struct page)) &&
+ MAX_FOLIO_ORDER >= OPTIMIZABLE_FOLIO_MIN_ORDER)
+ DEFINE(SPARSEMEM_VMEMMAP_OPTIMIZATION, 1);
/* End of constants */
return 0;
@@ -410,7 +410,7 @@ config SPARSEMEM_VMEMMAP
pfn_to_page and page_to_pfn operations. This is the most
efficient option when sufficient kernel resources are available.
-config SPARSEMEM_VMEMMAP_OPTIMIZATION
+config SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE
bool
depends on SPARSEMEM_VMEMMAP
@@ -22,6 +22,7 @@
#include "hugetlb_vmemmap.h"
#include "internal.h"
+#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION
/**
* struct vmemmap_remap_walk - walk vmemmap page table
*
@@ -693,3 +694,4 @@ static int __init hugetlb_vmemmap_init(void)
return 0;
}
late_initcall(hugetlb_vmemmap_init);
+#endif
@@ -10,7 +10,7 @@
#define _LINUX_HUGETLB_VMEMMAP_H
#include <linux/hugetlb.h>
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+#if defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && defined(SPARSEMEM_VMEMMAP_OPTIMIZATION)
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio);
long hugetlb_vmemmap_restore_folios(const struct hstate *h,
struct list_head *folio_list,
@@ -32,8 +32,6 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate
{
int size = hugetlb_vmemmap_size(h) - OPTIMIZED_FOLIO_VMEMMAP_SIZE;
- if (!is_power_of_2(sizeof(struct page)))
- return 0;
return size > 0 ? size : 0;
}
#else
@@ -1023,9 +1023,6 @@ static inline bool vmemmap_page_optimizable(const struct page *page)
unsigned long pfn = page_to_pfn(page);
unsigned long nr_pages = 1UL << pfn_to_section_order(pfn);
- if (!is_power_of_2(sizeof(struct page)))
- return false;
-
return (pfn & (nr_pages - 1)) >= OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES;
}
#else
@@ -301,10 +301,8 @@ void __init sparse_init(void)
unsigned long pnum_end, pnum_begin, map_count = 1;
int nid_begin;
- if (compound_info_has_mask()) {
- VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),
- MAX_FOLIO_VMEMMAP_ALIGN));
- }
+ VM_WARN_ON(IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION) &&
+ !IS_ALIGNED((unsigned long)pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN));
pnum_begin = first_present_section_nr();
nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
@@ -1338,7 +1338,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page)
foliop = (struct folio *)page;
} else {
/* See compound_head() */
- if (compound_info_has_mask()) {
+ if (IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) {
unsigned long p = (unsigned long)page;
foliop = (struct folio *)(p & info);
HVO depends on build-time conditions that are not fully expressible in Kconfig, including whether sizeof(struct page) is a power of two and whether the supported folio order range can use the optimized layout. Those checks are currently duplicated in several places. Define SPARSEMEM_VMEMMAP_OPTIMIZATION in bounds.c when the build-time requirements are met, and use that generated constant to guard the generic HVO code. This centralizes the build-time checks instead of repeating them throughout the HVO paths. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- arch/x86/entry/vdso/vdso32/fake_32bit_build.h | 1 - drivers/dax/Kconfig | 2 +- fs/Kconfig | 2 +- include/linux/mm_types.h | 3 +- include/linux/mmzone.h | 38 ++++++++----------- include/linux/page-flags-layout.h | 2 + include/linux/page-flags.h | 28 ++------------ kernel/bounds.c | 5 +++ mm/Kconfig | 2 +- mm/hugetlb_vmemmap.c | 2 + mm/hugetlb_vmemmap.h | 4 +- mm/internal.h | 3 -- mm/sparse.c | 6 +-- mm/util.c | 2 +- 14 files changed, 38 insertions(+), 62 deletions(-)