@@ -3201,21 +3201,7 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h,
unsigned long flags;
struct folio *folio, *tmp_f;
- /* Send list for bulk vmemmap optimization processing */
- hugetlb_vmemmap_optimize_folios(h, folio_list);
-
list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
- if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
- /*
- * If HVO fails, initialize all tail struct pages
- * We do not worry about potential long lock hold
- * time as this is early in boot and there should
- * be no contention.
- */
- hugetlb_folio_init_tail_vmemmap(folio, h,
- OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES,
- pages_per_huge_page(h));
- }
hugetlb_bootmem_init_migratetype(folio, h);
/* Subdivide locks to achieve better parallel performance */
spin_lock_irqsave(&hugetlb_lock, flags);
@@ -3238,6 +3224,8 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
struct page *page = virt_to_page(m);
struct folio *folio = (void *)page;
+ unsigned long pfn = PHYS_PFN(__pa(m));
+ unsigned long nr_pages = pages_per_huge_page(m->hstate);
h = m->hstate;
/*
@@ -3251,13 +3239,12 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
VM_BUG_ON(!hstate_is_gigantic(h));
WARN_ON(folio_ref_count(folio) != 1);
- hugetlb_folio_init_vmemmap(folio, h,
- OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES);
+ hugetlb_folio_init_vmemmap(folio, h, vmemmap_nr_struct_pages(pfn, nr_pages));
init_new_hugetlb_folio(folio);
- if (order_vmemmap_optimizable(pfn_to_section_order(folio_pfn(folio)))) {
+ if (order_vmemmap_optimizable(pfn_to_section_order(pfn))) {
folio_set_hugetlb_vmemmap_optimized(folio);
- section_set_order_range(folio_pfn(folio), folio_nr_pages(folio), 0);
+ section_set_order_range(pfn, nr_pages, 0);
}
if (hugetlb_early_cma(h))
@@ -3274,7 +3261,7 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
* (via hugetlb_bootmem_init_migratetype), so skip it here.
*/
if (!folio_test_hugetlb_cma(folio))
- adjust_managed_page_count(page, pages_per_huge_page(h));
+ adjust_managed_page_count(page, nr_pages);
cond_resched();
}
@@ -599,23 +599,17 @@ static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *fol
void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
{
struct folio *folio;
- unsigned long nr_to_optimize = 0;
LIST_HEAD(vmemmap_pages);
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
- list_for_each_entry(folio, folio_list, lru) {
- int ret;
-
- /*
- * Bootmem gigantic folios may already be marked optimized when
- * their vmemmap layout was prepared earlier, so skip them here.
- */
- if (folio_test_hugetlb_vmemmap_optimized(folio))
- continue;
+ if (!vmemmap_should_optimize(h))
+ return;
- nr_to_optimize++;
+ if (list_empty(folio_list))
+ return;
- ret = hugetlb_vmemmap_split_folio(h, folio);
+ list_for_each_entry(folio, folio_list, lru) {
+ int ret = hugetlb_vmemmap_split_folio(h, folio);
/*
* Splitting the PMD requires allocating a page, thus let's fail
@@ -627,9 +621,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
break;
}
- if (!nr_to_optimize)
- return;
-
flush_tlb_all();
list_for_each_entry(folio, folio_list, lru) {
@@ -949,6 +949,29 @@ void memmap_init_range(unsigned long, int, unsigned long, unsigned long,
unsigned long, enum meminit_context, struct vmem_altmap *, int,
bool);
+static inline int vmemmap_nr_struct_pages(unsigned long pfn, unsigned long nr_pages)
+{
+ const unsigned int order = pfn_to_section_order(pfn);
+ const unsigned long pages_per_compound = 1UL << order;
+
+ if (!order_vmemmap_optimizable(order))
+ return nr_pages;
+
+ if (order < PFN_SECTION_SHIFT) {
+ VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
+ return OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES * nr_pages / pages_per_compound;
+ }
+
+ VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
+ /* Ensure the requested range does not cross a compound page boundary. */
+ VM_WARN_ON_ONCE((pfn % pages_per_compound) + nr_pages > pages_per_compound);
+
+ if (IS_ALIGNED(pfn, pages_per_compound))
+ return OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES;
+
+ return 0;
+}
+
/*
* mm/sparse.c
*/
@@ -988,8 +1011,6 @@ static inline void __section_mark_present(struct mem_section *ms,
ms->section_mem_map |= SECTION_MARKED_PRESENT;
}
-int vmemmap_nr_struct_pages(unsigned long pfn, unsigned long nr_pages);
-
static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages)
{
VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
@@ -236,29 +236,6 @@ void __weak __meminit vmemmap_populate_print_last(void)
{
}
-int __meminit vmemmap_nr_struct_pages(unsigned long pfn, unsigned long nr_pages)
-{
- const unsigned int order = pfn_to_section_order(pfn);
- const unsigned long pages_per_compound = 1UL << order;
-
- if (!order_vmemmap_optimizable(order))
- return nr_pages;
-
- if (order < PFN_SECTION_SHIFT) {
- VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
- return OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES * nr_pages / pages_per_compound;
- }
-
- VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
- /* Ensure the requested range does not cross a compound page boundary. */
- VM_WARN_ON_ONCE((pfn % pages_per_compound) + nr_pages > pages_per_compound);
-
- if (IS_ALIGNED(pfn, pages_per_compound))
- return OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES;
-
- return 0;
-}
-
/*
* Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
* And number of present sections in this node is map_count.
HugeTLB HVO is currently supported on x86-64, riscv64, and LoongArch. On x86-64 and riscv64, gigantic HugeTLB pages are larger than the section size, so the existing section-based vmemmap optimization infrastructure is already sufficient to cover the whole folio. On LoongArch, HugeTLB HVO is supported without gigantic HugeTLB pages. Therefore, boot-time HugeTLB HVO folios can rely on the section-based vmemmap optimization infrastructure directly, without the extra bulk optimization and fallback handling. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- mm/hugetlb.c | 25 ++++++------------------- mm/hugetlb_vmemmap.c | 21 ++++++--------------- mm/internal.h | 25 +++++++++++++++++++++++-- mm/sparse.c | 23 ----------------------- 4 files changed, 35 insertions(+), 59 deletions(-)