@@ -3126,33 +3126,8 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
return true;
}
-/* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
-static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
- struct hstate *h,
- unsigned long start_page_number,
- unsigned long end_page_number)
-{
- enum zone_type zone = folio_zonenum(folio);
- int nid = folio_nid(folio);
- struct page *page = folio_page(folio, start_page_number);
- unsigned long head_pfn = folio_pfn(folio);
- unsigned long pfn, end_pfn = head_pfn + end_page_number;
- unsigned int order = huge_page_order(h);
-
- /*
- * As we marked all tail pages with memblock_reserved_mark_noinit(),
- * we must initialize them ourselves here.
- */
- for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) {
- __init_single_page(page, pfn, zone, nid);
- prep_compound_tail(page, &folio->page, order);
- set_page_count(page, 0);
- }
-}
-
-static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
- struct hstate *h,
- unsigned long nr_pages)
+static void __init hugetlb_folio_init_vmemmap(struct page *head, unsigned long pfn,
+ enum zone_type zone, int nid, unsigned int order, unsigned int nr_pages)
{
int ret;
@@ -3161,12 +3136,19 @@ static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
* walking pages twice by initializing/preparing+freezing them in the
* same go.
*/
- __folio_clear_reserved(folio);
- __folio_set_head(folio);
- ret = folio_ref_freeze(folio, 1);
+ __ClearPageReserved(head);
+ ret = page_ref_freeze(head, 1);
VM_BUG_ON(!ret);
- hugetlb_folio_init_tail_vmemmap(folio, h, 1, nr_pages);
- prep_compound_head(&folio->page, huge_page_order(h));
+
+ __SetPageHead(head);
+ for (int i = 1; i < nr_pages; i++) {
+ struct page *page = head + i;
+
+ __init_single_page(page, pfn + i, zone, nid);
+ prep_compound_tail(page, head, order);
+ set_page_count(page, 0);
+ }
+ prep_compound_head(head, order);
}
/*
@@ -3226,6 +3208,7 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
struct folio *folio = (void *)page;
unsigned long pfn = PHYS_PFN(__pa(m));
unsigned long nr_pages = pages_per_huge_page(m->hstate);
+ enum zone_type zone = folio_zonenum(folio);
h = m->hstate;
/*
@@ -3239,7 +3222,8 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
VM_BUG_ON(!hstate_is_gigantic(h));
WARN_ON(folio_ref_count(folio) != 1);
- hugetlb_folio_init_vmemmap(folio, h, vmemmap_nr_struct_pages(pfn, nr_pages));
+ hugetlb_folio_init_vmemmap(page, pfn, zone, nid, huge_page_order(h),
+ vmemmap_nr_struct_pages(pfn, nr_pages));
init_new_hugetlb_folio(folio);
if (order_vmemmap_optimizable(pfn_to_section_order(pfn))) {
hugetlb_folio_init_vmemmap() currently splits the open-coded compound-page setup across two helpers even though the tail-page initialization is only used here. Fold the tail-page initialization into the main helper and pass the precomputed page metadata in from the caller. This makes the initialization flow easier to follow. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- mm/hugetlb.c | 50 +++++++++++++++++--------------------------------- 1 file changed, 17 insertions(+), 33 deletions(-)