@@ -745,6 +745,8 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
return true;
}
+static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn);
+
/*
* Initialize memmap section for a gigantic page, HVO-style.
*/
@@ -752,6 +754,7 @@ void __init hugetlb_vmemmap_init_early(int nid)
{
unsigned long psize, paddr, section_size;
unsigned long ns, i, pnum, pfn, nr_pages;
+ unsigned long start, end;
struct huge_bootmem_page *m = NULL;
void *map;
@@ -761,6 +764,8 @@ void __init hugetlb_vmemmap_init_early(int nid)
section_size = (1UL << PA_SECTION_SHIFT);
list_for_each_entry(m, &huge_boot_pages[nid], list) {
+ struct zone *zone;
+
if (!vmemmap_should_optimize_bootmem_page(m))
continue;
@@ -769,6 +774,14 @@ void __init hugetlb_vmemmap_init_early(int nid)
paddr = virt_to_phys(m);
pfn = PHYS_PFN(paddr);
map = pfn_to_page(pfn);
+ start = (unsigned long)map;
+ end = start + hugetlb_vmemmap_size(m->hstate);
+ zone = pfn_to_zone(nid, pfn);
+
+ if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
+ zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
+ panic("Failed to allocate memmap for HugeTLB page\n");
+ memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
pnum = pfn_to_section_nr(pfn);
ns = psize / section_size;
@@ -800,60 +813,6 @@ static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
void __init hugetlb_vmemmap_init_late(int nid)
{
- struct huge_bootmem_page *m, *tm;
- unsigned long phys, nr_pages, start, end;
- unsigned long pfn, nr_mmap;
- struct zone *zone = NULL;
- struct hstate *h;
- void *map;
-
- if (!READ_ONCE(vmemmap_optimize_enabled))
- return;
-
- list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
- if (!(m->flags & HUGE_BOOTMEM_HVO))
- continue;
-
- phys = virt_to_phys(m);
- h = m->hstate;
- pfn = PHYS_PFN(phys);
- nr_pages = pages_per_huge_page(h);
- map = pfn_to_page(pfn);
- start = (unsigned long)map;
- end = start + nr_pages * sizeof(struct page);
-
- if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
- /*
- * Oops, the hugetlb page spans multiple zones.
- * Remove it from the list, and populate it normally.
- */
- list_del(&m->list);
-
- vmemmap_populate(start, end, nid, NULL);
- nr_mmap = end - start;
- memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
-
- memblock_phys_free(phys, huge_page_size(h));
- continue;
- }
-
- if (!zone || !zone_spans_pfn(zone, pfn))
- zone = pfn_to_zone(nid, pfn);
- if (WARN_ON_ONCE(!zone))
- continue;
-
- if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone,
- HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) {
- /* Fallback if HVO population fails */
- vmemmap_populate(start, end, nid, NULL);
- nr_mmap = end - start;
- } else {
- m->flags |= HUGE_BOOTMEM_ZONES_VALID;
- nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE;
- }
-
- memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
- }
}
#endif
Bootmem HugeTLB pages currently defer HVO setup to hugetlb_vmemmap_init_late(), because the optimization needs zone information. Now that zone initialization is available earlier, the bootmem HVO setup can be done directly from hugetlb_vmemmap_init_early(). This lets gigantic HugeTLB pages apply HVO as soon as they are allocated. Bootmem gigantic pages that span multiple zones are now filtered out when they are allocated, so the remaining bootmem gigantic pages seen by later hugetlb initialization are already zone-valid. As a result, hugetlb_vmemmap_init_late() no longer needs to handle bootmem HVO setup. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- mm/hugetlb_vmemmap.c | 67 +++++++++----------------------------------- 1 file changed, 13 insertions(+), 54 deletions(-)