@@ -4866,6 +4866,7 @@ int vmemmap_populate(unsigned long start, unsigned long end, int node,
void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
unsigned long headsize);
void vmemmap_populate_print_last(void);
+struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone);
#ifdef CONFIG_MEMORY_HOTPLUG
void vmemmap_free(unsigned long start, unsigned long end,
struct vmem_altmap *altmap);
@@ -499,32 +499,6 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *
return vmemmap_should_optimize(h);
}
-static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
-{
- const unsigned int idx = order - OPTIMIZABLE_FOLIO_MIN_ORDER;
- struct page *tail, *p;
- int node = zone_to_nid(zone);
-
- tail = READ_ONCE(zone->vmemmap_tails[idx]);
- if (likely(tail))
- return tail;
-
- tail = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
- if (!tail)
- return NULL;
-
- p = page_to_virt(tail);
- for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++)
- init_compound_tail(p + i, NULL, order, zone);
-
- if (cmpxchg(&zone->vmemmap_tails[idx], NULL, tail)) {
- __free_page(tail);
- tail = READ_ONCE(zone->vmemmap_tails[idx]);
- }
-
- return tail;
-}
-
static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
struct folio *folio,
struct list_head *vmemmap_pages,
@@ -541,7 +515,7 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
return ret;
nid = folio_nid(folio);
- vmemmap_tail = vmemmap_get_tail(h->order, folio_zone(folio));
+ vmemmap_tail = vmemmap_shared_tail_page(h->order, folio_zone(folio));
if (!vmemmap_tail)
return -ENOMEM;
@@ -34,27 +34,13 @@
#include "internal.h"
-/*
- * Allocate a block of memory to be used to back the virtual memory map
- * or to back the page tables that are used to create the mapping.
- * Uses the main allocators if they are available, else bootmem.
- */
-
-static void * __ref __earlyonly_bootmem_alloc(int node,
- unsigned long size,
- unsigned long align,
- unsigned long goal)
-{
- return memmap_alloc(size, align, goal, node, false);
-}
-
-void * __meminit vmemmap_alloc_block(unsigned long size, int node)
+void __ref *vmemmap_alloc_block(unsigned long size, int node)
{
/* If the main allocator is up use that, fallback to bootmem. */
if (slab_is_available()) {
gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
int order = get_order(size);
- static bool warned __meminitdata;
+ static bool warned;
struct page *page;
page = alloc_pages_node(node, gfp_mask, order);
@@ -68,8 +54,7 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
}
return NULL;
} else
- return __earlyonly_bootmem_alloc(node, size, size,
- __pa(MAX_DMA_ADDRESS));
+ return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), node, false);
}
static void * __meminit altmap_alloc_block_buf(unsigned long size,
@@ -138,8 +123,6 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
start, end - 1);
}
-static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone);
-
static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
struct vmem_altmap *altmap,
unsigned long ptpfn)
@@ -158,7 +141,7 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in
if (WARN_ON_ONCE(!zone))
return NULL;
- page = vmemmap_get_tail(section_order(ms), zone);
+ page = vmemmap_shared_tail_page(section_order(ms), zone);
if (!page)
return NULL;
ptpfn = page_to_pfn(page);
@@ -190,7 +173,7 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in
return pte;
}
-static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
+static void *vmemmap_alloc_block_zero(unsigned long size, int node)
{
void *p = vmemmap_alloc_block(size, node);
@@ -329,32 +312,36 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end,
}
}
-static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
+struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone)
{
- struct page *p, *tail;
- unsigned int idx;
- int node = zone_to_nid(zone);
+ void *addr;
+ struct page *page;
+ const unsigned int idx = order - OPTIMIZABLE_FOLIO_MIN_ORDER;
- if (WARN_ON_ONCE(order < OPTIMIZABLE_FOLIO_MIN_ORDER))
- return NULL;
- if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER))
+ if (WARN_ON_ONCE(idx >= ARRAY_SIZE(zone->vmemmap_tails)))
return NULL;
- idx = order - OPTIMIZABLE_FOLIO_MIN_ORDER;
- tail = zone->vmemmap_tails[idx];
- if (tail)
- return tail;
+ page = READ_ONCE(zone->vmemmap_tails[idx]);
+ if (likely(page))
+ return page;
- p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
- if (!p)
+ addr = vmemmap_alloc_block_zero(PAGE_SIZE, zone_to_nid(zone));
+ if (!addr)
return NULL;
- for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++)
- init_compound_tail(p + i, NULL, order, zone);
- tail = virt_to_page(p);
- zone->vmemmap_tails[idx] = tail;
+ for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++)
+ init_compound_tail((struct page *)addr + i, NULL, order, zone);
+
+ page = virt_to_page(addr);
+ if (cmpxchg(&zone->vmemmap_tails[idx], NULL, page) != NULL) {
+ if (slab_is_available())
+ __free_page(page);
+ else
+ memblock_free(page_to_virt(page), PAGE_SIZE);
+ page = READ_ONCE(zone->vmemmap_tails[idx]);
+ }
- return tail;
+ return page;
}
void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
HugeTLB and sparse-vmemmap each have their own helper to allocate the shared tail page used by vmemmap optimization. Factor that logic into a common vmemmap_shared_tail_page() helper in sparse-vmemmap.c. It allocates the page through vmemmap_alloc_block_zero(), initializes the tail struct pages, and uses cmpxchg() to install the per-zone shared page. This removes duplicate allocation logic while still handling both the early boot and runtime paths through the same helper. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- include/linux/mm.h | 1 + mm/hugetlb_vmemmap.c | 28 +----------------- mm/sparse-vmemmap.c | 67 ++++++++++++++++++-------------------------- 3 files changed, 29 insertions(+), 67 deletions(-)