@@ -4863,8 +4863,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap);
int vmemmap_populate(unsigned long start, unsigned long end, int node,
struct vmem_altmap *altmap);
-void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
- unsigned long headsize);
void vmemmap_populate_print_last(void);
struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone);
#ifdef CONFIG_MEMORY_HOTPLUG
@@ -3202,7 +3202,7 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h,
struct folio *folio, *tmp_f;
/* Send list for bulk vmemmap optimization processing */
- hugetlb_vmemmap_optimize_bootmem_folios(h, folio_list);
+ hugetlb_vmemmap_optimize_folios(h, folio_list);
list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
@@ -595,31 +595,22 @@ static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *fol
return vmemmap_remap_split(vmemmap_start, vmemmap_end);
}
-static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
- struct list_head *folio_list,
- bool boot)
+void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
{
struct folio *folio;
- int nr_to_optimize;
+ unsigned long nr_to_optimize = 0;
LIST_HEAD(vmemmap_pages);
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
- nr_to_optimize = 0;
list_for_each_entry(folio, folio_list, lru) {
int ret;
- unsigned long spfn, epfn;
-
- if (boot && folio_test_hugetlb_vmemmap_optimized(folio)) {
- /*
- * Already optimized by pre-HVO, just map the
- * mirrored tail page structs RO.
- */
- spfn = (unsigned long)&folio->page;
- epfn = spfn + hugetlb_vmemmap_size(h);
- vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
- OPTIMIZED_FOLIO_VMEMMAP_SIZE);
+
+ /*
+ * Bootmem gigantic folios may already be marked optimized when
+ * their vmemmap layout was prepared earlier, so skip them here.
+ */
+ if (folio_test_hugetlb_vmemmap_optimized(folio))
continue;
- }
nr_to_optimize++;
@@ -636,14 +627,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
}
if (!nr_to_optimize)
- /*
- * All pre-HVO folios, nothing left to do. It's ok if
- * there is a mix of pre-HVO and not yet HVO-ed folios
- * here, as __hugetlb_vmemmap_optimize_folio() will
- * skip any folios that already have the optimized flag
- * set, see vmemmap_should_optimize_folio().
- */
- goto out;
+ return;
flush_tlb_all();
@@ -668,21 +652,10 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
}
}
-out:
flush_tlb_all();
free_vmemmap_page_list(&vmemmap_pages);
}
-void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
-{
- __hugetlb_vmemmap_optimize_folios(h, folio_list, false);
-}
-
-void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list)
-{
- __hugetlb_vmemmap_optimize_folios(h, folio_list, true);
-}
-
void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m)
{
struct hstate *h = m->hstate;
@@ -17,7 +17,6 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
struct list_head *non_hvo_folios);
void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio);
void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);
-void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m);
static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
@@ -59,11 +58,6 @@ static inline void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list
{
}
-static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h,
- struct list_head *folio_list)
-{
-}
-
static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
return 0;
@@ -265,29 +265,6 @@ int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
return 0;
}
-/*
- * Write protect the mirrored tail page structs for HVO. This will be
- * called from the hugetlb code when gathering and initializing the
- * memblock allocated gigantic pages. The write protect can't be
- * done earlier, since it can't be guaranteed that the reserved
- * page structures will not be written to during initialization,
- * even if CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled.
- *
- * The PTEs are known to exist, and nothing else should be touching
- * these pages. The caller is responsible for any TLB flushing.
- */
-void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end,
- int node, unsigned long headsize)
-{
- unsigned long maddr;
- pte_t *pte;
-
- for (maddr = addr + headsize; maddr < end; maddr += PAGE_SIZE) {
- pte = virt_to_kpte(maddr);
- ptep_set_wrprotect(&init_mm, maddr, pte);
- }
-}
-
struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone)
{
void *addr;
Shared vmemmap tail pages are now mapped read-only when their PTEs are installed, so HugeTLB bootmem optimization no longer needs a separate write-protect pass afterwards. Remove vmemmap_wrprotect_hvo() and the bootmem-specific HugeTLB wrapper, and let bootmem folios use the normal hugetlb_vmemmap_optimize_folios() path. Signed-off-by: Muchun Song <songmuchun@bytedance.com> --- include/linux/mm.h | 2 -- mm/hugetlb.c | 2 +- mm/hugetlb_vmemmap.c | 45 +++++++++----------------------------------- mm/hugetlb_vmemmap.h | 6 ------ mm/sparse-vmemmap.c | 23 ---------------------- 5 files changed, 10 insertions(+), 68 deletions(-)