| Message ID | 20261003-b4-mmap-prepare-vma-flag-sanify-v4-7-a1f052500fd7@kernel.org |
|---|---|
| State | New |
| Headers | show |
| Series | mm: make VMA flag semantics explicit, eliminate VM_SPECIAL | expand |
On Sat, Oct 3, 2026 at 6:35 PM Lorenzo Stoakes (ARM) <ljs@kernel.org> wrote: > > The existing kernel page mapping mmap actions allow for partial and full > mapping of an array of struct page pointers. > > However some drivers require the mapping of discontiguous ranges. Permit > this by providing discontig_kernel_page_ops which allows a driver to > specify how the operation should begin and how batches of pages should be > retrieved. > > It uses the minimum exposed interface to do so, providing address, page > offset and both vm_private_data state and a local private state object. > > ops->init can establish state for the operation, and ops->get outputs the > pages to map and their count. Should an error arise the core unmaps the > VMA, invoking vm_ops->close, which is therefore where any state established > by ops->init is released. > > Also update the compatibility layer to do this too via > compat_mmap_action_abort(). > > Batches may not exceed the VMA, but may map less than its full range in > case the driver wishes to allow the user to map an area larger than the > available data. > > To use it, users invoke mmap_action_map_discontig_kernel_pages() with > initial local private state and a set of operations. > > Users can then use one of the provided helper functions to perform an > action: > > * discontig_kernel_map_abort() - Abort and leave the mapping as it has > been accumulated so far. > * discontig_kernel_map_page() - Map a single page, or a compound page given > its head page. > * discontig_kernel_map_page_range() - Maps a struct page ** array of a > specified count, mapping only those pages which fit in the VMA. > > The userland VMA tests are updated accordingly. > > Acked-by: Zi Yan <ziy@nvidia.com> > Signed-off-by: Lorenzo Stoakes (ARM) <ljs@kernel.org> > --- > include/linux/mm.h | 45 +++++++++++++++++ > include/linux/mm_types.h | 44 +++++++++++++++- > mm/internal.h | 3 ++ > mm/memory.c | 108 ++++++++++++++++++++++++++++++++++++++-- > mm/util.c | 27 ++++++++-- > tools/testing/vma/include/dup.h | 11 +++- > 6 files changed, 224 insertions(+), 14 deletions(-) > > diff --git a/include/linux/mm.h b/include/linux/mm.h > index a1f2d375cf7d..97e1eb4fe3a4 100644 > --- a/include/linux/mm.h > +++ b/include/linux/mm.h > @@ -4647,10 +4647,55 @@ static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, > vma_desc_pages(desc)); > } > > +static inline > +void mmap_action_map_discontig_kernel_pages(struct vm_area_desc *desc, > + void *init_private, const struct discontig_kernel_page_ops *ops) > +{ > + struct mmap_action *action = &desc->action; > + > + action->type = MMAP_DISCONTIG_KERNEL_PAGES; > + action->map_kernel_discontig.init_private = init_private; > + action->map_kernel_discontig.ops = ops; > +} > + > int mmap_action_prepare(struct vm_area_desc *desc); > int mmap_action_complete(struct vm_area_struct *vma, > struct mmap_action *action, bool is_compat); > > +static inline void > +discontig_kernel_map_abort(struct discontig_kernel_page_state *state) > +{ > + state->action = DISCONTIG_KERNEL_PAGE_ABORT; > +} > + > +static inline void > +discontig_kernel_map_page(struct discontig_kernel_page_state *state, > + struct page *page) > +{ > + struct folio *folio = page_folio(page); > + > + if (folio_test_large(folio)) { > + VM_WARN_ON_ONCE(page != folio_page(folio, 0)); > + state->action = DISCONTIG_KERNEL_PAGE_MAP_COMPOUND_PAGE; > + state->__folio = folio; > + state->__nr_pages = min(state->nr_pages_remain, > + folio_nr_pages(folio)); > + } else { > + state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGE; > + state->__page = page; > + state->__nr_pages = 1; > + } > +} > + > +static inline void > +discontig_kernel_map_page_range(struct discontig_kernel_page_state *state, > + struct page **page_arr, unsigned long nr_pages) > +{ > + state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE; > + state->__page_arr = page_arr; > + state->__nr_pages = min(nr_pages, state->nr_pages_remain); > +} > + > /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ > static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, > unsigned long vm_start, unsigned long vm_end) > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h > index 9ca2ea3664bc..0cb4f9603956 100644 > --- a/include/linux/mm_types.h > +++ b/include/linux/mm_types.h > @@ -818,8 +818,44 @@ enum mmap_action_type { > MMAP_NOTHING, > MMAP_REMAP_PFN, > MMAP_IO_REMAP_PFN, > - MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ > - MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ > + MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ > + MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ > + MMAP_DISCONTIG_KERNEL_PAGES, /* Map kernel discontig page range. */ > +}; > + > +enum discontig_kernel_page_action { > + DISCONTIG_KERNEL_PAGE_ABORT, > + DISCONTIG_KERNEL_PAGE_MAP_PAGE, > + DISCONTIG_KERNEL_PAGE_MAP_COMPOUND_PAGE, > + DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE, > +}; > + > +struct discontig_kernel_page_state { > + /* Map state. */ > + const unsigned long start; /* Start address of VMA. */ > + const unsigned long end; /* End address of VMA. */ > + unsigned long addr; /* The current address to be mapped. */ > + pgoff_t pgoff; /* The current pgoff to be mapped. */ > + unsigned long nr_pages_mapped; /* The number of pages mapped. */ > + unsigned long nr_pages_remain; /* The number of pages remaining. */ > + > + /* User-defined state. */ > + void *vm_private_data; /* VMA private data. */ > + void *private; /* Mapping private data. */ > + > + /* Users should not touch these, use discontig_kernel_map_*() helpers. */ > + enum discontig_kernel_page_action action; > + union { > + struct page *__page; > + struct folio *__folio; > + struct page **__page_arr; > + }; > + unsigned long __nr_pages; > +}; > + > +struct discontig_kernel_page_ops { > + int (*init)(void *vm_private_data, void **private); > + int (*get)(struct discontig_kernel_page_state *state); > }; > > /* > @@ -844,6 +880,10 @@ struct mmap_action { > unsigned long nr_pages; > pgoff_t pgoff; > } map_kernel; > + struct { > + void *init_private; > + const struct discontig_kernel_page_ops *ops; > + } map_kernel_discontig; > }; > enum mmap_action_type type; > > diff --git a/mm/internal.h b/mm/internal.h > index fc4e249e2343..557e629db8c5 100644 > --- a/mm/internal.h > +++ b/mm/internal.h > @@ -1515,6 +1515,9 @@ int simple_ioremap_prepare(struct vm_area_desc *desc); > int map_kernel_pages_prepare(struct vm_area_desc *desc); > int map_kernel_pages_complete(struct vm_area_struct *vma, > struct mmap_action *action); > +int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc); > +int map_discontig_kernel_pages_complete(struct vm_area_struct *vma, > + struct mmap_action *action); > > static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) > { > diff --git a/mm/memory.c b/mm/memory.c > index 448342883e9d..45b21bb04a18 100644 > --- a/mm/memory.c > +++ b/mm/memory.c > @@ -2609,17 +2609,23 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, > } > EXPORT_SYMBOL(vm_insert_pages); > > +static void __map_kernel_pages_prepare(struct vm_area_desc *desc) > +{ > + if (vma_desc_test(desc, VMA_MIXEDMAP_BIT)) > + return; > + > + VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm)); Sorry I missed this before and it is pre-existing code but is this a weird way of asserting that mmap_lock is write-locked? If so, why not just use mmap_assert_write_locked()? > + VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT)); > + vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT); > +} > + > int map_kernel_pages_prepare(struct vm_area_desc *desc) > { > const struct mmap_action *action = &desc->action; > const unsigned long addr = action->map_kernel.start; > unsigned long nr_pages, end; > > - if (!vma_desc_test(desc, VMA_MIXEDMAP_BIT)) { > - VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm)); > - VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT)); > - vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT); > - } > + __map_kernel_pages_prepare(desc); > > nr_pages = action->map_kernel.nr_pages; > end = addr + PAGE_SIZE * nr_pages; > @@ -2640,6 +2646,98 @@ int map_kernel_pages_complete(struct vm_area_struct *vma, > &nr_pages, vma->vm_page_prot); > } > > +int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc) > +{ > + const struct mmap_action *action = &desc->action; > + const struct discontig_kernel_page_ops *ops = > + action->map_kernel_discontig.ops; > + > + /* At minimum need to be able to get pages. */ > + if (WARN_ON_ONCE(!ops || !ops->get)) > + return -EINVAL; > + > + __map_kernel_pages_prepare(desc); > + return 0; > +} > + > +static int apply_discontig_action(struct vm_area_struct *vma, > + struct discontig_kernel_page_state *state) > +{ > + unsigned long nr_pages = state->__nr_pages; > + unsigned long addr = state->addr; > + unsigned long i; > + > + if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGE) > + return insert_page(vma, addr, state->__page, > + vma->vm_page_prot, /*mkwrite=*/false); > + if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE) > + return insert_pages(vma, addr, state->__page_arr, > + &nr_pages, vma->vm_page_prot); > + > + /* Compound folio - have to iterate through each page. */ > + for (i = 0; i < nr_pages; i++, addr += PAGE_SIZE) { > + struct page *page = folio_page(state->__folio, i); > + int err; > + > + err = insert_page(vma, addr, page, vma->vm_page_prot, > + /*mkwrite=*/false); > + if (err) > + return err; > + } > + return 0; > +} > + > +int map_discontig_kernel_pages_complete(struct vm_area_struct *vma, > + struct mmap_action *action) > +{ > + const struct discontig_kernel_page_ops *ops = > + action->map_kernel_discontig.ops; > + struct discontig_kernel_page_state state = { > + .start = vma->vm_start, > + .end = vma->vm_end, > + .addr = vma->vm_start, > + .pgoff = vma->vm_pgoff, > + .nr_pages_mapped = 0, > + .nr_pages_remain = vma_pages(vma), > + .vm_private_data = vma->vm_private_data, > + .private = action->map_kernel_discontig.init_private, > + }; > + int err = 0; > + > + if (ops->init) > + err = ops->init(vma->vm_private_data, &state.private); > + if (err) > + return err; > + > + do { > + unsigned long end, pgoff_end; > + unsigned long nr_pages; > + > + /* Default to abort. */ > + state.action = DISCONTIG_KERNEL_PAGE_ABORT; > + err = ops->get(&state); > + if (err || state.action == DISCONTIG_KERNEL_PAGE_ABORT) > + return err; > + nr_pages = state.__nr_pages; > + > + if (!nr_pages || nr_pages > state.nr_pages_remain) > + return -EINVAL; > + end = state.addr + PAGE_SIZE * nr_pages; > + pgoff_end = state.pgoff + nr_pages; > + > + err = apply_discontig_action(vma, &state); > + if (err) > + return err; > + > + state.addr = end; > + state.pgoff = pgoff_end; > + state.nr_pages_mapped += nr_pages; > + state.nr_pages_remain -= nr_pages; > + } while (state.addr < vma->vm_end); > + > + return 0; > +} > + > /** > * vm_insert_page - insert single page into user vma > * @vma: user vma to map to > diff --git a/mm/util.c b/mm/util.c > index 1660910e2a22..5b2e1ab4aeba 100644 > --- a/mm/util.c > +++ b/mm/util.c > @@ -1401,6 +1401,15 @@ static int call_vma_mapped(struct vm_area_struct *vma) > return 0; > } > > +/* An mmap action failed under the compatibility layer - unmap and close. */ > +static void compat_mmap_action_abort(struct vm_area_struct *vma) > +{ > +#ifdef CONFIG_MMU > + zap_vma_range(vma, vma->vm_start, vma_pages(vma) << PAGE_SHIFT); > +#endif > + vma_close(vma); > +} > + > static int mmap_action_finish(struct vm_area_struct *vma, > struct mmap_action *action, int err, > bool is_compat) > @@ -1412,12 +1421,13 @@ static int mmap_action_finish(struct vm_area_struct *vma, > > /* do_munmap() might take rmap lock, so release if held. */ > maybe_rmap_unlock_action(vma, action); > - /* > - * If this is invoked from the compatibility layer, post-mmap() hook > - * logic will handle cleanup for us. > - */ > - if (!err || is_compat) > + if (!err) > + return 0; > + > + if (is_compat) { > + compat_mmap_action_abort(vma); > return err; > + } > > /* > * If an error occurs, unmap the VMA altogether and return an error. We > @@ -1469,6 +1479,8 @@ int mmap_action_prepare(struct vm_area_desc *desc) > return simple_ioremap_prepare(desc); > case MMAP_KERNEL_PAGES: > return map_kernel_pages_prepare(desc); > + case MMAP_DISCONTIG_KERNEL_PAGES: > + return map_discontig_kernel_pages_prepare(desc); > } > > WARN_ON_ONCE(1); > @@ -1501,6 +1513,9 @@ int mmap_action_complete(struct vm_area_struct *vma, > case MMAP_KERNEL_PAGES: > err = map_kernel_pages_complete(vma, action); > break; > + case MMAP_DISCONTIG_KERNEL_PAGES: > + err = map_discontig_kernel_pages_complete(vma, action); > + break; > case MMAP_IO_REMAP_PFN: > case MMAP_SIMPLE_IO_REMAP: > /* Should have been delegated. */ > @@ -1522,6 +1537,7 @@ int mmap_action_prepare(struct vm_area_desc *desc) > case MMAP_IO_REMAP_PFN: > case MMAP_SIMPLE_IO_REMAP: > case MMAP_KERNEL_PAGES: > + case MMAP_DISCONTIG_KERNEL_PAGES: > WARN_ON_ONCE(1); /* nommu cannot handle these. */ > break; > } > @@ -1543,6 +1559,7 @@ int mmap_action_complete(struct vm_area_struct *vma, > case MMAP_IO_REMAP_PFN: > case MMAP_SIMPLE_IO_REMAP: > case MMAP_KERNEL_PAGES: > + case MMAP_DISCONTIG_KERNEL_PAGES: > WARN_ON_ONCE(1); /* nommu cannot handle this. */ > > err = -EINVAL; > diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h > index 023ad6092e9c..a6a8e53c4b38 100644 > --- a/tools/testing/vma/include/dup.h > +++ b/tools/testing/vma/include/dup.h > @@ -457,14 +457,17 @@ enum mmap_action_type { > MMAP_NOTHING, > MMAP_REMAP_PFN, > MMAP_IO_REMAP_PFN, > - MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ > - MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ > + MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ > + MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ > + MMAP_DISCONTIG_KERNEL_PAGES, /* Map kernel discontig page range. */ > }; > > /* > * Describes an action an mmap_prepare hook can instruct to be taken to complete > * the mapping of a VMA. Specified in vm_area_desc. > */ > +struct discontig_kernel_page_ops; > + > struct mmap_action { > union { > struct { > @@ -483,6 +486,10 @@ struct mmap_action { > unsigned long nr_pages; > pgoff_t pgoff; > } map_kernel; > + struct { > + void *init_private; > + const struct discontig_kernel_page_ops *ops; > + } map_kernel_discontig; > }; > enum mmap_action_type type; > > > -- > 2.55.0 >
diff --git a/include/linux/mm.h b/include/linux/mm.h index a1f2d375cf7d..97e1eb4fe3a4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4647,10 +4647,55 @@ static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, vma_desc_pages(desc)); } +static inline +void mmap_action_map_discontig_kernel_pages(struct vm_area_desc *desc, + void *init_private, const struct discontig_kernel_page_ops *ops) +{ + struct mmap_action *action = &desc->action; + + action->type = MMAP_DISCONTIG_KERNEL_PAGES; + action->map_kernel_discontig.init_private = init_private; + action->map_kernel_discontig.ops = ops; +} + int mmap_action_prepare(struct vm_area_desc *desc); int mmap_action_complete(struct vm_area_struct *vma, struct mmap_action *action, bool is_compat); +static inline void +discontig_kernel_map_abort(struct discontig_kernel_page_state *state) +{ + state->action = DISCONTIG_KERNEL_PAGE_ABORT; +} + +static inline void +discontig_kernel_map_page(struct discontig_kernel_page_state *state, + struct page *page) +{ + struct folio *folio = page_folio(page); + + if (folio_test_large(folio)) { + VM_WARN_ON_ONCE(page != folio_page(folio, 0)); + state->action = DISCONTIG_KERNEL_PAGE_MAP_COMPOUND_PAGE; + state->__folio = folio; + state->__nr_pages = min(state->nr_pages_remain, + folio_nr_pages(folio)); + } else { + state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGE; + state->__page = page; + state->__nr_pages = 1; + } +} + +static inline void +discontig_kernel_map_page_range(struct discontig_kernel_page_state *state, + struct page **page_arr, unsigned long nr_pages) +{ + state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE; + state->__page_arr = page_arr; + state->__nr_pages = min(nr_pages, state->nr_pages_remain); +} + /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, unsigned long vm_start, unsigned long vm_end) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 9ca2ea3664bc..0cb4f9603956 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -818,8 +818,44 @@ enum mmap_action_type { MMAP_NOTHING, MMAP_REMAP_PFN, MMAP_IO_REMAP_PFN, - MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ - MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ + MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ + MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ + MMAP_DISCONTIG_KERNEL_PAGES, /* Map kernel discontig page range. */ +}; + +enum discontig_kernel_page_action { + DISCONTIG_KERNEL_PAGE_ABORT, + DISCONTIG_KERNEL_PAGE_MAP_PAGE, + DISCONTIG_KERNEL_PAGE_MAP_COMPOUND_PAGE, + DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE, +}; + +struct discontig_kernel_page_state { + /* Map state. */ + const unsigned long start; /* Start address of VMA. */ + const unsigned long end; /* End address of VMA. */ + unsigned long addr; /* The current address to be mapped. */ + pgoff_t pgoff; /* The current pgoff to be mapped. */ + unsigned long nr_pages_mapped; /* The number of pages mapped. */ + unsigned long nr_pages_remain; /* The number of pages remaining. */ + + /* User-defined state. */ + void *vm_private_data; /* VMA private data. */ + void *private; /* Mapping private data. */ + + /* Users should not touch these, use discontig_kernel_map_*() helpers. */ + enum discontig_kernel_page_action action; + union { + struct page *__page; + struct folio *__folio; + struct page **__page_arr; + }; + unsigned long __nr_pages; +}; + +struct discontig_kernel_page_ops { + int (*init)(void *vm_private_data, void **private); + int (*get)(struct discontig_kernel_page_state *state); }; /* @@ -844,6 +880,10 @@ struct mmap_action { unsigned long nr_pages; pgoff_t pgoff; } map_kernel; + struct { + void *init_private; + const struct discontig_kernel_page_ops *ops; + } map_kernel_discontig; }; enum mmap_action_type type; diff --git a/mm/internal.h b/mm/internal.h index fc4e249e2343..557e629db8c5 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1515,6 +1515,9 @@ int simple_ioremap_prepare(struct vm_area_desc *desc); int map_kernel_pages_prepare(struct vm_area_desc *desc); int map_kernel_pages_complete(struct vm_area_struct *vma, struct mmap_action *action); +int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc); +int map_discontig_kernel_pages_complete(struct vm_area_struct *vma, + struct mmap_action *action); static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) { diff --git a/mm/memory.c b/mm/memory.c index 448342883e9d..45b21bb04a18 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2609,17 +2609,23 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, } EXPORT_SYMBOL(vm_insert_pages); +static void __map_kernel_pages_prepare(struct vm_area_desc *desc) +{ + if (vma_desc_test(desc, VMA_MIXEDMAP_BIT)) + return; + + VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm)); + VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT)); + vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT); +} + int map_kernel_pages_prepare(struct vm_area_desc *desc) { const struct mmap_action *action = &desc->action; const unsigned long addr = action->map_kernel.start; unsigned long nr_pages, end; - if (!vma_desc_test(desc, VMA_MIXEDMAP_BIT)) { - VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm)); - VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT)); - vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT); - } + __map_kernel_pages_prepare(desc); nr_pages = action->map_kernel.nr_pages; end = addr + PAGE_SIZE * nr_pages; @@ -2640,6 +2646,98 @@ int map_kernel_pages_complete(struct vm_area_struct *vma, &nr_pages, vma->vm_page_prot); } +int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc) +{ + const struct mmap_action *action = &desc->action; + const struct discontig_kernel_page_ops *ops = + action->map_kernel_discontig.ops; + + /* At minimum need to be able to get pages. */ + if (WARN_ON_ONCE(!ops || !ops->get)) + return -EINVAL; + + __map_kernel_pages_prepare(desc); + return 0; +} + +static int apply_discontig_action(struct vm_area_struct *vma, + struct discontig_kernel_page_state *state) +{ + unsigned long nr_pages = state->__nr_pages; + unsigned long addr = state->addr; + unsigned long i; + + if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGE) + return insert_page(vma, addr, state->__page, + vma->vm_page_prot, /*mkwrite=*/false); + if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE) + return insert_pages(vma, addr, state->__page_arr, + &nr_pages, vma->vm_page_prot); + + /* Compound folio - have to iterate through each page. */ + for (i = 0; i < nr_pages; i++, addr += PAGE_SIZE) { + struct page *page = folio_page(state->__folio, i); + int err; + + err = insert_page(vma, addr, page, vma->vm_page_prot, + /*mkwrite=*/false); + if (err) + return err; + } + return 0; +} + +int map_discontig_kernel_pages_complete(struct vm_area_struct *vma, + struct mmap_action *action) +{ + const struct discontig_kernel_page_ops *ops = + action->map_kernel_discontig.ops; + struct discontig_kernel_page_state state = { + .start = vma->vm_start, + .end = vma->vm_end, + .addr = vma->vm_start, + .pgoff = vma->vm_pgoff, + .nr_pages_mapped = 0, + .nr_pages_remain = vma_pages(vma), + .vm_private_data = vma->vm_private_data, + .private = action->map_kernel_discontig.init_private, + }; + int err = 0; + + if (ops->init) + err = ops->init(vma->vm_private_data, &state.private); + if (err) + return err; + + do { + unsigned long end, pgoff_end; + unsigned long nr_pages; + + /* Default to abort. */ + state.action = DISCONTIG_KERNEL_PAGE_ABORT; + err = ops->get(&state); + if (err || state.action == DISCONTIG_KERNEL_PAGE_ABORT) + return err; + nr_pages = state.__nr_pages; + + if (!nr_pages || nr_pages > state.nr_pages_remain) + return -EINVAL; + end = state.addr + PAGE_SIZE * nr_pages; + pgoff_end = state.pgoff + nr_pages; + + err = apply_discontig_action(vma, &state); + if (err) + return err; + + state.addr = end; + state.pgoff = pgoff_end; + state.nr_pages_mapped += nr_pages; + state.nr_pages_remain -= nr_pages; + } while (state.addr < vma->vm_end); + + return 0; +} + /** * vm_insert_page - insert single page into user vma * @vma: user vma to map to diff --git a/mm/util.c b/mm/util.c index 1660910e2a22..5b2e1ab4aeba 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1401,6 +1401,15 @@ static int call_vma_mapped(struct vm_area_struct *vma) return 0; } +/* An mmap action failed under the compatibility layer - unmap and close. */ +static void compat_mmap_action_abort(struct vm_area_struct *vma) +{ +#ifdef CONFIG_MMU + zap_vma_range(vma, vma->vm_start, vma_pages(vma) << PAGE_SHIFT); +#endif + vma_close(vma); +} + static int mmap_action_finish(struct vm_area_struct *vma, struct mmap_action *action, int err, bool is_compat) @@ -1412,12 +1421,13 @@ static int mmap_action_finish(struct vm_area_struct *vma, /* do_munmap() might take rmap lock, so release if held. */ maybe_rmap_unlock_action(vma, action); - /* - * If this is invoked from the compatibility layer, post-mmap() hook - * logic will handle cleanup for us. - */ - if (!err || is_compat) + if (!err) + return 0; + + if (is_compat) { + compat_mmap_action_abort(vma); return err; + } /* * If an error occurs, unmap the VMA altogether and return an error. We @@ -1469,6 +1479,8 @@ int mmap_action_prepare(struct vm_area_desc *desc) return simple_ioremap_prepare(desc); case MMAP_KERNEL_PAGES: return map_kernel_pages_prepare(desc); + case MMAP_DISCONTIG_KERNEL_PAGES: + return map_discontig_kernel_pages_prepare(desc); } WARN_ON_ONCE(1); @@ -1501,6 +1513,9 @@ int mmap_action_complete(struct vm_area_struct *vma, case MMAP_KERNEL_PAGES: err = map_kernel_pages_complete(vma, action); break; + case MMAP_DISCONTIG_KERNEL_PAGES: + err = map_discontig_kernel_pages_complete(vma, action); + break; case MMAP_IO_REMAP_PFN: case MMAP_SIMPLE_IO_REMAP: /* Should have been delegated. */ @@ -1522,6 +1537,7 @@ int mmap_action_prepare(struct vm_area_desc *desc) case MMAP_IO_REMAP_PFN: case MMAP_SIMPLE_IO_REMAP: case MMAP_KERNEL_PAGES: + case MMAP_DISCONTIG_KERNEL_PAGES: WARN_ON_ONCE(1); /* nommu cannot handle these. */ break; } @@ -1543,6 +1559,7 @@ int mmap_action_complete(struct vm_area_struct *vma, case MMAP_IO_REMAP_PFN: case MMAP_SIMPLE_IO_REMAP: case MMAP_KERNEL_PAGES: + case MMAP_DISCONTIG_KERNEL_PAGES: WARN_ON_ONCE(1); /* nommu cannot handle this. */ err = -EINVAL; diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 023ad6092e9c..a6a8e53c4b38 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -457,14 +457,17 @@ enum mmap_action_type { MMAP_NOTHING, MMAP_REMAP_PFN, MMAP_IO_REMAP_PFN, - MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ - MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ + MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ + MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ + MMAP_DISCONTIG_KERNEL_PAGES, /* Map kernel discontig page range. */ }; /* * Describes an action an mmap_prepare hook can instruct to be taken to complete * the mapping of a VMA. Specified in vm_area_desc. */ +struct discontig_kernel_page_ops; + struct mmap_action { union { struct { @@ -483,6 +486,10 @@ struct mmap_action { unsigned long nr_pages; pgoff_t pgoff; } map_kernel; + struct { + void *init_private; + const struct discontig_kernel_page_ops *ops; + } map_kernel_discontig; }; enum mmap_action_type type;