| Message ID | 20191022044343.6901-4-saeedm@mellanox.com |
|---|---|
| State | Superseded |
| Delegated to: | David Miller |
| Headers | show |
| Series | page_pool: API for numa node change handling | expand |
On Tue, Oct 22, 2019 at 04:44:24AM +0000, Saeed Mahameed wrote: > From: Jonathan Lemon <jonathan.lemon@gmail.com> > > 1) Rename functions to reflect what they are actually doing. > > 2) Unify the condition to keep a page. > > 3) When page can't be kept in cache, fallback to releasing page to page > allocator in one place, instead of calling it from multiple conditions, > and reuse __page_pool_return_page(). > > Signed-off-by: Jonathan Lemon <jonathan.lemon@gmail.com> > Signed-off-by: Saeed Mahameed <saeedm@mellanox.com> > --- > net/core/page_pool.c | 38 +++++++++++++++++++------------------- > 1 file changed, 19 insertions(+), 19 deletions(-) > > diff --git a/net/core/page_pool.c b/net/core/page_pool.c > index 8120aec999ce..65680aaa0818 100644 > --- a/net/core/page_pool.c > +++ b/net/core/page_pool.c > @@ -258,6 +258,7 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool, > struct page *page) > { > int ret; > + > /* BH protection not needed if current is serving softirq */ > if (in_serving_softirq()) > ret = ptr_ring_produce(&pool->ring, page); > @@ -272,8 +273,8 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool, > * > * Caller must provide appropriate safe context. > */ > -static bool __page_pool_recycle_direct(struct page *page, > - struct page_pool *pool) > +static bool __page_pool_recycle_into_cache(struct page *page, > + struct page_pool *pool) > { > if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) > return false; > @@ -283,15 +284,18 @@ static bool __page_pool_recycle_direct(struct page *page, > return true; > } > > -/* page is NOT reusable when: > - * 1) allocated when system is under some pressure. (page_is_pfmemalloc) > - * 2) belongs to a different NUMA node than pool->p.nid. > +/* Keep page in caches only if page: > + * 1) wasn't allocated when system is under some pressure (page_is_pfmemalloc). > + * 2) belongs to pool's numa node (pool->p.nid). > + * 3) refcount is 1 (owned by page pool). > * > * To update pool->p.nid users must call page_pool_update_nid. > */ > -static bool pool_page_reusable(struct page_pool *pool, struct page *page) > +static bool page_pool_keep_page(struct page_pool *pool, struct page *page) > { > - return !page_is_pfmemalloc(page) && page_to_nid(page) == pool->p.nid; > + return !page_is_pfmemalloc(page) && > + page_to_nid(page) == pool->p.nid && > + page_ref_count(page) == 1; > } > > void __page_pool_put_page(struct page_pool *pool, > @@ -300,22 +304,19 @@ void __page_pool_put_page(struct page_pool *pool, > /* This allocator is optimized for the XDP mode that uses > * one-frame-per-page, but have fallbacks that act like the > * regular page allocator APIs. > - * > - * refcnt == 1 means page_pool owns page, and can recycle it. > */ > - if (likely(page_ref_count(page) == 1 && > - pool_page_reusable(pool, page))) { > + > + if (likely(page_pool_keep_page(pool, page))) { > /* Read barrier done in page_ref_count / READ_ONCE */ > > if (allow_direct && in_serving_softirq()) > - if (__page_pool_recycle_direct(page, pool)) > + if (__page_pool_recycle_into_cache(page, pool)) > return; > > - if (!__page_pool_recycle_into_ring(pool, page)) { > - /* Cache full, fallback to free pages */ > - __page_pool_return_page(pool, page); > - } > - return; > + if (__page_pool_recycle_into_ring(pool, page)) > + return; > + > + /* Cache full, fallback to return pages */ > } > /* Fallback/non-XDP mode: API user have elevated refcnt. > * > @@ -330,8 +331,7 @@ void __page_pool_put_page(struct page_pool *pool, > * doing refcnt based recycle tricks, meaning another process > * will be invoking put_page. > */ > - __page_pool_clean_page(pool, page); > - put_page(page); > + __page_pool_return_page(pool, page); I think Jesper had a reason for calling them separately instead of __page_pool_return_page + put_page() (which in fact does the same thing). In the future he was planning on removing the __page_pool_clean_page call from there, since someone might call __page_pool_put_page() after someone has called __page_pool_clean_page() Can we leave the calls there as-is? > } > EXPORT_SYMBOL(__page_pool_put_page); > > -- > 2.21.0 > Thanks /Ilias
On Wed, 23 Oct 2019 11:45:15 +0300 Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote: > On Tue, Oct 22, 2019 at 04:44:24AM +0000, Saeed Mahameed wrote: > > From: Jonathan Lemon <jonathan.lemon@gmail.com> > > > > 1) Rename functions to reflect what they are actually doing. > > > > 2) Unify the condition to keep a page. > > > > 3) When page can't be kept in cache, fallback to releasing page to page > > allocator in one place, instead of calling it from multiple conditions, > > and reuse __page_pool_return_page(). > > > > Signed-off-by: Jonathan Lemon <jonathan.lemon@gmail.com> > > Signed-off-by: Saeed Mahameed <saeedm@mellanox.com> > > --- > > net/core/page_pool.c | 38 +++++++++++++++++++------------------- > > 1 file changed, 19 insertions(+), 19 deletions(-) > > > > diff --git a/net/core/page_pool.c b/net/core/page_pool.c > > index 8120aec999ce..65680aaa0818 100644 > > --- a/net/core/page_pool.c > > +++ b/net/core/page_pool.c > > @@ -258,6 +258,7 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool, > > struct page *page) > > { > > int ret; > > + > > /* BH protection not needed if current is serving softirq */ > > if (in_serving_softirq()) > > ret = ptr_ring_produce(&pool->ring, page); > > @@ -272,8 +273,8 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool, > > * > > * Caller must provide appropriate safe context. > > */ > > -static bool __page_pool_recycle_direct(struct page *page, > > - struct page_pool *pool) > > +static bool __page_pool_recycle_into_cache(struct page *page, > > + struct page_pool *pool) > > { > > if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) > > return false; > > @@ -283,15 +284,18 @@ static bool __page_pool_recycle_direct(struct page *page, > > return true; > > } > > > > -/* page is NOT reusable when: > > - * 1) allocated when system is under some pressure. (page_is_pfmemalloc) > > - * 2) belongs to a different NUMA node than pool->p.nid. > > +/* Keep page in caches only if page: > > + * 1) wasn't allocated when system is under some pressure (page_is_pfmemalloc). > > + * 2) belongs to pool's numa node (pool->p.nid). > > + * 3) refcount is 1 (owned by page pool). > > * > > * To update pool->p.nid users must call page_pool_update_nid. > > */ > > -static bool pool_page_reusable(struct page_pool *pool, struct page *page) > > +static bool page_pool_keep_page(struct page_pool *pool, struct page *page) > > { > > - return !page_is_pfmemalloc(page) && page_to_nid(page) == pool->p.nid; > > + return !page_is_pfmemalloc(page) && > > + page_to_nid(page) == pool->p.nid && > > + page_ref_count(page) == 1; > > } > > > > void __page_pool_put_page(struct page_pool *pool, > > @@ -300,22 +304,19 @@ void __page_pool_put_page(struct page_pool *pool, > > /* This allocator is optimized for the XDP mode that uses > > * one-frame-per-page, but have fallbacks that act like the > > * regular page allocator APIs. > > - * > > - * refcnt == 1 means page_pool owns page, and can recycle it. > > */ > > - if (likely(page_ref_count(page) == 1 && > > - pool_page_reusable(pool, page))) { > > + > > + if (likely(page_pool_keep_page(pool, page))) { > > /* Read barrier done in page_ref_count / READ_ONCE */ > > > > if (allow_direct && in_serving_softirq()) > > - if (__page_pool_recycle_direct(page, pool)) > > + if (__page_pool_recycle_into_cache(page, pool)) > > return; > > > > - if (!__page_pool_recycle_into_ring(pool, page)) { > > - /* Cache full, fallback to free pages */ > > - __page_pool_return_page(pool, page); > > - } > > - return; > > + if (__page_pool_recycle_into_ring(pool, page)) > > + return; > > + > > + /* Cache full, fallback to return pages */ > > } > > /* Fallback/non-XDP mode: API user have elevated refcnt. > > * > > @@ -330,8 +331,7 @@ void __page_pool_put_page(struct page_pool *pool, > > * doing refcnt based recycle tricks, meaning another process > > * will be invoking put_page. > > */ > > - __page_pool_clean_page(pool, page); > > - put_page(page); > > + __page_pool_return_page(pool, page); > > I think Jesper had a reason for calling them separately instead of > __page_pool_return_page + put_page() (which in fact does the same thing). > > In the future he was planning on removing the __page_pool_clean_page call from > there, since someone might call __page_pool_put_page() after someone has called > __page_pool_clean_page() Yes. We need to work on removing this __page_pool_clean_page() call, to fulfill the plans of SKB returning/recycling page_pool pages. > Can we leave the calls there as-is? Yes, please.
On Wed, 2019-10-23 at 20:31 +0200, Jesper Dangaard Brouer wrote: > On Wed, 23 Oct 2019 11:45:15 +0300 > Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote: > > > On Tue, Oct 22, 2019 at 04:44:24AM +0000, Saeed Mahameed wrote: > > > From: Jonathan Lemon <jonathan.lemon@gmail.com> > > > > > > 1) Rename functions to reflect what they are actually doing. > > > > > > 2) Unify the condition to keep a page. > > > > > > 3) When page can't be kept in cache, fallback to releasing page > > > to page > > > allocator in one place, instead of calling it from multiple > > > conditions, > > > and reuse __page_pool_return_page(). > > > > > > Signed-off-by: Jonathan Lemon <jonathan.lemon@gmail.com> > > > Signed-off-by: Saeed Mahameed <saeedm@mellanox.com> > > > --- > > > net/core/page_pool.c | 38 +++++++++++++++++++------------------- > > > 1 file changed, 19 insertions(+), 19 deletions(-) > > > > > > diff --git a/net/core/page_pool.c b/net/core/page_pool.c > > > index 8120aec999ce..65680aaa0818 100644 > > > --- a/net/core/page_pool.c > > > +++ b/net/core/page_pool.c > > > @@ -258,6 +258,7 @@ static bool > > > __page_pool_recycle_into_ring(struct page_pool *pool, > > > struct page *page) > > > { > > > int ret; > > > + > > > /* BH protection not needed if current is serving softirq */ > > > if (in_serving_softirq()) > > > ret = ptr_ring_produce(&pool->ring, page); > > > @@ -272,8 +273,8 @@ static bool > > > __page_pool_recycle_into_ring(struct page_pool *pool, > > > * > > > * Caller must provide appropriate safe context. > > > */ > > > -static bool __page_pool_recycle_direct(struct page *page, > > > - struct page_pool *pool) > > > +static bool __page_pool_recycle_into_cache(struct page *page, > > > + struct page_pool *pool) > > > { > > > if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) > > > return false; > > > @@ -283,15 +284,18 @@ static bool > > > __page_pool_recycle_direct(struct page *page, > > > return true; > > > } > > > > > > -/* page is NOT reusable when: > > > - * 1) allocated when system is under some pressure. > > > (page_is_pfmemalloc) > > > - * 2) belongs to a different NUMA node than pool->p.nid. > > > +/* Keep page in caches only if page: > > > + * 1) wasn't allocated when system is under some pressure > > > (page_is_pfmemalloc). > > > + * 2) belongs to pool's numa node (pool->p.nid). > > > + * 3) refcount is 1 (owned by page pool). > > > * > > > * To update pool->p.nid users must call page_pool_update_nid. > > > */ > > > -static bool pool_page_reusable(struct page_pool *pool, struct > > > page *page) > > > +static bool page_pool_keep_page(struct page_pool *pool, struct > > > page *page) > > > { > > > - return !page_is_pfmemalloc(page) && page_to_nid(page) == pool- > > > >p.nid; > > > + return !page_is_pfmemalloc(page) && > > > + page_to_nid(page) == pool->p.nid && > > > + page_ref_count(page) == 1; > > > } > > > > > > void __page_pool_put_page(struct page_pool *pool, > > > @@ -300,22 +304,19 @@ void __page_pool_put_page(struct page_pool > > > *pool, > > > /* This allocator is optimized for the XDP mode that uses > > > * one-frame-per-page, but have fallbacks that act like the > > > * regular page allocator APIs. > > > - * > > > - * refcnt == 1 means page_pool owns page, and can recycle it. > > > */ > > > - if (likely(page_ref_count(page) == 1 && > > > - pool_page_reusable(pool, page))) { > > > + > > > + if (likely(page_pool_keep_page(pool, page))) { > > > /* Read barrier done in page_ref_count / READ_ONCE */ > > > > > > if (allow_direct && in_serving_softirq()) > > > - if (__page_pool_recycle_direct(page, pool)) > > > + if (__page_pool_recycle_into_cache(page, pool)) > > > return; > > > > > > - if (!__page_pool_recycle_into_ring(pool, page)) { > > > - /* Cache full, fallback to free pages */ > > > - __page_pool_return_page(pool, page); > > > - } > > > - return; > > > + if (__page_pool_recycle_into_ring(pool, page)) > > > + return; > > > + > > > + /* Cache full, fallback to return pages */ > > > } > > > /* Fallback/non-XDP mode: API user have elevated refcnt. > > > * > > > @@ -330,8 +331,7 @@ void __page_pool_put_page(struct page_pool > > > *pool, > > > * doing refcnt based recycle tricks, meaning another process > > > * will be invoking put_page. > > > */ > > > - __page_pool_clean_page(pool, page); > > > - put_page(page); > > > + __page_pool_return_page(pool, page); > > > > I think Jesper had a reason for calling them separately instead of > > __page_pool_return_page + put_page() (which in fact does the same > > thing). > > > > In the future he was planning on removing the > > __page_pool_clean_page call from > > there, since someone might call __page_pool_put_page() after > > someone has called > > __page_pool_clean_page() > > Yes. We need to work on removing this __page_pool_clean_page() > call, > to fulfill the plans of SKB returning/recycling page_pool pages. > > > Can we leave the calls there as-is? > > Yes, please. > Sure, i will drop this patch for now.
diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 8120aec999ce..65680aaa0818 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -258,6 +258,7 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool, struct page *page) { int ret; + /* BH protection not needed if current is serving softirq */ if (in_serving_softirq()) ret = ptr_ring_produce(&pool->ring, page); @@ -272,8 +273,8 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool, * * Caller must provide appropriate safe context. */ -static bool __page_pool_recycle_direct(struct page *page, - struct page_pool *pool) +static bool __page_pool_recycle_into_cache(struct page *page, + struct page_pool *pool) { if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) return false; @@ -283,15 +284,18 @@ static bool __page_pool_recycle_direct(struct page *page, return true; } -/* page is NOT reusable when: - * 1) allocated when system is under some pressure. (page_is_pfmemalloc) - * 2) belongs to a different NUMA node than pool->p.nid. +/* Keep page in caches only if page: + * 1) wasn't allocated when system is under some pressure (page_is_pfmemalloc). + * 2) belongs to pool's numa node (pool->p.nid). + * 3) refcount is 1 (owned by page pool). * * To update pool->p.nid users must call page_pool_update_nid. */ -static bool pool_page_reusable(struct page_pool *pool, struct page *page) +static bool page_pool_keep_page(struct page_pool *pool, struct page *page) { - return !page_is_pfmemalloc(page) && page_to_nid(page) == pool->p.nid; + return !page_is_pfmemalloc(page) && + page_to_nid(page) == pool->p.nid && + page_ref_count(page) == 1; } void __page_pool_put_page(struct page_pool *pool, @@ -300,22 +304,19 @@ void __page_pool_put_page(struct page_pool *pool, /* This allocator is optimized for the XDP mode that uses * one-frame-per-page, but have fallbacks that act like the * regular page allocator APIs. - * - * refcnt == 1 means page_pool owns page, and can recycle it. */ - if (likely(page_ref_count(page) == 1 && - pool_page_reusable(pool, page))) { + + if (likely(page_pool_keep_page(pool, page))) { /* Read barrier done in page_ref_count / READ_ONCE */ if (allow_direct && in_serving_softirq()) - if (__page_pool_recycle_direct(page, pool)) + if (__page_pool_recycle_into_cache(page, pool)) return; - if (!__page_pool_recycle_into_ring(pool, page)) { - /* Cache full, fallback to free pages */ - __page_pool_return_page(pool, page); - } - return; + if (__page_pool_recycle_into_ring(pool, page)) + return; + + /* Cache full, fallback to return pages */ } /* Fallback/non-XDP mode: API user have elevated refcnt. * @@ -330,8 +331,7 @@ void __page_pool_put_page(struct page_pool *pool, * doing refcnt based recycle tricks, meaning another process * will be invoking put_page. */ - __page_pool_clean_page(pool, page); - put_page(page); + __page_pool_return_page(pool, page); } EXPORT_SYMBOL(__page_pool_put_page);