@@ -152,6 +152,7 @@ struct scatterlist;
struct pipe_inode_info;
struct iov_iter;
struct napi_struct;
+struct qmempool;
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
struct nf_conntrack {
@@ -557,8 +558,8 @@ struct sk_buff {
fclone:2,
peeked:1,
head_frag:1,
+ qmempool:1,
xmit_more:1;
- /* one bit hole */
kmemcheck_bitfield_end(flags1);
/* fields enclosed in headers_start/headers_end are copied
@@ -755,6 +756,7 @@ void skb_tx_error(struct sk_buff *skb);
void consume_skb(struct sk_buff *skb);
void __kfree_skb(struct sk_buff *skb);
extern struct kmem_cache *skbuff_head_cache;
+extern struct qmempool *skbuff_head_pool;
void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
@@ -135,6 +135,7 @@
#include <linux/if_macvlan.h>
#include <linux/errqueue.h>
#include <linux/hrtimer.h>
+#include <linux/qmempool.h>
#include "net-sysfs.h"
@@ -4125,7 +4126,9 @@ static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
case GRO_MERGED_FREE:
if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
- kmem_cache_free(skbuff_head_cache, skb);
+ (skb->qmempool) ?
+ qmempool_free(skbuff_head_pool, skb) :
+ kmem_cache_free(skbuff_head_cache, skb);
else
__kfree_skb(skb);
break;
@@ -74,10 +74,24 @@
#include <asm/uaccess.h>
#include <trace/events/skb.h>
#include <linux/highmem.h>
+#include <linux/qmempool.h>
+#include <linux/log2.h>
struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;
+/* Keep max 32 skbs per CPU = 8192 bytes per CPU (as skb is
+ * SLAB_HWCACHE_ALIGN). Sharedq cache is limited to max 1024 elems
+ * which is max 262KB skb memory, on small systems it allocs 32*2
+ * elems * NR_CPUS.
+ */
+struct qmempool *skbuff_head_pool __read_mostly;
+#define QMEMPOOL_LOCALQ 32
+#define QMEMPOOL_SCALE (QMEMPOOL_LOCALQ * 2)
+#define QMEMPOOL_SYSTEM_SIZE roundup_pow_of_two(NR_CPUS * QMEMPOOL_SCALE)
+#define QMEMPOOL_SHAREDQ min(1024UL, QMEMPOOL_SYSTEM_SIZE)
+#define QMEMPOOL_PREALLOC 0
+
/**
* skb_panic - private function for out-of-line support
* @skb: buffer
@@ -278,13 +292,14 @@ nodata:
EXPORT_SYMBOL(__alloc_skb);
/**
- * build_skb - build a network buffer
+ * __build_skb - build a network buffer
* @data: data buffer provided by caller
* @frag_size: size of fragment, or 0 if head was kmalloced
*
* Allocate a new &sk_buff. Caller provides space holding head and
* skb_shared_info. @data must have been allocated by kmalloc() only if
* @frag_size is 0, otherwise data should come from the page allocator.
+ * @flags: FIXME-DESCRIBE
* The return is the new skb buffer.
* On a failure the return is %NULL, and @data is not freed.
* Notes :
@@ -295,13 +310,16 @@ EXPORT_SYMBOL(__alloc_skb);
* before giving packet to stack.
* RX rings only contains data buffers, not full skbs.
*/
-struct sk_buff *build_skb(void *data, unsigned int frag_size)
+static struct sk_buff *__build_skb(void *data, unsigned int frag_size,
+ int flags)
{
struct skb_shared_info *shinfo;
struct sk_buff *skb;
unsigned int size = frag_size ? : ksize(data);
- skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
+ skb = (flags & SKB_ALLOC_NAPI) ?
+ qmempool_alloc_softirq(skbuff_head_pool, GFP_ATOMIC) :
+ kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
if (!skb)
return NULL;
@@ -310,6 +328,7 @@ struct sk_buff *build_skb(void *data, unsigned int frag_size)
memset(skb, 0, offsetof(struct sk_buff, tail));
skb->truesize = SKB_TRUESIZE(size);
skb->head_frag = frag_size != 0;
+ skb->qmempool = !!(flags & SKB_ALLOC_NAPI);
atomic_set(&skb->users, 1);
skb->head = data;
skb->data = data;
@@ -326,6 +345,11 @@ struct sk_buff *build_skb(void *data, unsigned int frag_size)
return skb;
}
+
+struct sk_buff *build_skb(void *data, unsigned int frag_size)
+{
+ return __build_skb(data, frag_size, 0);
+}
EXPORT_SYMBOL(build_skb);
struct netdev_alloc_cache {
@@ -477,7 +501,7 @@ static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask,
__netdev_alloc_frag(fragsz, gfp_mask);
if (likely(data)) {
- skb = build_skb(data, fragsz);
+ skb = __build_skb(data, fragsz, flags);
if (unlikely(!skb))
put_page(virt_to_head_page(data));
}
@@ -637,7 +661,9 @@ static void kfree_skbmem(struct sk_buff *skb)
switch (skb->fclone) {
case SKB_FCLONE_UNAVAILABLE:
- kmem_cache_free(skbuff_head_cache, skb);
+ (skb->qmempool) ?
+ qmempool_free(skbuff_head_pool, skb) :
+ kmem_cache_free(skbuff_head_cache, skb);
return;
case SKB_FCLONE_ORIG:
@@ -862,6 +888,7 @@ static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
C(end);
C(head);
C(head_frag);
+ C(qmempool);
C(data);
C(truesize);
atomic_set(&n->users, 1);
@@ -3370,6 +3397,12 @@ void __init skb_init(void)
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
+ /* connect qmempools to slabs */
+ skbuff_head_pool = qmempool_create(QMEMPOOL_LOCALQ,
+ QMEMPOOL_SHAREDQ,
+ QMEMPOOL_PREALLOC,
+ skbuff_head_cache, GFP_ATOMIC);
+ BUG_ON(skbuff_head_pool == NULL);
}
/**
This patch uses qmempool for faster than SLAB caching of SKBs. Only use this caching in connection with napi_alloc_skb() which runs in softirq context. This softirq context provides the needed protection for qmempool and the underlying alf_queue. Current caching settings are max 32 elements per CPU, which is 8192 bytes given SKB is SLAB_HWCACHE_ALIGN'ed. The shared queue max limit is 1024 which corresponds to worst-case 263KB memory usage. Systems with a NR_CPUS <= 8 will get a smaller max shared queue. Benchmarked on a E5-2695 12-cores (no-HT) with ixgbe. Baseline is Alex'es napi_alloc_skb patchset. Single flow/CPU, early drop in iptables RAW table (fast path compare): * baseline: 3,159,160 pps * qmempool: 3,282,508 pps - Diff to baseline: +123348 pps => -11.89 ns IP-forward single flow/cpu (slower path compare): * baseline: 1,137,284 pps * qmempool: 1,191,856 pps - Diff to baseline: +54572 pps => -40.26 ns Some of the improvements also come from qmempool_{alloc,free} have smaller code size than kmem_cache_{alloc,free}, which helps reduce instruction-cache misses. Also did some scaling tests, to stress qmempool sharedq allocs (which stress the alf_queue's concurrency). IP-forward MULTI flow/cpu (12 CPUs E5-2695 no-HT, 12 HWQs): * baseline: 11,946,666 pps * qmempool: 11,988,042 pps - Diff to baseline: +41376 pps => -0.29 ns Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com> --- include/linux/skbuff.h | 4 +++- net/core/dev.c | 5 ++++- net/core/skbuff.c | 43 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 45 insertions(+), 7 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html