@@ -1154,6 +1154,9 @@ struct softnet_data
struct sk_buff *completion_queue;
struct napi_struct backlog;
+
+ struct page *alloc_skb_page[2];
+ unsigned int alloc_skb_off[2];
};
DECLARE_PER_CPU(struct softnet_data,softnet_data);
@@ -144,7 +144,8 @@ struct skb_shared_info {
unsigned short gso_size;
/* Warning: this field is not always filled in (UFO)! */
unsigned short gso_segs;
- unsigned short gso_type;
+ __u8 gso_type;
+ __u8 alloc_paged;
__be32 ip6_frag_id;
#ifdef CONFIG_HAS_DMA
unsigned int num_dma_maps;
@@ -5243,6 +5243,9 @@ static int __init net_dev_init(void)
queue->backlog.poll = process_backlog;
queue->backlog.weight = weight_p;
queue->backlog.gro_list = NULL;
+
+ queue->alloc_skb_page[0] = NULL;
+ queue->alloc_skb_page[1] = NULL;
}
dev_boot_phase = 0;
@@ -151,6 +151,55 @@ void skb_truesize_bug(struct sk_buff *sk
}
EXPORT_SYMBOL(skb_truesize_bug);
+static inline void *alloc_paged(unsigned int size, gfp_t gfp_mask)
+{
+ struct softnet_data *sd;
+ unsigned long flags;
+ unsigned int off;
+ struct page *p;
+ void *ret;
+ int i;
+
+ if (size < 1400 || size > 2000)
+ return NULL;
+
+ if (gfp_mask == GFP_ATOMIC)
+ i = 0;
+ else if (gfp_mask == GFP_KERNEL)
+ i = 1;
+ else
+ return NULL;
+
+ local_irq_save(flags);
+ sd = &__get_cpu_var(softnet_data);
+ p = sd->alloc_skb_page[i];
+
+ if (p) {
+ off = sd->alloc_skb_off[i];
+ if (off + size > PAGE_SIZE) {
+ put_page(p);
+ goto new_page;
+ }
+ } else {
+new_page:
+ p = sd->alloc_skb_page[i] = alloc_pages(gfp_mask, 0);
+ if (!p) {
+ ret = NULL;
+ goto out;
+ }
+
+ off = 0;
+ /* hold one ref to this page until it's full */
+ }
+
+ get_page(p);
+ ret = page_address(p) + off;
+ sd->alloc_skb_off[i] = off + size;
+out:
+ local_irq_restore(flags);
+ return ret;
+}
+
/* Allocate a new skbuff. We do this ourselves so we can fill in a few
* 'private' fields and also do memory statistics to find all the
* [BEEP] leaks.
@@ -178,7 +227,7 @@ struct sk_buff *__alloc_skb(unsigned int
struct kmem_cache *cache;
struct skb_shared_info *shinfo;
struct sk_buff *skb;
- u8 *data;
+ u8 *data, paged = 0;
cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
@@ -188,8 +237,13 @@ struct sk_buff *__alloc_skb(unsigned int
goto out;
size = SKB_DATA_ALIGN(size);
- data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
- gfp_mask, node);
+ data = alloc_paged(size + sizeof(struct skb_shared_info), gfp_mask);
+ if (data)
+ paged = 1;
+ else
+ data = kmalloc_node_track_caller(size +
+ sizeof(struct skb_shared_info),
+ gfp_mask, node);
if (!data)
goto nodata;
@@ -214,6 +268,7 @@ struct sk_buff *__alloc_skb(unsigned int
shinfo->gso_type = 0;
shinfo->ip6_frag_id = 0;
shinfo->frag_list = NULL;
+ shinfo->alloc_paged = paged;
if (fclone) {
struct sk_buff *child = skb + 1;
@@ -341,7 +396,10 @@ static void skb_release_data(struct sk_b
if (skb_shinfo(skb)->frag_list)
skb_drop_fraglist(skb);
- kfree(skb->head);
+ if (skb_shinfo(skb)->alloc_paged)
+ put_page(virt_to_page(skb->head));
+ else
+ kfree(skb->head);
}
}
@@ -1380,7 +1438,7 @@ static inline int spd_fill_page(struct s
if (unlikely(spd->nr_pages == PIPE_BUFFERS))
return 1;
- if (linear) {
+ if (linear && !skb_shinfo(skb)->alloc_paged) {
page = linear_to_page(page, len, &offset, skb);
if (!page)
return 1;