@@ -190,6 +190,8 @@ struct sock_common {
* @sk_user_data: RPC layer private data
* @sk_sndmsg_page: cached page for sendmsg
* @sk_sndmsg_off: cached offset for sendmsg
+ * @sk_splice_page: cached page for splice
+ * @sk_splice_off: cached offset for splice
* @sk_send_head: front of stuff to transmit
* @sk_security: used by security modules
* @sk_mark: generic packet mark
@@ -279,6 +281,8 @@ struct sock {
struct page *sk_sndmsg_page;
struct sk_buff *sk_send_head;
__u32 sk_sndmsg_off;
+ struct page *sk_splice_page;
+ __u32 sk_splice_off;
int sk_write_pending;
#ifdef CONFIG_SECURITY
void *sk_security;
@@ -1334,13 +1334,33 @@ static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
}
static inline struct page *linear_to_page(struct page *page, unsigned int len,
- unsigned int offset)
+ unsigned int *offset,
+ struct sk_buff *skb)
{
- struct page *p = alloc_pages(GFP_KERNEL, 0);
+ struct sock *sk = skb->sk;
+ struct page *p = sk->sk_splice_page;
+ unsigned int off;
- if (!p)
- return NULL;
- memcpy(page_address(p) + offset, page_address(page) + offset, len);
+ if (!p) {
+new_page:
+ p = sk->sk_splice_page = alloc_pages(sk->sk_allocation, 0);
+ if (!p)
+ return NULL;
+
+ off = sk->sk_splice_off = 0;
+ /* hold this page until it's full or unneeded */
+ get_page(p);
+ } else {
+ off = sk->sk_splice_off;
+ if (off + len > PAGE_SIZE) {
+ put_page(p);
+ goto new_page;
+ }
+ }
+
+ memcpy(page_address(p) + off, page_address(page) + *offset, len);
+ sk->sk_splice_off += len;
+ *offset = off;
return p;
}
@@ -1356,7 +1376,7 @@ static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
return 1;
if (linear) {
- page = linear_to_page(page, len, offset);
+ page = linear_to_page(page, len, &offset, skb);
if (!page)
return 1;
} else
@@ -1732,6 +1732,8 @@ void sock_init_data(struct socket *sock, struct sock *sk)
sk->sk_sndmsg_page = NULL;
sk->sk_sndmsg_off = 0;
+ sk->sk_splice_page = NULL;
+ sk->sk_splice_off = 0;
sk->sk_peercred.pid = 0;
sk->sk_peercred.uid = -1;
@@ -1848,6 +1848,14 @@ void tcp_v4_destroy_sock(struct sock *sk)
sk->sk_sndmsg_page = NULL;
}
+ /*
+ * If splice cached page exists, toss it.
+ */
+ if (sk->sk_splice_page) {
+ __free_page(sk->sk_splice_page);
+ sk->sk_splice_page = NULL;
+ }
+
percpu_counter_dec(&tcp_sockets_allocated);
}