@@ -1530,6 +1530,12 @@ struct task_struct {
* cache last used pipe for splice
*/
struct pipe_inode_info *splice_pipe;
+ /*
+ * cache for page frag allocator
+ */
+ struct page *sndmsg_page;
+ unsigned int sndmsg_off;
+
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
@@ -247,8 +247,8 @@ struct cg_proto;
* @sk_stamp: time stamp of last packet received
* @sk_socket: Identd and reporting IO signals
* @sk_user_data: RPC layer private data
- * @sk_sndmsg_page: cached page for sendmsg
- * @sk_sndmsg_off: cached offset for sendmsg
+ * @sk_sndmsg_page: cached page for splice/ip6_append_data()
+ * @sk_sndmsg_off: cached offset for splice/ip6_append_data()
* @sk_peek_off: current peek_offset value
* @sk_send_head: front of stuff to transmit
* @sk_security: used by security modules
@@ -2034,11 +2034,17 @@ static inline void sk_stream_moderate_sndbuf(struct sock *sk)
struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
+/* On 32bit arches, an skb frag is limited to 2^15, because
+ * (struct skb_frag_struct)->size/offset are u16
+ */
+#define SNDMSG_PAGE_ORDER min(get_order(32768), PAGE_ALLOC_COSTLY_ORDER)
+#define SNDMSG_PAGE_SIZE (PAGE_SIZE << SNDMSG_PAGE_ORDER)
+
static inline struct page *sk_stream_alloc_page(struct sock *sk)
{
struct page *page = NULL;
- page = alloc_pages(sk->sk_allocation, 0);
+ page = alloc_pages(sk->sk_allocation | __GFP_COMP, SNDMSG_PAGE_ORDER);
if (!page) {
sk_enter_memory_pressure(sk);
sk_stream_moderate_sndbuf(sk);
@@ -1046,6 +1046,9 @@ void do_exit(long code)
if (tsk->splice_pipe)
__free_pipe_info(tsk->splice_pipe);
+ if (tsk->sndmsg_page)
+ put_page(tsk->sndmsg_page);
+
validate_creds_for_do_exit(tsk);
preempt_disable();
@@ -330,6 +330,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
tsk->btrace_seq = 0;
#endif
tsk->splice_pipe = NULL;
+ tsk->sndmsg_page = NULL;
account_kernel_stack(ti, 1);
@@ -1152,16 +1152,16 @@ new_segment:
} else {
bool merge = false;
int i = skb_shinfo(skb)->nr_frags;
- struct page *page = sk->sk_sndmsg_page;
+ struct page *page = current->sndmsg_page;
int off;
if (page && page_count(page) == 1)
- sk->sk_sndmsg_off = 0;
+ current->sndmsg_off = 0;
- off = sk->sk_sndmsg_off;
+ off = current->sndmsg_off;
if (skb_can_coalesce(skb, i, page, off) &&
- off != PAGE_SIZE) {
+ off != SNDMSG_PAGE_SIZE) {
/* We can extend the last page
* fragment. */
merge = true;
@@ -1173,16 +1173,16 @@ new_segment:
tcp_mark_push(tp, skb);
goto new_segment;
} else if (page) {
- if (off == PAGE_SIZE) {
+ if (off == SNDMSG_PAGE_SIZE) {
put_page(page);
- sk->sk_sndmsg_page = page = NULL;
+ current->sndmsg_page = page = NULL;
off = 0;
}
} else
off = 0;
- if (copy > PAGE_SIZE - off)
- copy = PAGE_SIZE - off;
+ if (copy > SNDMSG_PAGE_SIZE - off)
+ copy = SNDMSG_PAGE_SIZE - off;
if (!sk_wmem_schedule(sk, copy))
goto wait_for_memory;
@@ -1198,12 +1198,12 @@ new_segment:
err = skb_copy_to_page_nocache(sk, from, skb,
page, off, copy);
if (err) {
- /* If this page was new, give it to the
- * socket so it does not get leaked.
+ /* If this page was new, remember it
+ * so it does not get leaked.
*/
- if (!sk->sk_sndmsg_page) {
- sk->sk_sndmsg_page = page;
- sk->sk_sndmsg_off = 0;
+ if (!current->sndmsg_page) {
+ current->sndmsg_page = page;
+ current->sndmsg_off = 0;
}
goto do_error;
}
@@ -1213,15 +1213,15 @@ new_segment:
skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
} else {
skb_fill_page_desc(skb, i, page, off, copy);
- if (sk->sk_sndmsg_page) {
+ if (current->sndmsg_page) {
get_page(page);
- } else if (off + copy < PAGE_SIZE) {
+ } else if (off + copy < SNDMSG_PAGE_SIZE) {
get_page(page);
- sk->sk_sndmsg_page = page;
+ current->sndmsg_page = page;
}
}
- sk->sk_sndmsg_off = off + copy;
+ current->sndmsg_off = off + copy;
}
if (!copied)
@@ -2196,9 +2196,7 @@ void tcp_v4_destroy_sock(struct sock *sk)
if (inet_csk(sk)->icsk_bind_hash)
inet_put_port(sk);
- /*
- * If sendmsg cached page exists, toss it.
- */
+ /* If cached page exists, toss it. */
if (sk->sk_sndmsg_page) {
__free_page(sk->sk_sndmsg_page);
sk->sk_sndmsg_page = NULL;