@@ -741,6 +741,7 @@ enum sock_flags {
SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */
SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
+ SOCK_SHORT_WRITE, /* Couldn't fill sndbuf due to memory pressure */
};
#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
@@ -1114,6 +1115,11 @@ static inline bool sk_stream_is_writeable(const struct sock *sk)
sk_stream_memory_free(sk);
}
+static inline void sk_set_short_write(struct sock *sk)
+{
+ if (sk->sk_wmem_queued > 0 && sk_stream_is_writeable(sk))
+ sock_set_flag(sk, SOCK_SHORT_WRITE);
+}
static inline bool sk_has_memory_pressure(const struct sock *sk)
{
@@ -517,7 +517,8 @@ unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
mask |= POLLIN | POLLRDNORM;
if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
- if (sk_stream_is_writeable(sk)) {
+ if (sk_stream_is_writeable(sk) &&
+ !sock_flag(sk, SOCK_SHORT_WRITE)) {
mask |= POLLOUT | POLLWRNORM;
} else { /* send SIGIO later */
sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
@@ -529,7 +530,8 @@ unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
* pairs with the input side.
*/
smp_mb__after_atomic();
- if (sk_stream_is_writeable(sk))
+ if (sk_stream_is_writeable(sk) &&
+ !sock_flag(sk, SOCK_SHORT_WRITE))
mask |= POLLOUT | POLLWRNORM;
}
} else
@@ -917,8 +919,10 @@ new_segment:
skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
skb_queue_empty(&sk->sk_write_queue));
- if (!skb)
+ if (!skb) {
+ sk_set_short_write(sk);
goto wait_for_memory;
+ }
skb_entail(sk, skb);
copy = size_goal;
@@ -933,8 +937,10 @@ new_segment:
tcp_mark_push(tp, skb);
goto new_segment;
}
- if (!sk_wmem_schedule(sk, copy))
+ if (!sk_wmem_schedule(sk, copy)) {
+ sk_set_short_write(sk);
goto wait_for_memory;
+ }
if (can_coalesce) {
skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
@@ -1176,8 +1182,10 @@ new_segment:
select_size(sk, sg),
sk->sk_allocation,
skb_queue_empty(&sk->sk_write_queue));
- if (!skb)
+ if (!skb) {
+ sk_set_short_write(sk);
goto wait_for_memory;
+ }
process_backlog = true;
/*
@@ -1214,8 +1222,10 @@ new_segment:
int i = skb_shinfo(skb)->nr_frags;
struct page_frag *pfrag = sk_page_frag(sk);
- if (!sk_page_frag_refill(sk, pfrag))
+ if (!sk_page_frag_refill(sk, pfrag)) {
+ sk_set_short_write(sk);
goto wait_for_memory;
+ }
if (!skb_can_coalesce(skb, i, pfrag->page,
pfrag->offset)) {
@@ -1228,8 +1238,10 @@ new_segment:
copy = min_t(int, copy, pfrag->size - pfrag->offset);
- if (!sk_wmem_schedule(sk, copy))
+ if (!sk_wmem_schedule(sk, copy)) {
+ sk_set_short_write(sk);
goto wait_for_memory;
+ }
err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
pfrag->page,
@@ -4985,7 +4985,8 @@ static void tcp_new_space(struct sock *sk)
static void tcp_check_space(struct sock *sk)
{
if (sock_flag(sk, SOCK_QUEUE_SHRUNK)) {
- sock_reset_flag(sk, SOCK_QUEUE_SHRUNK);
+ sk->sk_flags &= ~((1UL << SOCK_QUEUE_SHRUNK) |
+ (1UL << SOCK_SHORT_WRITE));
/* pairs with tcp_poll() */
smp_mb();
if (sk->sk_socket &&