@@ -463,6 +463,12 @@ struct tcp_sock {
/* When the cookie options are generated and exchanged, then this
* object holds a reference to them (cookie_values->kref). Also
* contains related tcp_cookie_transactions fields.
+ *
+ * Note:
+ * cookie_values are partially under RCU:
+ * only s_data_constant, s_data_desired and s_data_payload should be
+ * changed via RCU. Writers should use a socket lock.
+ * So readers which hold this lock may omit rcu_reader_lock.
*/
struct tcp_cookie_values *cookie_values;
};
@@ -1498,6 +1498,7 @@ extern int tcp_cookie_generator(u32 *bakery);
* cookie option is present.
*/
struct tcp_cookie_values {
+ struct rcu_head rcu_head;
struct kref kref;
u8 cookie_pair[TCP_COOKIE_PAIR_SIZE];
u8 cookie_pair_size;
@@ -1510,18 +1511,33 @@ struct tcp_cookie_values {
u8 s_data_payload[0];
};
+extern struct tcp_cookie_values def_cookie_values;
+
static inline void tcp_cookie_values_release(struct kref *kref)
{
kfree(container_of(kref, struct tcp_cookie_values, kref));
}
+/* RCU-protected desired size of cookie
+ */
+static inline u8 rcu_tcp_cookie_desired(const struct tcp_sock *tp)
+{
+ u8 res;
+
+ rcu_read_lock();
+ res = rcu_dereference(tp->cookie_values)->cookie_desired;
+ rcu_read_unlock();
+
+ return res;
+}
+
/* The length of constant payload data. Note that s_data_desired is
* overloaded, depending on s_data_constant: either the length of constant
* data (returned here) or the limit on variable data.
*/
static inline int tcp_s_data_size(const struct tcp_sock *tp)
{
- return (tp->cookie_values != NULL && tp->cookie_values->s_data_constant)
+ return tp->cookie_values->s_data_constant
? tp->cookie_values->s_data_desired
: 0;
}
@@ -309,6 +309,20 @@ struct tcp_splice_state {
};
/*
+ * Default cookie values struct
+ * Initialization is for clarity only
+ */
+struct tcp_cookie_values def_cookie_values __read_mostly = {
+ .cookie_pair_size = 0,
+ .cookie_desired = 0,
+ .s_data_desired = 0,
+ .s_data_constant = 0,
+ .s_data_in = 0,
+ .s_data_out = 0
+};
+EXPORT_SYMBOL(def_cookie_values);
+
+/*
* Pressure flag: try to collapse.
* Technical note: it is used by multiple contexts non atomically.
* All the __sk_mem_schedule() is of this nature: accounting
@@ -2145,6 +2159,7 @@ static int do_tcp_setsockopt(struct sock *sk, int level,
case TCP_COOKIE_TRANSACTIONS: {
struct tcp_cookie_transactions ctd;
struct tcp_cookie_values *cvp = NULL;
+ struct kref *oldkref = NULL;
if (sizeof(ctd) > optlen)
return -EINVAL;
@@ -2166,66 +2181,63 @@ static int do_tcp_setsockopt(struct sock *sk, int level,
if (TCP_COOKIE_OUT_NEVER & ctd.tcpct_flags) {
/* Supercedes all other values */
lock_sock(sk);
- if (tp->cookie_values != NULL) {
- kref_put(&tp->cookie_values->kref,
- tcp_cookie_values_release);
- tp->cookie_values = NULL;
+
+ if (tp->cookie_values != &def_cookie_values) {
+ oldkref = &tp->cookie_values->kref;
+ rcu_assign_pointer(tp->cookie_values,
+ &def_cookie_values);
}
+
tp->rx_opt.cookie_in_always = 0; /* false */
tp->rx_opt.cookie_out_never = 1; /* true */
release_sock(sk);
+
+ if (oldkref) {
+ synchronize_rcu();
+ kref_put(oldkref, tcp_cookie_values_release);
+ }
+
return err;
}
- /* Allocate ancillary memory before locking.
- */
- if (ctd.tcpct_used > 0 ||
- (tp->cookie_values == NULL &&
- (sysctl_tcp_cookie_size > 0 ||
- ctd.tcpct_cookie_desired > 0 ||
- ctd.tcpct_s_data_desired > 0))) {
- cvp = kzalloc(sizeof(*cvp) + ctd.tcpct_used,
- GFP_KERNEL);
- if (cvp == NULL)
- return -ENOMEM;
-
- kref_init(&cvp->kref);
+ /* Setup cvp before locking */
+ cvp = kzalloc(sizeof(*cvp) + ctd.tcpct_used,
+ GFP_KERNEL);
+ if (cvp == NULL)
+ return -ENOMEM;
+
+ kref_init(&cvp->kref);
+
+ cvp->cookie_desired = ctd.tcpct_cookie_desired;
+
+ if (ctd.tcpct_used > 0) {
+ memcpy(cvp->s_data_payload, ctd.tcpct_value,
+ ctd.tcpct_used);
+ cvp->s_data_desired = ctd.tcpct_used;
+ cvp->s_data_constant = 1; /* true */
+ } else {
+ /* No constant payload data. */
+ cvp->s_data_desired = ctd.tcpct_s_data_desired;
+ cvp->s_data_constant = 0; /* false */
}
+
lock_sock(sk);
tp->rx_opt.cookie_in_always =
(TCP_COOKIE_IN_ALWAYS & ctd.tcpct_flags);
tp->rx_opt.cookie_out_never = 0; /* false */
- if (tp->cookie_values != NULL) {
- if (cvp != NULL) {
- /* Changed values are recorded by a changed
- * pointer, ensuring the cookie will differ,
- * without separately hashing each value later.
- */
- kref_put(&tp->cookie_values->kref,
- tcp_cookie_values_release);
- } else {
- cvp = tp->cookie_values;
- }
+ if (tp->cookie_values != &def_cookie_values) {
+ /* We have to release old structure, later */
+ oldkref = &tp->cookie_values->kref;
}
- if (cvp != NULL) {
- cvp->cookie_desired = ctd.tcpct_cookie_desired;
+ rcu_assign_pointer(tp->cookie_values, cvp);
+ release_sock(sk);
+ synchronize_rcu();
- if (ctd.tcpct_used > 0) {
- memcpy(cvp->s_data_payload, ctd.tcpct_value,
- ctd.tcpct_used);
- cvp->s_data_desired = ctd.tcpct_used;
- cvp->s_data_constant = 1; /* true */
- } else {
- /* No constant payload data. */
- cvp->s_data_desired = ctd.tcpct_s_data_desired;
- cvp->s_data_constant = 0; /* false */
- }
+ if (oldkref)
+ kref_put(oldkref, tcp_cookie_values_release);
- tp->cookie_values = cvp;
- }
- release_sock(sk);
return err;
}
default:
@@ -2572,7 +2584,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
case TCP_COOKIE_TRANSACTIONS: {
struct tcp_cookie_transactions ctd;
- struct tcp_cookie_values *cvp = tp->cookie_values;
+ struct tcp_cookie_values *cvp;
if (get_user(len, optlen))
return -EFAULT;
@@ -2585,19 +2597,21 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
| (tp->rx_opt.cookie_out_never ?
TCP_COOKIE_OUT_NEVER : 0);
- if (cvp != NULL) {
- ctd.tcpct_flags |= (cvp->s_data_in ?
- TCP_S_DATA_IN : 0)
- | (cvp->s_data_out ?
- TCP_S_DATA_OUT : 0);
+ rcu_read_lock();
- ctd.tcpct_cookie_desired = cvp->cookie_desired;
- ctd.tcpct_s_data_desired = cvp->s_data_desired;
+ cvp = rcu_dereference(tp->cookie_values);
+ ctd.tcpct_flags |= (cvp->s_data_in ?
+ TCP_S_DATA_IN : 0)
+ | (cvp->s_data_out ?
+ TCP_S_DATA_OUT : 0);
- memcpy(&ctd.tcpct_value[0], &cvp->cookie_pair[0],
- cvp->cookie_pair_size);
- ctd.tcpct_used = cvp->cookie_pair_size;
- }
+ ctd.tcpct_cookie_desired = cvp->cookie_desired;
+ ctd.tcpct_s_data_desired = cvp->s_data_desired;
+
+ memcpy(&ctd.tcpct_value[0], &cvp->cookie_pair[0],
+ cvp->cookie_pair_size);
+ ctd.tcpct_used = cvp->cookie_pair_size;
+ rcu_read_unlock();
if (put_user(sizeof(ctd), optlen))
return -EFAULT;
@@ -5450,6 +5450,7 @@ static int tcp_rcv_synsent_state_process(struct
sock *sk, struct sk_buff *skb,
u8 *hash_location;
struct inet_connection_sock *icsk = inet_csk(sk);
struct tcp_sock *tp = tcp_sk(sk);
+ /* We're under socket lock, no need for RCU */
struct tcp_cookie_values *cvp = tp->cookie_values;
int saved_clamp = tp->rx_opt.mss_clamp;
@@ -5551,8 +5552,7 @@ static int tcp_rcv_synsent_state_process(struct
sock *sk, struct sk_buff *skb,
* is initialized. */
tp->copied_seq = tp->rcv_nxt;
- if (cvp != NULL &&
- cvp->cookie_pair_size > 0 &&
+ if (cvp->cookie_pair_size > 0 &&
tp->rx_opt.cookie_plus > 0) {
int cookie_size = tp->rx_opt.cookie_plus
- TCPOLEN_COOKIE_BASE;
@@ -1380,8 +1380,7 @@ int tcp_v4_conn_request(struct sock *sk, struct
sk_buff *skb)
tmp_opt.saw_tstamp &&
!tp->rx_opt.cookie_out_never &&
(sysctl_tcp_cookie_size > 0 ||
- (tp->cookie_values != NULL &&
- tp->cookie_values->cookie_desired > 0))) {
+ rcu_tcp_cookie_desired(tp) > 0)) {
u8 *c;
u32 *mess = &tmp_ext.cookie_bakery[COOKIE_DIGEST_WORDS];
int l = tmp_opt.cookie_plus - TCPOLEN_COOKIE_BASE;
@@ -1403,14 +1402,15 @@ int tcp_v4_conn_request(struct sock *sk,
struct sk_buff *skb)
#endif
tmp_ext.cookie_out_never = 0; /* false */
tmp_ext.cookie_plus = tmp_opt.cookie_plus;
+ tmp_ext.cookie_in_always = tp->rx_opt.cookie_in_always;
} else if (!tp->rx_opt.cookie_in_always) {
/* redundant indications, but ensure initialization. */
tmp_ext.cookie_out_never = 1; /* true */
tmp_ext.cookie_plus = 0;
+ tmp_ext.cookie_in_always = 0;
} else {
goto drop_and_release;
}
- tmp_ext.cookie_in_always = tp->rx_opt.cookie_in_always;
if (want_cookie && !tmp_opt.saw_tstamp)
tcp_clear_options(&tmp_opt);
@@ -1990,9 +1990,11 @@ static int tcp_v4_init_sock(struct sock *sk)
tp->cookie_values =
kzalloc(sizeof(*tp->cookie_values),
sk->sk_allocation);
- if (tp->cookie_values != NULL)
- kref_init(&tp->cookie_values->kref);
}
+ if (tp->cookie_values != NULL)
+ kref_init(&tp->cookie_values->kref);
+ else
+ tp->cookie_values = &def_cookie_values;
/* Presumed zeroed, in order of appearance:
* cookie_in_always, cookie_out_never,
* s_data_constant, s_data_in, s_data_out
@@ -2007,6 +2009,13 @@ static int tcp_v4_init_sock(struct sock *sk)
return 0;
}
+static void tcp_cookie_values_put(struct rcu_head *rcu_head)
+{
+ struct tcp_cookie_values *cvp =
+ container_of(rcu_head, struct tcp_cookie_values, rcu_head);
+ kref_put(&cvp->kref, tcp_cookie_values_release);
+}
+
void tcp_v4_destroy_sock(struct sock *sk)
{
struct tcp_sock *tp = tcp_sk(sk);
@@ -2047,10 +2056,11 @@ void tcp_v4_destroy_sock(struct sock *sk)
}
/* TCP Cookie Transactions */
- if (tp->cookie_values != NULL) {
- kref_put(&tp->cookie_values->kref,
- tcp_cookie_values_release);
- tp->cookie_values = NULL;
+ if (tp->cookie_values != &def_cookie_values) {
+ struct rcu_head *rcu_head = &tp->cookie_values->rcu_head;
+
+ rcu_assign_pointer(tp->cookie_values, &def_cookie_values);
+ call_rcu(rcu_head, tcp_cookie_values_put);
}
percpu_counter_dec(&tcp_sockets_allocated);
@@ -375,7 +375,8 @@ struct sock *tcp_create_openreq_child(struct sock
*sk, struct request_sock *req,
struct inet_connection_sock *newicsk = inet_csk(newsk);
struct tcp_sock *newtp = tcp_sk(newsk);
struct tcp_sock *oldtp = tcp_sk(sk);
- struct tcp_cookie_values *oldcvp = oldtp->cookie_values;
+ struct tcp_cookie_values *oldcvp;
+ int s_data_size;
/* TCP Cookie Transactions require space for the cookie pair,
* as it differs for each connection. There is no need to
@@ -385,7 +386,10 @@ struct sock *tcp_create_openreq_child(struct sock
*sk, struct request_sock *req,
* Presumed copied, in order of appearance:
* cookie_in_always, cookie_out_never
*/
- if (oldcvp != NULL) {
+
+ rcu_read_lock();
+ oldcvp = rcu_dereference(oldtp->cookie_values);
+ if (oldcvp != &def_cookie_values) {
struct tcp_cookie_values *newcvp =
kzalloc(sizeof(*newtp->cookie_values),
GFP_ATOMIC);
@@ -397,9 +401,15 @@ struct sock *tcp_create_openreq_child(struct sock
*sk, struct request_sock *req,
newtp->cookie_values = newcvp;
} else {
/* Not Yet Implemented */
- newtp->cookie_values = NULL;
+ newtp->cookie_values = &def_cookie_values;
}
+ } else {
+ newtp->cookie_values = &def_cookie_values;
}
+ s_data_size = oldcvp->s_data_constant ?
+ oldcvp->s_data_desired :
+ 0;
+ rcu_read_unlock();
/* Now setup tcp_sock */
newtp->pred_flags = 0;
@@ -409,7 +419,7 @@ struct sock *tcp_create_openreq_child(struct sock
*sk, struct request_sock *req,
newtp->snd_sml = newtp->snd_una =
newtp->snd_nxt = newtp->snd_up =
- treq->snt_isn + 1 + tcp_s_data_size(oldtp);
+ treq->snt_isn + 1 + s_data_size;
tcp_prequeue_init(newtp);
@@ -443,7 +453,7 @@ struct sock *tcp_create_openreq_child(struct sock
*sk, struct request_sock *req,
tcp_init_xmit_timers(newsk);
skb_queue_head_init(&newtp->out_of_order_queue);
newtp->write_seq = newtp->pushed_seq =
- treq->snt_isn + 1 + tcp_s_data_size(oldtp);
+ treq->snt_isn + 1 + s_data_size;
newtp->rx_opt.saw_tstamp = 0;
@@ -562,11 +562,14 @@ static unsigned tcp_syn_options(struct sock *sk,
struct sk_buff *skb,
struct tcp_out_options *opts,
struct tcp_md5sig_key **md5) {
struct tcp_sock *tp = tcp_sk(sk);
+ /* As long as tcp_syn_options is called under socket lock,
+ * we don't need RCU here */
struct tcp_cookie_values *cvp = tp->cookie_values;
unsigned remaining = MAX_TCP_OPTION_SPACE;
- u8 cookie_size = (!tp->rx_opt.cookie_out_never && cvp != NULL) ?
- tcp_cookie_size_check(cvp->cookie_desired) :
- 0;
+ u8 cookie_size =
+ (!tp->rx_opt.cookie_out_never && cvp != &def_cookie_values) ?
+ tcp_cookie_size_check(cvp->cookie_desired) :
+ 0;
#ifdef CONFIG_TCP_MD5SIG
*md5 = tp->af_specific->md5_get(sk, sk);
@@ -2407,19 +2410,28 @@ struct sk_buff *tcp_make_synack(struct sock
*sk, struct dst_entry *dst,
struct tcp_extend_values *xvp = tcp_xv(rvp);
struct inet_request_sock *ireq = inet_rsk(req);
struct tcp_sock *tp = tcp_sk(sk);
- const struct tcp_cookie_values *cvp = tp->cookie_values;
+ struct tcp_cookie_values *cvp;
struct tcphdr *th;
struct sk_buff *skb;
struct tcp_md5sig_key *md5;
int tcp_header_size;
int mss;
- int s_data_desired = 0;
+ int s_data_desired;
+
+ rcu_read_lock();
+ cvp = rcu_dereference(tp->cookie_values);
+ if (cvp != &def_cookie_values)
+ kref_get(&cvp->kref);
+ rcu_read_unlock();
+
+ s_data_desired = cvp->s_data_constant ? cvp->s_data_desired : 0;
- if (cvp != NULL && cvp->s_data_constant && cvp->s_data_desired)
- s_data_desired = cvp->s_data_desired;
skb = sock_wmalloc(sk, MAX_TCP_HEADER + 15 + s_data_desired, 1, GFP_ATOMIC);
- if (skb == NULL)
+ if (skb == NULL) {
+ if (cvp != &def_cookie_values)
+ kref_put(&cvp->kref, tcp_cookie_values_release);
return NULL;
+ }
/* Reserve space for headers. */
skb_reserve(skb, MAX_TCP_HEADER);
@@ -2506,6 +2518,9 @@ struct sk_buff *tcp_make_synack(struct sock *sk,
struct dst_entry *dst,
}
}
+ if (cvp != &def_cookie_values)
+ kref_put(&cvp->kref, tcp_cookie_values_release);
+
th->seq = htonl(TCP_SKB_CB(skb)->seq);
th->ack_seq = htonl(tcp_rsk(req)->rcv_isn + 1);
@@ -1306,8 +1306,7 @@ static int tcp_v6_conn_request(struct sock *sk,
struct sk_buff *skb)
tmp_opt.saw_tstamp &&
!tp->rx_opt.cookie_out_never &&
(sysctl_tcp_cookie_size > 0 ||
- (tp->cookie_values != NULL &&
- tp->cookie_values->cookie_desired > 0))) {
+ rcu_tcp_cookie_desired(tp) > 0)) {
u8 *c;
u32 *d;