diff mbox

[net-next,4/5] net: sctp: decouple cleaning socket data from endpoint

Message ID 1371545720-22950-5-git-send-email-dborkman@redhat.com
State Changes Requested, archived
Delegated to: David Miller
Headers show

Commit Message

Daniel Borkmann June 18, 2013, 8:55 a.m. UTC
Rather instead of having the endpoint clean the garbage for the
socket, use a sk_destruct handler sctp_destruct_sock(), that does
the job for that when there are no more references on the socket.

Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
---
 net/sctp/endpointola.c |  7 -------
 net/sctp/socket.c      | 20 +++++++++++++++++++-
 2 files changed, 19 insertions(+), 8 deletions(-)

Comments

Neil Horman June 18, 2013, 2:22 p.m. UTC | #1
On Tue, Jun 18, 2013 at 10:55:19AM +0200, Daniel Borkmann wrote:
> Rather instead of having the endpoint clean the garbage for the
> socket, use a sk_destruct handler sctp_destruct_sock(), that does
> the job for that when there are no more references on the socket.
> 
> Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
> ---
>  net/sctp/endpointola.c |  7 -------
>  net/sctp/socket.c      | 20 +++++++++++++++++++-
>  2 files changed, 19 insertions(+), 8 deletions(-)
> 
> diff --git a/net/sctp/endpointola.c b/net/sctp/endpointola.c
> index a8b2674..9a9ebd2 100644
> --- a/net/sctp/endpointola.c
> +++ b/net/sctp/endpointola.c
> @@ -249,9 +249,6 @@ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
>  {
>  	SCTP_ASSERT(ep->base.dead, "Endpoint is not dead", return);
>  
> -	/* Free up the HMAC transform. */
> -	crypto_free_hash(sctp_sk(ep->base.sk)->hmac);
> -
>  	/* Free the digest buffer */
>  	kfree(ep->digest);
>  
> @@ -271,10 +268,6 @@ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
>  
>  	memset(ep->secret_key, 0, sizeof(ep->secret_key));
>  
> -	/* Remove and free the port */
> -	if (sctp_sk(ep->base.sk)->bind_hash)
> -		sctp_put_port(ep->base.sk);
> -
>  	/* Give up our hold on the sock. */
>  	if (ep->base.sk)
>  		sock_put(ep->base.sk);
> diff --git a/net/sctp/socket.c b/net/sctp/socket.c
> index 68a6b70..67f721e 100644
> --- a/net/sctp/socket.c
> +++ b/net/sctp/socket.c
> @@ -93,6 +93,7 @@ static int sctp_wait_for_packet(struct sock * sk, int *err, long *timeo_p);
>  static int sctp_wait_for_connect(struct sctp_association *, long *timeo_p);
>  static int sctp_wait_for_accept(struct sock *sk, long timeo);
>  static void sctp_wait_for_close(struct sock *sk, long timeo);
> +static void sctp_destruct_sock(struct sock *sk);
>  static struct sctp_af *sctp_sockaddr_af(struct sctp_sock *opt,
>  					union sctp_addr *addr, int len);
>  static int sctp_bindx_add(struct sock *, struct sockaddr *, int);
> @@ -3966,6 +3967,8 @@ static int sctp_init_sock(struct sock *sk)
>  
>  	sp->hmac = NULL;
>  
> +	sk->sk_destruct = sctp_destruct_sock;
> +
>  	SCTP_DBG_OBJCNT_INC(sock);
>  
>  	local_bh_disable();
> @@ -4002,6 +4005,21 @@ static void sctp_destroy_sock(struct sock *sk)
>  	local_bh_enable();
>  }
>  
> +/* Triggered when there are no references on the socket anymore */
> +static void sctp_destruct_sock(struct sock *sk)
> +{
> +	struct sctp_sock *sp = sctp_sk(sk);
> +
> +	/* Free up the HMAC transform. */
> +	crypto_free_hash(sp->hmac);
> +
> +	/* Remove and free the port */
> +	if (sp->bind_hash)
> +		sctp_put_port(sk);
> +
> +	inet_sock_destruct(sk);
> +}
> +
>  /* API 4.1.7 shutdown() - TCP Style Syntax
>   *     int shutdown(int socket, int how);
>   *
> @@ -6842,7 +6860,7 @@ void sctp_copy_sock(struct sock *newsk, struct sock *sk,
>  	newsk->sk_reuse = sk->sk_reuse;
>  
>  	newsk->sk_shutdown = sk->sk_shutdown;
> -	newsk->sk_destruct = inet_sock_destruct;
> +	newsk->sk_destruct = sctp_destruct_sock;
>  	newsk->sk_family = sk->sk_family;
>  	newsk->sk_protocol = IPPROTO_SCTP;
>  	newsk->sk_backlog_rcv = sk->sk_prot->backlog_rcv;
> -- 
> 1.7.11.7
> 

I like this idea, but I think I'm maybe missing something from it - we reference
the socket in both the receive and send paths (sctp_unpack_cookie, is
specifically called from the rx path, which makes use of sp->hmac).  a socket
destructor can be called from __sk_free when sk_wmem_alloc reaches zero, but we
use sk_refcnt in the rx path to prevent premature socket cleanup.  If we drain
our send queeue while wer'e still processing rx messages, what prevents us from
freeing the socket in the tx path, via sk_free while we're still using the
socket in the rx path.  Note I don't think this patch is wrong per-se, but it
seems to me there is more work to do to properly interlock the use of sk_refcnt
and sk_wmem_alloc here (unless I'm just missing something obvious, which is
entirely possible, I've been in the sun alot lately :) ).

Neil
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Daniel Borkmann June 18, 2013, 4:02 p.m. UTC | #2
On 06/18/2013 04:22 PM, Neil Horman wrote:
> I like this idea, but I think I'm maybe missing something from it - we reference
> the socket in both the receive and send paths (sctp_unpack_cookie, is
> specifically called from the rx path, which makes use of sp->hmac).  a socket
> destructor can be called from __sk_free when sk_wmem_alloc reaches zero, but we
> use sk_refcnt in the rx path to prevent premature socket cleanup.  If we drain
> our send queeue while wer'e still processing rx messages, what prevents us from
> freeing the socket in the tx path, via sk_free while we're still using the
> socket in the rx path.  Note I don't think this patch is wrong per-se, but it
> seems to me there is more work to do to properly interlock the use of sk_refcnt
> and sk_wmem_alloc here (unless I'm just missing something obvious, which is
> entirely possible, I've been in the sun alot lately :) ).

Hm, __sk_free() calls sk_prot_free() which frees our socket structure and in
sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).

So no matter if having this patch or not, couldn't this use-after-free like
scenario already happen with the current code?

F.e. through a given call graph like that:

sctp_wfree(skb):
  1) sock_wfree(skb)
     -> __sk_free()
      -> sk_prot_free(.., sk)
       -> kmem_cache_free(.., sk) or kfree(sk)
  2) __sctp_write_space(asoc)
  3) sctp_association_put(asoc)
     -> sctp_association_destroy(asoc)
      -> sctp_endpoint_put(asoc->ep)
       -> sctp_endpoint_destroy(ep)
        -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
           (etc, all unconditionally accessed while sk is
            already dead/freed)

Then, this might need a fix in general. :-)

Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF, ..),
you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and a call to
sk->sk_write_space(sk), which is sctp_write_space() and calls __sctp_write_space()
on all asocs belonging to the socket, but it seems not to alter the current
sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.

[*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
     SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
     on skb->truesize as well?
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Vladislav Yasevich June 18, 2013, 4:24 p.m. UTC | #3
On 06/18/2013 12:02 PM, Daniel Borkmann wrote:
> On 06/18/2013 04:22 PM, Neil Horman wrote:
>> I like this idea, but I think I'm maybe missing something from it - we
>> reference
>> the socket in both the receive and send paths (sctp_unpack_cookie, is
>> specifically called from the rx path, which makes use of sp->hmac).  a
>> socket
>> destructor can be called from __sk_free when sk_wmem_alloc reaches
>> zero, but we
>> use sk_refcnt in the rx path to prevent premature socket cleanup.  If
>> we drain
>> our send queeue while wer'e still processing rx messages, what
>> prevents us from
>> freeing the socket in the tx path, via sk_free while we're still using
>> the
>> socket in the rx path.  Note I don't think this patch is wrong per-se,
>> but it
>> seems to me there is more work to do to properly interlock the use of
>> sk_refcnt
>> and sk_wmem_alloc here (unless I'm just missing something obvious,
>> which is
>> entirely possible, I've been in the sun alot lately :) ).
>
> Hm, __sk_free() calls sk_prot_free() which frees our socket structure
> and in
> sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).
>
> So no matter if having this patch or not, couldn't this use-after-free like
> scenario already happen with the current code?
>
> F.e. through a given call graph like that:
>
> sctp_wfree(skb):
>   1) sock_wfree(skb)
>      -> __sk_free()

I don't think this can happen.  sk_wmem_alloc is set to 1 in sk_alloc()
and that acts as a guard to make sure that sk_free() has been called
before we try to free things up.  So, in this partcular case, for 
__sk_free() to be called, sk_free() had to be called meaning the
last ref on the socket was released.  However, that's not possible since
we are still holding the association and thus holding the socket
associated with it.

-vlad

>       -> sk_prot_free(.., sk)
>        -> kmem_cache_free(.., sk) or kfree(sk)
>   2) __sctp_write_space(asoc)
>   3) sctp_association_put(asoc)
>      -> sctp_association_destroy(asoc)
>       -> sctp_endpoint_put(asoc->ep)
>        -> sctp_endpoint_destroy(ep)
>         -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
>            (etc, all unconditionally accessed while sk is
>             already dead/freed)
>
> Then, this might need a fix in general. :-)
>
> Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF,
> ..),
> you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and
> a call to
> sk->sk_write_space(sk), which is sctp_write_space() and calls
> __sctp_write_space()
> on all asocs belonging to the socket, but it seems not to alter the current
> sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.
>
> [*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
>      SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
>      on skb->truesize as well?
> --
> To unsubscribe from this list: send the line "unsubscribe linux-sctp" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Neil Horman June 18, 2013, 5:45 p.m. UTC | #4
On Tue, Jun 18, 2013 at 12:24:31PM -0400, Vlad Yasevich wrote:
> On 06/18/2013 12:02 PM, Daniel Borkmann wrote:
> >On 06/18/2013 04:22 PM, Neil Horman wrote:
> >>I like this idea, but I think I'm maybe missing something from it - we
> >>reference
> >>the socket in both the receive and send paths (sctp_unpack_cookie, is
> >>specifically called from the rx path, which makes use of sp->hmac).  a
> >>socket
> >>destructor can be called from __sk_free when sk_wmem_alloc reaches
> >>zero, but we
> >>use sk_refcnt in the rx path to prevent premature socket cleanup.  If
> >>we drain
> >>our send queeue while wer'e still processing rx messages, what
> >>prevents us from
> >>freeing the socket in the tx path, via sk_free while we're still using
> >>the
> >>socket in the rx path.  Note I don't think this patch is wrong per-se,
> >>but it
> >>seems to me there is more work to do to properly interlock the use of
> >>sk_refcnt
> >>and sk_wmem_alloc here (unless I'm just missing something obvious,
> >>which is
> >>entirely possible, I've been in the sun alot lately :) ).
> >
> >Hm, __sk_free() calls sk_prot_free() which frees our socket structure
> >and in
> >sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).
> >
> >So no matter if having this patch or not, couldn't this use-after-free like
> >scenario already happen with the current code?
> >
> >F.e. through a given call graph like that:
> >
> >sctp_wfree(skb):
> >  1) sock_wfree(skb)
> >     -> __sk_free()
> 
> I don't think this can happen.  sk_wmem_alloc is set to 1 in sk_alloc()
> and that acts as a guard to make sure that sk_free() has been called
> before we try to free things up.  So, in this partcular case, for
> __sk_free() to be called, sk_free() had to be called meaning the
> last ref on the socket was released.  However, that's not possible since
> we are still holding the association and thus holding the socket
> associated with it.
> 
I see what your saying, and I agree, with that bias added in sk_alloc, it looks
like we won't ever call __sk_free until sk_wmem_alloc is 1 _and_ sk_refcnt is 0.
It still seems messy and confusing though.  It would make more sense to me to
increment the refcount an additional time when the socket is initalized, and
then decrement it again when the socket is closed and sk_wmem_alloc reaches
zero.  That would isolate the refcounting to a single variable.
Neil

> -vlad
> 
> >      -> sk_prot_free(.., sk)
> >       -> kmem_cache_free(.., sk) or kfree(sk)
> >  2) __sctp_write_space(asoc)
> >  3) sctp_association_put(asoc)
> >     -> sctp_association_destroy(asoc)
> >      -> sctp_endpoint_put(asoc->ep)
> >       -> sctp_endpoint_destroy(ep)
> >        -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
> >           (etc, all unconditionally accessed while sk is
> >            already dead/freed)
> >
> >Then, this might need a fix in general. :-)
> >
> >Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF,
> >..),
> >you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and
> >a call to
> >sk->sk_write_space(sk), which is sctp_write_space() and calls
> >__sctp_write_space()
> >on all asocs belonging to the socket, but it seems not to alter the current
> >sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.
> >
> >[*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
> >     SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
> >     on skb->truesize as well?
> >--
> >To unsubscribe from this list: send the line "unsubscribe linux-sctp" in
> >the body of a message to majordomo@vger.kernel.org
> >More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 
> 
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Vladislav Yasevich June 18, 2013, 6:15 p.m. UTC | #5
On 06/18/2013 01:45 PM, Neil Horman wrote:
> On Tue, Jun 18, 2013 at 12:24:31PM -0400, Vlad Yasevich wrote:
>> On 06/18/2013 12:02 PM, Daniel Borkmann wrote:
>>> On 06/18/2013 04:22 PM, Neil Horman wrote:
>>>> I like this idea, but I think I'm maybe missing something from it - we
>>>> reference
>>>> the socket in both the receive and send paths (sctp_unpack_cookie, is
>>>> specifically called from the rx path, which makes use of sp->hmac).  a
>>>> socket
>>>> destructor can be called from __sk_free when sk_wmem_alloc reaches
>>>> zero, but we
>>>> use sk_refcnt in the rx path to prevent premature socket cleanup.  If
>>>> we drain
>>>> our send queeue while wer'e still processing rx messages, what
>>>> prevents us from
>>>> freeing the socket in the tx path, via sk_free while we're still using
>>>> the
>>>> socket in the rx path.  Note I don't think this patch is wrong per-se,
>>>> but it
>>>> seems to me there is more work to do to properly interlock the use of
>>>> sk_refcnt
>>>> and sk_wmem_alloc here (unless I'm just missing something obvious,
>>>> which is
>>>> entirely possible, I've been in the sun alot lately :) ).
>>>
>>> Hm, __sk_free() calls sk_prot_free() which frees our socket structure
>>> and in
>>> sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).
>>>
>>> So no matter if having this patch or not, couldn't this use-after-free like
>>> scenario already happen with the current code?
>>>
>>> F.e. through a given call graph like that:
>>>
>>> sctp_wfree(skb):
>>>   1) sock_wfree(skb)
>>>      -> __sk_free()
>>
>> I don't think this can happen.  sk_wmem_alloc is set to 1 in sk_alloc()
>> and that acts as a guard to make sure that sk_free() has been called
>> before we try to free things up.  So, in this partcular case, for
>> __sk_free() to be called, sk_free() had to be called meaning the
>> last ref on the socket was released.  However, that's not possible since
>> we are still holding the association and thus holding the socket
>> associated with it.
>>
> I see what your saying, and I agree, with that bias added in sk_alloc, it looks
> like we won't ever call __sk_free until sk_wmem_alloc is 1 _and_ sk_refcnt is 0.
> It still seems messy and confusing though.  It would make more sense to me to
> increment the refcount an additional time when the socket is initalized, and
> then decrement it again when the socket is closed and sk_wmem_alloc reaches
> zero.  That would isolate the refcounting to a single variable.

See commit 2b85a34e911bf483c27cfdd124aeb1605145dc80.  The whole 
sk_wmem_alloc tric was done so that we dont have to do 
sock_hold/sock_put on transmits.

It might be good to see if we can do that in sctp as well.

-vlad


> Neil
>
>> -vlad
>>
>>>       -> sk_prot_free(.., sk)
>>>        -> kmem_cache_free(.., sk) or kfree(sk)
>>>   2) __sctp_write_space(asoc)
>>>   3) sctp_association_put(asoc)
>>>      -> sctp_association_destroy(asoc)
>>>       -> sctp_endpoint_put(asoc->ep)
>>>        -> sctp_endpoint_destroy(ep)
>>>         -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
>>>            (etc, all unconditionally accessed while sk is
>>>             already dead/freed)
>>>
>>> Then, this might need a fix in general. :-)
>>>
>>> Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF,
>>> ..),
>>> you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and
>>> a call to
>>> sk->sk_write_space(sk), which is sctp_write_space() and calls
>>> __sctp_write_space()
>>> on all asocs belonging to the socket, but it seems not to alter the current
>>> sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.
>>>
>>> [*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
>>>      SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
>>>      on skb->truesize as well?
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-sctp" in
>>> the body of a message to majordomo@vger.kernel.org
>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
>>

--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Neil Horman June 18, 2013, 7:12 p.m. UTC | #6
On Tue, Jun 18, 2013 at 02:15:21PM -0400, Vlad Yasevich wrote:
> On 06/18/2013 01:45 PM, Neil Horman wrote:
> >On Tue, Jun 18, 2013 at 12:24:31PM -0400, Vlad Yasevich wrote:
> >>On 06/18/2013 12:02 PM, Daniel Borkmann wrote:
> >>>On 06/18/2013 04:22 PM, Neil Horman wrote:
> >>>>I like this idea, but I think I'm maybe missing something from it - we
> >>>>reference
> >>>>the socket in both the receive and send paths (sctp_unpack_cookie, is
> >>>>specifically called from the rx path, which makes use of sp->hmac).  a
> >>>>socket
> >>>>destructor can be called from __sk_free when sk_wmem_alloc reaches
> >>>>zero, but we
> >>>>use sk_refcnt in the rx path to prevent premature socket cleanup.  If
> >>>>we drain
> >>>>our send queeue while wer'e still processing rx messages, what
> >>>>prevents us from
> >>>>freeing the socket in the tx path, via sk_free while we're still using
> >>>>the
> >>>>socket in the rx path.  Note I don't think this patch is wrong per-se,
> >>>>but it
> >>>>seems to me there is more work to do to properly interlock the use of
> >>>>sk_refcnt
> >>>>and sk_wmem_alloc here (unless I'm just missing something obvious,
> >>>>which is
> >>>>entirely possible, I've been in the sun alot lately :) ).
> >>>
> >>>Hm, __sk_free() calls sk_prot_free() which frees our socket structure
> >>>and in
> >>>sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).
> >>>
> >>>So no matter if having this patch or not, couldn't this use-after-free like
> >>>scenario already happen with the current code?
> >>>
> >>>F.e. through a given call graph like that:
> >>>
> >>>sctp_wfree(skb):
> >>>  1) sock_wfree(skb)
> >>>     -> __sk_free()
> >>
> >>I don't think this can happen.  sk_wmem_alloc is set to 1 in sk_alloc()
> >>and that acts as a guard to make sure that sk_free() has been called
> >>before we try to free things up.  So, in this partcular case, for
> >>__sk_free() to be called, sk_free() had to be called meaning the
> >>last ref on the socket was released.  However, that's not possible since
> >>we are still holding the association and thus holding the socket
> >>associated with it.
> >>
> >I see what your saying, and I agree, with that bias added in sk_alloc, it looks
> >like we won't ever call __sk_free until sk_wmem_alloc is 1 _and_ sk_refcnt is 0.
> >It still seems messy and confusing though.  It would make more sense to me to
> >increment the refcount an additional time when the socket is initalized, and
> >then decrement it again when the socket is closed and sk_wmem_alloc reaches
> >zero.  That would isolate the refcounting to a single variable.
> 
> See commit 2b85a34e911bf483c27cfdd124aeb1605145dc80.  The whole
> sk_wmem_alloc tric was done so that we dont have to do
> sock_hold/sock_put on transmits.
> 
I know about why it was done, I'm just proposing that we don't have to do it
that way to get the speedup it gives us, and make the code a bit more clear at
the same time.  If we set the refcnt to 2 at init time, we can decrement that
added initial bias when the sk is marked as SOCK_DEAD and the sk_wmem_queued
reaches 0.  We still don't have to do a sock_hold/put on every tx, and we keep
all the reference counting in one location.

Alternatively we can move all the reference counting to sk_wmem_alloc, and have
sock_hold/put increment that field by one instead of sk_refcnt, meaning we could
remove that field

We don't really have to do any of this, of course, but not having looked at it
in some time, it seems confusing to me to have have a single additional ref
count tracked in sk_wmem_alloc.

> It might be good to see if we can do that in sctp as well.
Not sure what you mean by "that" here.  You mean remove the additional uses of
sctp_hold/put?

Neil

> 
> -vlad
> 
> 
> >Neil
> >
> >>-vlad
> >>
> >>>      -> sk_prot_free(.., sk)
> >>>       -> kmem_cache_free(.., sk) or kfree(sk)
> >>>  2) __sctp_write_space(asoc)
> >>>  3) sctp_association_put(asoc)
> >>>     -> sctp_association_destroy(asoc)
> >>>      -> sctp_endpoint_put(asoc->ep)
> >>>       -> sctp_endpoint_destroy(ep)
> >>>        -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
> >>>           (etc, all unconditionally accessed while sk is
> >>>            already dead/freed)
> >>>
> >>>Then, this might need a fix in general. :-)
> >>>
> >>>Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF,
> >>>..),
> >>>you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and
> >>>a call to
> >>>sk->sk_write_space(sk), which is sctp_write_space() and calls
> >>>__sctp_write_space()
> >>>on all asocs belonging to the socket, but it seems not to alter the current
> >>>sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.
> >>>
> >>>[*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
> >>>     SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
> >>>     on skb->truesize as well?
> >>>--
> >>>To unsubscribe from this list: send the line "unsubscribe linux-sctp" in
> >>>the body of a message to majordomo@vger.kernel.org
> >>>More majordomo info at  http://vger.kernel.org/majordomo-info.html
> >>
> >>
> 
> 
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Vladislav Yasevich June 18, 2013, 10:55 p.m. UTC | #7
On 06/18/2013 03:12 PM, Neil Horman wrote:
> On Tue, Jun 18, 2013 at 02:15:21PM -0400, Vlad Yasevich wrote:
>> On 06/18/2013 01:45 PM, Neil Horman wrote:
>>> On Tue, Jun 18, 2013 at 12:24:31PM -0400, Vlad Yasevich wrote:
>>>> On 06/18/2013 12:02 PM, Daniel Borkmann wrote:
>>>>> On 06/18/2013 04:22 PM, Neil Horman wrote:
>>>>>> I like this idea, but I think I'm maybe missing something from it - we
>>>>>> reference
>>>>>> the socket in both the receive and send paths (sctp_unpack_cookie, is
>>>>>> specifically called from the rx path, which makes use of sp->hmac).  a
>>>>>> socket
>>>>>> destructor can be called from __sk_free when sk_wmem_alloc reaches
>>>>>> zero, but we
>>>>>> use sk_refcnt in the rx path to prevent premature socket cleanup.  If
>>>>>> we drain
>>>>>> our send queeue while wer'e still processing rx messages, what
>>>>>> prevents us from
>>>>>> freeing the socket in the tx path, via sk_free while we're still using
>>>>>> the
>>>>>> socket in the rx path.  Note I don't think this patch is wrong per-se,
>>>>>> but it
>>>>>> seems to me there is more work to do to properly interlock the use of
>>>>>> sk_refcnt
>>>>>> and sk_wmem_alloc here (unless I'm just missing something obvious,
>>>>>> which is
>>>>>> entirely possible, I've been in the sun alot lately :) ).
>>>>>
>>>>> Hm, __sk_free() calls sk_prot_free() which frees our socket structure
>>>>> and in
>>>>> sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).
>>>>>
>>>>> So no matter if having this patch or not, couldn't this use-after-free like
>>>>> scenario already happen with the current code?
>>>>>
>>>>> F.e. through a given call graph like that:
>>>>>
>>>>> sctp_wfree(skb):
>>>>>   1) sock_wfree(skb)
>>>>>      -> __sk_free()
>>>>
>>>> I don't think this can happen.  sk_wmem_alloc is set to 1 in sk_alloc()
>>>> and that acts as a guard to make sure that sk_free() has been called
>>>> before we try to free things up.  So, in this partcular case, for
>>>> __sk_free() to be called, sk_free() had to be called meaning the
>>>> last ref on the socket was released.  However, that's not possible since
>>>> we are still holding the association and thus holding the socket
>>>> associated with it.
>>>>
>>> I see what your saying, and I agree, with that bias added in sk_alloc, it looks
>>> like we won't ever call __sk_free until sk_wmem_alloc is 1 _and_ sk_refcnt is 0.
>>> It still seems messy and confusing though.  It would make more sense to me to
>>> increment the refcount an additional time when the socket is initalized, and
>>> then decrement it again when the socket is closed and sk_wmem_alloc reaches
>>> zero.  That would isolate the refcounting to a single variable.
>>
>> See commit 2b85a34e911bf483c27cfdd124aeb1605145dc80.  The whole
>> sk_wmem_alloc tric was done so that we dont have to do
>> sock_hold/sock_put on transmits.
>>
> I know about why it was done, I'm just proposing that we don't have to do it
> that way to get the speedup it gives us, and make the code a bit more clear at
> the same time.  If we set the refcnt to 2 at init time, we can decrement that
> added initial bias when the sk is marked as SOCK_DEAD and the sk_wmem_queued
> reaches 0.  We still don't have to do a sock_hold/put on every tx, and we keep
> all the reference counting in one location.

sorry, not following..  regardless, this seems to have gotten a bit off 
topic.  I am going to take a deeper look at Daniel's to make sure it 
doesn't introduce any races.

-vlad

>
> Alternatively we can move all the reference counting to sk_wmem_alloc, and have
> sock_hold/put increment that field by one instead of sk_refcnt, meaning we could
> remove that field
>
> We don't really have to do any of this, of course, but not having looked at it
> in some time, it seems confusing to me to have have a single additional ref
> count tracked in sk_wmem_alloc.
>
>> It might be good to see if we can do that in sctp as well.
> Not sure what you mean by "that" here.  You mean remove the additional uses of
> sctp_hold/put?
>
> Neil
>
>>
>> -vlad
>>
>>
>>> Neil
>>>
>>>> -vlad
>>>>
>>>>>       -> sk_prot_free(.., sk)
>>>>>        -> kmem_cache_free(.., sk) or kfree(sk)
>>>>>   2) __sctp_write_space(asoc)
>>>>>   3) sctp_association_put(asoc)
>>>>>      -> sctp_association_destroy(asoc)
>>>>>       -> sctp_endpoint_put(asoc->ep)
>>>>>        -> sctp_endpoint_destroy(ep)
>>>>>         -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
>>>>>            (etc, all unconditionally accessed while sk is
>>>>>             already dead/freed)
>>>>>
>>>>> Then, this might need a fix in general. :-)
>>>>>
>>>>> Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF,
>>>>> ..),
>>>>> you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and
>>>>> a call to
>>>>> sk->sk_write_space(sk), which is sctp_write_space() and calls
>>>>> __sctp_write_space()
>>>>> on all asocs belonging to the socket, but it seems not to alter the current
>>>>> sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.
>>>>>
>>>>> [*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
>>>>>      SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
>>>>>      on skb->truesize as well?
>>>>> --
>>>>> To unsubscribe from this list: send the line "unsubscribe linux-sctp" in
>>>>> the body of a message to majordomo@vger.kernel.org
>>>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>>>
>>>>
>>
>>

--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Neil Horman June 19, 2013, 11:55 a.m. UTC | #8
On Tue, Jun 18, 2013 at 06:55:27PM -0400, Vlad Yasevich wrote:
> On 06/18/2013 03:12 PM, Neil Horman wrote:
> >On Tue, Jun 18, 2013 at 02:15:21PM -0400, Vlad Yasevich wrote:
> >>On 06/18/2013 01:45 PM, Neil Horman wrote:
> >>>On Tue, Jun 18, 2013 at 12:24:31PM -0400, Vlad Yasevich wrote:
> >>>>On 06/18/2013 12:02 PM, Daniel Borkmann wrote:
> >>>>>On 06/18/2013 04:22 PM, Neil Horman wrote:
> >>>>>>I like this idea, but I think I'm maybe missing something from it - we
> >>>>>>reference
> >>>>>>the socket in both the receive and send paths (sctp_unpack_cookie, is
> >>>>>>specifically called from the rx path, which makes use of sp->hmac).  a
> >>>>>>socket
> >>>>>>destructor can be called from __sk_free when sk_wmem_alloc reaches
> >>>>>>zero, but we
> >>>>>>use sk_refcnt in the rx path to prevent premature socket cleanup.  If
> >>>>>>we drain
> >>>>>>our send queeue while wer'e still processing rx messages, what
> >>>>>>prevents us from
> >>>>>>freeing the socket in the tx path, via sk_free while we're still using
> >>>>>>the
> >>>>>>socket in the rx path.  Note I don't think this patch is wrong per-se,
> >>>>>>but it
> >>>>>>seems to me there is more work to do to properly interlock the use of
> >>>>>>sk_refcnt
> >>>>>>and sk_wmem_alloc here (unless I'm just missing something obvious,
> >>>>>>which is
> >>>>>>entirely possible, I've been in the sun alot lately :) ).
> >>>>>
> >>>>>Hm, __sk_free() calls sk_prot_free() which frees our socket structure
> >>>>>and in
> >>>>>sctp_wfree() we do a sctp_association_put(asoc) after sock_wfree(skb).
> >>>>>
> >>>>>So no matter if having this patch or not, couldn't this use-after-free like
> >>>>>scenario already happen with the current code?
> >>>>>
> >>>>>F.e. through a given call graph like that:
> >>>>>
> >>>>>sctp_wfree(skb):
> >>>>>  1) sock_wfree(skb)
> >>>>>     -> __sk_free()
> >>>>
> >>>>I don't think this can happen.  sk_wmem_alloc is set to 1 in sk_alloc()
> >>>>and that acts as a guard to make sure that sk_free() has been called
> >>>>before we try to free things up.  So, in this partcular case, for
> >>>>__sk_free() to be called, sk_free() had to be called meaning the
> >>>>last ref on the socket was released.  However, that's not possible since
> >>>>we are still holding the association and thus holding the socket
> >>>>associated with it.
> >>>>
> >>>I see what your saying, and I agree, with that bias added in sk_alloc, it looks
> >>>like we won't ever call __sk_free until sk_wmem_alloc is 1 _and_ sk_refcnt is 0.
> >>>It still seems messy and confusing though.  It would make more sense to me to
> >>>increment the refcount an additional time when the socket is initalized, and
> >>>then decrement it again when the socket is closed and sk_wmem_alloc reaches
> >>>zero.  That would isolate the refcounting to a single variable.
> >>
> >>See commit 2b85a34e911bf483c27cfdd124aeb1605145dc80.  The whole
> >>sk_wmem_alloc tric was done so that we dont have to do
> >>sock_hold/sock_put on transmits.
> >>
> >I know about why it was done, I'm just proposing that we don't have to do it
> >that way to get the speedup it gives us, and make the code a bit more clear at
> >the same time.  If we set the refcnt to 2 at init time, we can decrement that
> >added initial bias when the sk is marked as SOCK_DEAD and the sk_wmem_queued
> >reaches 0.  We still don't have to do a sock_hold/put on every tx, and we keep
> >all the reference counting in one location.
> 
> sorry, not following..  regardless, this seems to have gotten a bit
SOCK_DEAD set in sk->flags IIRC should indicate that the socket has been
orphaned from any user space file descriptors (i.e. the same meaning that the
initial bias to sk_wmem_alloc is telling us).  I'm just saying we can use that
to replace the sk_wmem_alloc == 0 test.

> off topic.  I am going to take a deeper look at Daniel's to make
> sure it doesn't introduce any races.
Yes, this is off topic, I was really just complaining that the current scheme
makes it harder to evaluate patches like this.

Neil

> 
> -vlad
> 
> >
> >Alternatively we can move all the reference counting to sk_wmem_alloc, and have
> >sock_hold/put increment that field by one instead of sk_refcnt, meaning we could
> >remove that field
> >
> >We don't really have to do any of this, of course, but not having looked at it
> >in some time, it seems confusing to me to have have a single additional ref
> >count tracked in sk_wmem_alloc.
> >
> >>It might be good to see if we can do that in sctp as well.
> >Not sure what you mean by "that" here.  You mean remove the additional uses of
> >sctp_hold/put?
> >
> >Neil
> >
> >>
> >>-vlad
> >>
> >>
> >>>Neil
> >>>
> >>>>-vlad
> >>>>
> >>>>>      -> sk_prot_free(.., sk)
> >>>>>       -> kmem_cache_free(.., sk) or kfree(sk)
> >>>>>  2) __sctp_write_space(asoc)
> >>>>>  3) sctp_association_put(asoc)
> >>>>>     -> sctp_association_destroy(asoc)
> >>>>>      -> sctp_endpoint_put(asoc->ep)
> >>>>>       -> sctp_endpoint_destroy(ep)
> >>>>>        -> crypto_free_hash(sctp_sk(ep->base.sk)->hmac)
> >>>>>           (etc, all unconditionally accessed while sk is
> >>>>>            already dead/freed)
> >>>>>
> >>>>>Then, this might need a fix in general. :-)
> >>>>>
> >>>>>Assuming you would reduce the buffer space via setsockopt(.., SO_SNDBUF,
> >>>>>..),
> >>>>>you might end up with a minimum buffer space of SOCK_MIN_SNDBUF [*] and
> >>>>>a call to
> >>>>>sk->sk_write_space(sk), which is sctp_write_space() and calls
> >>>>>__sctp_write_space()
> >>>>>on all asocs belonging to the socket, but it seems not to alter the current
> >>>>>sk->sk_wmem_alloc I think, but rather sk->sk_sndbuf.
> >>>>>
> >>>>>[*] Btw, shouldn't this rather be (2048 + sizeof(struct sk_buff)) or
> >>>>>     SKB_TRUESIZE(2048), at least like in SOCK_MIN_RCVBUF since we operate
> >>>>>     on skb->truesize as well?
> >>>>>--
> >>>>>To unsubscribe from this list: send the line "unsubscribe linux-sctp" in
> >>>>>the body of a message to majordomo@vger.kernel.org
> >>>>>More majordomo info at  http://vger.kernel.org/majordomo-info.html
> >>>>
> >>>>
> >>
> >>
> 
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Vladislav Yasevich June 20, 2013, 2:35 p.m. UTC | #9
On 06/18/2013 04:55 AM, Daniel Borkmann wrote:
> Rather instead of having the endpoint clean the garbage for the
> socket, use a sk_destruct handler sctp_destruct_sock(), that does
> the job for that when there are no more references on the socket.
>

With this patch it is possible to run sctp_put_port while the socket
is not locked.

The flow goes something like this:

sctp_close()
   sk_bh_lock_sock();
   sk_common_release()
     sctp_destroy_sock()
       endpoint_put()
         endpoint_destroy() <-- This is where we used to do sctp_put_port
   sk_bh_unlock_sock();
   sock_put()
     sk_free()
       __sk_free()
         sctp_destruct_sock()
           sctp_put_port()


I haven't found any race conditions yet, but that doesn't mean they 
don't exist.

I think an easy solution would be to do sctp_put_port in sctp_unhash(),
but I haven't traced all the paths.

Daniel, please take a look at this.

Thanks
-vlad


> Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
> ---
>   net/sctp/endpointola.c |  7 -------
>   net/sctp/socket.c      | 20 +++++++++++++++++++-
>   2 files changed, 19 insertions(+), 8 deletions(-)
>
> diff --git a/net/sctp/endpointola.c b/net/sctp/endpointola.c
> index a8b2674..9a9ebd2 100644
> --- a/net/sctp/endpointola.c
> +++ b/net/sctp/endpointola.c
> @@ -249,9 +249,6 @@ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
>   {
>   	SCTP_ASSERT(ep->base.dead, "Endpoint is not dead", return);
>
> -	/* Free up the HMAC transform. */
> -	crypto_free_hash(sctp_sk(ep->base.sk)->hmac);
> -
>   	/* Free the digest buffer */
>   	kfree(ep->digest);
>
> @@ -271,10 +268,6 @@ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
>
>   	memset(ep->secret_key, 0, sizeof(ep->secret_key));
>
> -	/* Remove and free the port */
> -	if (sctp_sk(ep->base.sk)->bind_hash)
> -		sctp_put_port(ep->base.sk);
> -
>   	/* Give up our hold on the sock. */
>   	if (ep->base.sk)
>   		sock_put(ep->base.sk);
> diff --git a/net/sctp/socket.c b/net/sctp/socket.c
> index 68a6b70..67f721e 100644
> --- a/net/sctp/socket.c
> +++ b/net/sctp/socket.c
> @@ -93,6 +93,7 @@ static int sctp_wait_for_packet(struct sock * sk, int *err, long *timeo_p);
>   static int sctp_wait_for_connect(struct sctp_association *, long *timeo_p);
>   static int sctp_wait_for_accept(struct sock *sk, long timeo);
>   static void sctp_wait_for_close(struct sock *sk, long timeo);
> +static void sctp_destruct_sock(struct sock *sk);
>   static struct sctp_af *sctp_sockaddr_af(struct sctp_sock *opt,
>   					union sctp_addr *addr, int len);
>   static int sctp_bindx_add(struct sock *, struct sockaddr *, int);
> @@ -3966,6 +3967,8 @@ static int sctp_init_sock(struct sock *sk)
>
>   	sp->hmac = NULL;
>
> +	sk->sk_destruct = sctp_destruct_sock;
> +
>   	SCTP_DBG_OBJCNT_INC(sock);
>
>   	local_bh_disable();
> @@ -4002,6 +4005,21 @@ static void sctp_destroy_sock(struct sock *sk)
>   	local_bh_enable();
>   }
>
> +/* Triggered when there are no references on the socket anymore */
> +static void sctp_destruct_sock(struct sock *sk)
> +{
> +	struct sctp_sock *sp = sctp_sk(sk);
> +
> +	/* Free up the HMAC transform. */
> +	crypto_free_hash(sp->hmac);
> +
> +	/* Remove and free the port */
> +	if (sp->bind_hash)
> +		sctp_put_port(sk);
> +
> +	inet_sock_destruct(sk);
> +}
> +
>   /* API 4.1.7 shutdown() - TCP Style Syntax
>    *     int shutdown(int socket, int how);
>    *
> @@ -6842,7 +6860,7 @@ void sctp_copy_sock(struct sock *newsk, struct sock *sk,
>   	newsk->sk_reuse = sk->sk_reuse;
>
>   	newsk->sk_shutdown = sk->sk_shutdown;
> -	newsk->sk_destruct = inet_sock_destruct;
> +	newsk->sk_destruct = sctp_destruct_sock;
>   	newsk->sk_family = sk->sk_family;
>   	newsk->sk_protocol = IPPROTO_SCTP;
>   	newsk->sk_backlog_rcv = sk->sk_prot->backlog_rcv;
>

--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Daniel Borkmann June 20, 2013, 5:29 p.m. UTC | #10
On 06/20/2013 04:35 PM, Vlad Yasevich wrote:
> On 06/18/2013 04:55 AM, Daniel Borkmann wrote:
>> Rather instead of having the endpoint clean the garbage for the
>> socket, use a sk_destruct handler sctp_destruct_sock(), that does
>> the job for that when there are no more references on the socket.
>
> With this patch it is possible to run sctp_put_port while the socket
> is not locked.
>
> The flow goes something like this:
>
> sctp_close()
>    sk_bh_lock_sock();
>    sk_common_release()
>      sctp_destroy_sock()
>        endpoint_put()
>          endpoint_destroy() <-- This is where we used to do sctp_put_port
>    sk_bh_unlock_sock();
>    sock_put()
>      sk_free()
>        __sk_free()
>          sctp_destruct_sock()
>            sctp_put_port()
>
> I haven't found any race conditions yet, but that doesn't mean they don't exist.
>
> I think an easy solution would be to do sctp_put_port in sctp_unhash(),
> but I haven't traced all the paths.

Hm, compared to the current (pre-patch) solution, sctp_put_port() does not
necessarily need to be called at sk_common_release() time if refs are still
on the endpoint, so that endpoint_destroy() is further deferred in time. Thus,
if we would do the sctp_put_port() in sctp_unhash(), we could free it at an
earlier time than with endpoint_destroy(). This does not necessarily need to
be a bad or wrong way, but with the current approach it's done at an later
point in time afaik. If it's only about the locking, what if we just hold
that socket lock around sctp_put_port() in the current patch?

But besides that, if at such a late point in time someone still has access to
that socket member (right before we do the kfree(sk)), we would be pretty much
screwed. :-) Despite having the socket lock or not, the port hashtable has it's
own protection from what I see.
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Vladislav Yasevich June 21, 2013, 1:12 a.m. UTC | #11
On 06/20/2013 01:29 PM, Daniel Borkmann wrote:
> On 06/20/2013 04:35 PM, Vlad Yasevich wrote:
>> On 06/18/2013 04:55 AM, Daniel Borkmann wrote:
>>> Rather instead of having the endpoint clean the garbage for the
>>> socket, use a sk_destruct handler sctp_destruct_sock(), that does
>>> the job for that when there are no more references on the socket.
>>
>> With this patch it is possible to run sctp_put_port while the socket
>> is not locked.
>>
>> The flow goes something like this:
>>
>> sctp_close()
>>    sk_bh_lock_sock();
>>    sk_common_release()
>>      sctp_destroy_sock()
>>        endpoint_put()
>>          endpoint_destroy() <-- This is where we used to do sctp_put_port
>>    sk_bh_unlock_sock();
>>    sock_put()
>>      sk_free()
>>        __sk_free()
>>          sctp_destruct_sock()
>>            sctp_put_port()
>>
>> I haven't found any race conditions yet, but that doesn't mean they
>> don't exist.
>>
>> I think an easy solution would be to do sctp_put_port in sctp_unhash(),
>> but I haven't traced all the paths.
>
> Hm, compared to the current (pre-patch) solution, sctp_put_port() does not
> necessarily need to be called at sk_common_release() time if refs are still
> on the endpoint, so that endpoint_destroy() is further deferred in time.
> Thus,
> if we would do the sctp_put_port() in sctp_unhash(), we could free it at an
> earlier time than with endpoint_destroy(). This does not necessarily
> need to
> be a bad or wrong way, but with the current approach it's done at an later
> point in time afaik.

You are right. Doing it in sctp_unhash() could be possibly too early.

> If it's only about the locking, what if we just hold
> that socket lock around sctp_put_port() in the current patch?
>
> But besides that, if at such a late point in time someone still has
> access to
> that socket member (right before we do the kfree(sk)), we would be
> pretty much
> screwed. :-) Despite having the socket lock or not, the port hashtable
> has it's
> own protection from what I see.

Yes it does and I've been looking to see if this is sufficient enough
for our purposes.  It looks like our saving grace is the fact that
we set the sk_state to CLOSED sctp_endpoint_free().  Otherwise, we'd
have a race between sctp_endpoint_destroy() and conflict detection
in sctp_get_port_local.  This seems a bit fragile and we are making
it a bit so more with this patch.

I think it would be better to see if we can remove the socket from
the port table a bit earlier if possbile.

-vlad

--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Daniel Borkmann June 22, 2013, 9:38 p.m. UTC | #12
On 06/21/2013 03:12 AM, Vlad Yasevich wrote:
> On 06/20/2013 01:29 PM, Daniel Borkmann wrote:
>> On 06/20/2013 04:35 PM, Vlad Yasevich wrote:
>>> On 06/18/2013 04:55 AM, Daniel Borkmann wrote:
>>>> Rather instead of having the endpoint clean the garbage for the
>>>> socket, use a sk_destruct handler sctp_destruct_sock(), that does
>>>> the job for that when there are no more references on the socket.
>>>
>>> With this patch it is possible to run sctp_put_port while the socket
>>> is not locked.
>>>
>>> The flow goes something like this:
>>>
>>> sctp_close()
>>>    sk_bh_lock_sock();
>>>    sk_common_release()
>>>      sctp_destroy_sock()
>>>        endpoint_put()
>>>          endpoint_destroy() <-- This is where we used to do sctp_put_port
>>>    sk_bh_unlock_sock();
>>>    sock_put()
>>>      sk_free()
>>>        __sk_free()
>>>          sctp_destruct_sock()
>>>            sctp_put_port()
>>>
>>> I haven't found any race conditions yet, but that doesn't mean they
>>> don't exist.
>>>
>>> I think an easy solution would be to do sctp_put_port in sctp_unhash(),
>>> but I haven't traced all the paths.
>>
>> Hm, compared to the current (pre-patch) solution, sctp_put_port() does not
>> necessarily need to be called at sk_common_release() time if refs are still
>> on the endpoint, so that endpoint_destroy() is further deferred in time.
>> Thus,
>> if we would do the sctp_put_port() in sctp_unhash(), we could free it at an
>> earlier time than with endpoint_destroy(). This does not necessarily
>> need to
>> be a bad or wrong way, but with the current approach it's done at an later
>> point in time afaik.
>
> You are right. Doing it in sctp_unhash() could be possibly too early.
>
>> If it's only about the locking, what if we just hold
>> that socket lock around sctp_put_port() in the current patch?
>>
>> But besides that, if at such a late point in time someone still has
>> access to
>> that socket member (right before we do the kfree(sk)), we would be
>> pretty much
>> screwed. :-) Despite having the socket lock or not, the port hashtable
>> has it's
>> own protection from what I see.
>
> Yes it does and I've been looking to see if this is sufficient enough
> for our purposes.  It looks like our saving grace is the fact that
> we set the sk_state to CLOSED sctp_endpoint_free().  Otherwise, we'd
> have a race between sctp_endpoint_destroy() and conflict detection
> in sctp_get_port_local.  This seems a bit fragile and we are making
> it a bit so more with this patch.
>
> I think it would be better to see if we can remove the socket from
> the port table a bit earlier if possbile.

Ok, let me come back to this on Monday.
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/net/sctp/endpointola.c b/net/sctp/endpointola.c
index a8b2674..9a9ebd2 100644
--- a/net/sctp/endpointola.c
+++ b/net/sctp/endpointola.c
@@ -249,9 +249,6 @@  static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
 {
 	SCTP_ASSERT(ep->base.dead, "Endpoint is not dead", return);
 
-	/* Free up the HMAC transform. */
-	crypto_free_hash(sctp_sk(ep->base.sk)->hmac);
-
 	/* Free the digest buffer */
 	kfree(ep->digest);
 
@@ -271,10 +268,6 @@  static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
 
 	memset(ep->secret_key, 0, sizeof(ep->secret_key));
 
-	/* Remove and free the port */
-	if (sctp_sk(ep->base.sk)->bind_hash)
-		sctp_put_port(ep->base.sk);
-
 	/* Give up our hold on the sock. */
 	if (ep->base.sk)
 		sock_put(ep->base.sk);
diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index 68a6b70..67f721e 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -93,6 +93,7 @@  static int sctp_wait_for_packet(struct sock * sk, int *err, long *timeo_p);
 static int sctp_wait_for_connect(struct sctp_association *, long *timeo_p);
 static int sctp_wait_for_accept(struct sock *sk, long timeo);
 static void sctp_wait_for_close(struct sock *sk, long timeo);
+static void sctp_destruct_sock(struct sock *sk);
 static struct sctp_af *sctp_sockaddr_af(struct sctp_sock *opt,
 					union sctp_addr *addr, int len);
 static int sctp_bindx_add(struct sock *, struct sockaddr *, int);
@@ -3966,6 +3967,8 @@  static int sctp_init_sock(struct sock *sk)
 
 	sp->hmac = NULL;
 
+	sk->sk_destruct = sctp_destruct_sock;
+
 	SCTP_DBG_OBJCNT_INC(sock);
 
 	local_bh_disable();
@@ -4002,6 +4005,21 @@  static void sctp_destroy_sock(struct sock *sk)
 	local_bh_enable();
 }
 
+/* Triggered when there are no references on the socket anymore */
+static void sctp_destruct_sock(struct sock *sk)
+{
+	struct sctp_sock *sp = sctp_sk(sk);
+
+	/* Free up the HMAC transform. */
+	crypto_free_hash(sp->hmac);
+
+	/* Remove and free the port */
+	if (sp->bind_hash)
+		sctp_put_port(sk);
+
+	inet_sock_destruct(sk);
+}
+
 /* API 4.1.7 shutdown() - TCP Style Syntax
  *     int shutdown(int socket, int how);
  *
@@ -6842,7 +6860,7 @@  void sctp_copy_sock(struct sock *newsk, struct sock *sk,
 	newsk->sk_reuse = sk->sk_reuse;
 
 	newsk->sk_shutdown = sk->sk_shutdown;
-	newsk->sk_destruct = inet_sock_destruct;
+	newsk->sk_destruct = sctp_destruct_sock;
 	newsk->sk_family = sk->sk_family;
 	newsk->sk_protocol = IPPROTO_SCTP;
 	newsk->sk_backlog_rcv = sk->sk_prot->backlog_rcv;