From patchwork Fri Apr 5 21:36:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Long Li X-Patchwork-Id: 1078823 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=linux-cifs-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=linuxonhyperv.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 44bY7t35Lvz9sPg for ; Sat, 6 Apr 2019 08:37:46 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726204AbfDEVhp (ORCPT ); Fri, 5 Apr 2019 17:37:45 -0400 Received: from a2nlsmtp01-03.prod.iad2.secureserver.net ([198.71.225.37]:53672 "EHLO a2nlsmtp01-03.prod.iad2.secureserver.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1725973AbfDEVhp (ORCPT ); Fri, 5 Apr 2019 17:37:45 -0400 Received: from linuxonhyperv2.linuxonhyperv.com ([107.180.71.197]) by : HOSTING RELAY : with ESMTP id CWVvhh1UqiCMnCWVvhowpd; Fri, 05 Apr 2019 14:36:43 -0700 x-originating-ip: 107.180.71.197 Received: from longli by linuxonhyperv2.linuxonhyperv.com with local (Exim 4.91) (envelope-from ) id 1hCWVv-0006Lw-2r; Fri, 05 Apr 2019 14:36:43 -0700 From: Long Li To: Steve French , linux-cifs@vger.kernel.org, samba-technical@lists.samba.org, linux-kernel@vger.kernel.org Cc: Long Li Subject: [Patch (resend) 1/5] cifs: smbd: Don't destroy transport on RDMA disconnect Date: Fri, 5 Apr 2019 21:36:31 +0000 Message-Id: <20190405213635.24383-1-longli@linuxonhyperv.com> X-Mailer: git-send-email 2.19.1 MIME-Version: 1.0 Reply-To: longli@microsoft.com X-CMAE-Envelope: MS4wfEk+qRx/dCxIImn/vsS34Cf9Zf7yd+fapvd7fsuU8+aPsqRI3OHmP3vmGRVZzjZ1v8uFJ0ZT2A/1RxTJKBG3T49cT6XN1TQdImyjjSjP/1mVzGsiVJ5m 5mimR5WipjRf3OnYOAGb2f04M8Kx6jFd/rg2xdE0XHAT0UkLGp9/crBMGgbaxQ5feeqZ3ts/sM5rXmKw2I+mikMX4LmVMNgWtPgLEOxx6ycy+wuVv6kEmPuL gQ+zLDqoREyvM4nbtKpubWrV74G1DPqXe1ciP++9A3ACHmgu8qaptIGmhXc219PGHBRZXPh4ByQXvVpE3/EdZp60hrCkHuAgfT6LHcjOQkZlLqHdXqzrk/Io rU6GSTtrXi4bDH6NyNh/qdK/T0tsqg== Sender: linux-cifs-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-cifs@vger.kernel.org From: Long Li Now upper layer is handling the transport shutdown and reconnect, remove the code that handling transport shutdown on RDMA disconnect. Signed-off-by: Long Li --- fs/cifs/cifs_debug.c | 8 ++-- fs/cifs/smbdirect.c | 120 +++------------------------------------------------ fs/cifs/smbdirect.h | 9 ---- 3 files changed, 10 insertions(+), 127 deletions(-) diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index 19ed9ab..5ff0b3d 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -312,12 +312,10 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) atomic_read(&server->smbd_conn->send_credits), atomic_read(&server->smbd_conn->receive_credits), server->smbd_conn->receive_credit_target); - seq_printf(m, "\nPending send_pending: %x send_payload_pending:" - " %x smbd_send_pending: %x smbd_recv_pending: %x", + seq_printf(m, "\nPending send_pending: %x " + "send_payload_pending: %x", atomic_read(&server->smbd_conn->send_pending), - atomic_read(&server->smbd_conn->send_payload_pending), - server->smbd_conn->smbd_send_pending, - server->smbd_conn->smbd_recv_pending); + atomic_read(&server->smbd_conn->send_payload_pending)); seq_printf(m, "\nReceive buffers count_receive_queue: %x " "count_empty_packet_queue: %x", server->smbd_conn->count_receive_queue, diff --git a/fs/cifs/smbdirect.c b/fs/cifs/smbdirect.c index 06449cf..7259427 100644 --- a/fs/cifs/smbdirect.c +++ b/fs/cifs/smbdirect.c @@ -164,95 +164,6 @@ do { \ #define log_rdma_mr(level, fmt, args...) \ log_rdma(level, LOG_RDMA_MR, fmt, ##args) -/* - * Destroy the transport and related RDMA and memory resources - * Need to go through all the pending counters and make sure on one is using - * the transport while it is destroyed - */ -static void smbd_destroy_rdma_work(struct work_struct *work) -{ - struct smbd_response *response; - struct smbd_connection *info = - container_of(work, struct smbd_connection, destroy_work); - unsigned long flags; - - log_rdma_event(INFO, "destroying qp\n"); - ib_drain_qp(info->id->qp); - rdma_destroy_qp(info->id); - - /* Unblock all I/O waiting on the send queue */ - wake_up_interruptible_all(&info->wait_send_queue); - - log_rdma_event(INFO, "cancelling idle timer\n"); - cancel_delayed_work_sync(&info->idle_timer_work); - log_rdma_event(INFO, "cancelling send immediate work\n"); - cancel_delayed_work_sync(&info->send_immediate_work); - - log_rdma_event(INFO, "wait for all send to finish\n"); - wait_event(info->wait_smbd_send_pending, - info->smbd_send_pending == 0); - - log_rdma_event(INFO, "wait for all recv to finish\n"); - wake_up_interruptible(&info->wait_reassembly_queue); - wait_event(info->wait_smbd_recv_pending, - info->smbd_recv_pending == 0); - - log_rdma_event(INFO, "wait for all send posted to IB to finish\n"); - wait_event(info->wait_send_pending, - atomic_read(&info->send_pending) == 0); - wait_event(info->wait_send_payload_pending, - atomic_read(&info->send_payload_pending) == 0); - - log_rdma_event(INFO, "freeing mr list\n"); - wake_up_interruptible_all(&info->wait_mr); - wait_event(info->wait_for_mr_cleanup, - atomic_read(&info->mr_used_count) == 0); - destroy_mr_list(info); - - /* It's not posssible for upper layer to get to reassembly */ - log_rdma_event(INFO, "drain the reassembly queue\n"); - do { - spin_lock_irqsave(&info->reassembly_queue_lock, flags); - response = _get_first_reassembly(info); - if (response) { - list_del(&response->list); - spin_unlock_irqrestore( - &info->reassembly_queue_lock, flags); - put_receive_buffer(info, response); - } else - spin_unlock_irqrestore(&info->reassembly_queue_lock, flags); - } while (response); - - info->reassembly_data_length = 0; - - log_rdma_event(INFO, "free receive buffers\n"); - wait_event(info->wait_receive_queues, - info->count_receive_queue + info->count_empty_packet_queue - == info->receive_credit_max); - destroy_receive_buffers(info); - - ib_free_cq(info->send_cq); - ib_free_cq(info->recv_cq); - ib_dealloc_pd(info->pd); - rdma_destroy_id(info->id); - - /* free mempools */ - mempool_destroy(info->request_mempool); - kmem_cache_destroy(info->request_cache); - - mempool_destroy(info->response_mempool); - kmem_cache_destroy(info->response_cache); - - info->transport_status = SMBD_DESTROYED; - wake_up_all(&info->wait_destroy); -} - -static int smbd_process_disconnected(struct smbd_connection *info) -{ - schedule_work(&info->destroy_work); - return 0; -} - static void smbd_disconnect_rdma_work(struct work_struct *work) { struct smbd_connection *info = @@ -319,8 +230,7 @@ static int smbd_conn_upcall( } info->transport_status = SMBD_DISCONNECTED; - smbd_process_disconnected(info); - wake_up(&info->disconn_wait); + wake_up_interruptible(&info->disconn_wait); wake_up_interruptible(&info->wait_reassembly_queue); wake_up_interruptible_all(&info->wait_send_queue); break; @@ -1501,7 +1411,7 @@ void smbd_destroy(struct TCP_Server_Info *server) if (info->transport_status != SMBD_DISCONNECTED) { rdma_disconnect(server->smbd_conn->id); log_rdma_event(INFO, "wait for transport being disconnected\n"); - wait_event( + wait_event_interruptible( info->disconn_wait, info->transport_status == SMBD_DISCONNECTED); } @@ -1849,12 +1759,6 @@ static struct smbd_connection *_smbd_get_connection( queue_delayed_work(info->workqueue, &info->idle_timer_work, info->keep_alive_interval*HZ); - init_waitqueue_head(&info->wait_smbd_send_pending); - info->smbd_send_pending = 0; - - init_waitqueue_head(&info->wait_smbd_recv_pending); - info->smbd_recv_pending = 0; - init_waitqueue_head(&info->wait_send_pending); atomic_set(&info->send_pending, 0); @@ -1862,7 +1766,6 @@ static struct smbd_connection *_smbd_get_connection( atomic_set(&info->send_payload_pending, 0); INIT_WORK(&info->disconnect_work, smbd_disconnect_rdma_work); - INIT_WORK(&info->destroy_work, smbd_destroy_rdma_work); INIT_WORK(&info->recv_done_work, smbd_recv_done_work); INIT_WORK(&info->post_send_credits_work, smbd_post_send_credits); info->new_credits_offered = 0; @@ -1956,11 +1859,6 @@ static int smbd_recv_buf(struct smbd_connection *info, char *buf, int rc; again: - if (info->transport_status != SMBD_CONNECTED) { - log_read(ERR, "disconnected\n"); - return -ENODEV; - } - /* * No need to hold the reassembly queue lock all the time as we are * the only one reading from the front of the queue. The transport @@ -2076,6 +1974,11 @@ static int smbd_recv_buf(struct smbd_connection *info, char *buf, if (rc) return -ENODEV; + if (info->transport_status != SMBD_CONNECTED) { + log_read(ERR, "disconnected\n"); + return 0; + } + goto again; } @@ -2126,8 +2029,6 @@ int smbd_recv(struct smbd_connection *info, struct msghdr *msg) unsigned int to_read, page_offset; int rc; - info->smbd_recv_pending++; - if (iov_iter_rw(&msg->msg_iter) == WRITE) { /* It's a bug in upper layer to get there */ cifs_dbg(VFS, "CIFS: invalid msg iter dir %u\n", @@ -2158,9 +2059,6 @@ int smbd_recv(struct smbd_connection *info, struct msghdr *msg) } out: - info->smbd_recv_pending--; - wake_up(&info->wait_smbd_recv_pending); - /* SMBDirect will read it all or nothing */ if (rc > 0) msg->msg_iter.count = 0; @@ -2186,7 +2084,6 @@ int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst) struct kvec *iov; int rc; - info->smbd_send_pending++; if (info->transport_status != SMBD_CONNECTED) { rc = -ENODEV; goto done; @@ -2342,9 +2239,6 @@ int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst) wait_event(info->wait_send_payload_pending, atomic_read(&info->send_payload_pending) == 0); - info->smbd_send_pending--; - wake_up(&info->wait_smbd_send_pending); - return rc; } diff --git a/fs/cifs/smbdirect.h b/fs/cifs/smbdirect.h index 73cc923..a3c7b3d 100644 --- a/fs/cifs/smbdirect.h +++ b/fs/cifs/smbdirect.h @@ -70,13 +70,11 @@ struct smbd_connection { int ri_rc; struct completion ri_done; wait_queue_head_t conn_wait; - wait_queue_head_t wait_destroy; wait_queue_head_t disconn_wait; struct completion negotiate_completion; bool negotiate_done; - struct work_struct destroy_work; struct work_struct disconnect_work; struct work_struct recv_done_work; struct work_struct post_send_credits_work; @@ -124,13 +122,6 @@ struct smbd_connection { wait_queue_head_t wait_for_mr_cleanup; /* Activity accoutning */ - /* Pending reqeusts issued from upper layer */ - int smbd_send_pending; - wait_queue_head_t wait_smbd_send_pending; - - int smbd_recv_pending; - wait_queue_head_t wait_smbd_recv_pending; - atomic_t send_pending; wait_queue_head_t wait_send_pending; atomic_t send_payload_pending;