From patchwork Thu Aug 27 22:08:51 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Kamal Mostafa X-Patchwork-Id: 511659 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from huckleberry.canonical.com (huckleberry.canonical.com [91.189.94.19]) by ozlabs.org (Postfix) with ESMTP id A983A14012C; Fri, 28 Aug 2015 08:22:25 +1000 (AEST) Received: from localhost ([127.0.0.1] helo=huckleberry.canonical.com) by huckleberry.canonical.com with esmtp (Exim 4.76) (envelope-from ) id 1ZV5Yr-0008Jy-Fg; Thu, 27 Aug 2015 22:22:21 +0000 Received: from youngberry.canonical.com ([91.189.89.112]) by huckleberry.canonical.com with esmtp (Exim 4.76) (envelope-from ) id 1ZV5Lq-0001Cc-Pv for kernel-team@lists.ubuntu.com; Thu, 27 Aug 2015 22:08:54 +0000 Received: from 1.general.kamal.us.vpn ([10.172.68.52] helo=fourier) by youngberry.canonical.com with esmtpsa (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.76) (envelope-from ) id 1ZV5Lq-0005N1-DS; Thu, 27 Aug 2015 22:08:54 +0000 Received: from kamal by fourier with local (Exim 4.82) (envelope-from ) id 1ZV5Lo-0000uo-6O; Thu, 27 Aug 2015 15:08:52 -0700 From: Kamal Mostafa To: Nicholas Bellinger Subject: [3.19.y-ckt stable] Patch "iscsi-target: Convert iscsi_thread_set usage to kthread.h" has been added to staging queue Date: Thu, 27 Aug 2015 15:08:51 -0700 Message-Id: <1440713331-3488-1-git-send-email-kamal@canonical.com> X-Mailer: git-send-email 1.9.1 X-Extended-Stable: 3.19 Cc: Sagi Grimberg , Kamal Mostafa , Slava Shwartsman , kernel-team@lists.ubuntu.com X-BeenThere: kernel-team@lists.ubuntu.com X-Mailman-Version: 2.1.14 Precedence: list List-Id: Kernel team discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: kernel-team-bounces@lists.ubuntu.com Sender: kernel-team-bounces@lists.ubuntu.com This is a note to let you know that I have just added a patch titled iscsi-target: Convert iscsi_thread_set usage to kthread.h to the linux-3.19.y-queue branch of the 3.19.y-ckt extended stable tree which can be found at: http://kernel.ubuntu.com/git/ubuntu/linux.git/log/?h=linux-3.19.y-queue This patch is scheduled to be released in version 3.19.8-ckt6. If you, or anyone else, feels it should not be added to this tree, please reply to this email. For more information about the 3.19.y-ckt tree, see https://wiki.ubuntu.com/Kernel/Dev/ExtendedStable Thanks. -Kamal ------ From b840dbf7a64d5c0f44df76d8d6bd3a99c2502021 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Thu, 26 Feb 2015 22:19:15 -0800 Subject: iscsi-target: Convert iscsi_thread_set usage to kthread.h commit 88dcd2dab5c23b1c9cfc396246d8f476c872f0ca upstream. This patch converts iscsi-target code to use modern kthread.h API callers for creating RX/TX threads for each new iscsi_conn descriptor, and releasing associated RX/TX threads during connection shutdown. This is done using iscsit_start_kthreads() -> kthread_run() to start new kthreads from within iscsi_post_login_handler(), and invoking kthread_stop() from existing iscsit_close_connection() code. Also, convert iscsit_logout_post_handler_closesession() code to use cmpxchg when determing when iscsit_cause_connection_reinstatement() needs to sleep waiting for completion. Reported-by: Sagi Grimberg Tested-by: Sagi Grimberg Cc: Slava Shwartsman Signed-off-by: Nicholas Bellinger [ kamal: backport to 3.19-stable: iscsi_target_core.h path ] Signed-off-by: Kamal Mostafa --- drivers/target/iscsi/iscsi_target.c | 104 +++++++++++++----------------- drivers/target/iscsi/iscsi_target_core.h | 7 ++ drivers/target/iscsi/iscsi_target_erl0.c | 13 ++-- drivers/target/iscsi/iscsi_target_login.c | 59 +++++++++++++++-- 4 files changed, 114 insertions(+), 69 deletions(-) -- 1.9.1 diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 89eb2ab..0b5a1e8 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -537,7 +537,7 @@ static struct iscsit_transport iscsi_target_transport = { static int __init iscsi_target_init_module(void) { - int ret = 0; + int ret = 0, size; pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); @@ -546,6 +546,7 @@ static int __init iscsi_target_init_module(void) pr_err("Unable to allocate memory for iscsit_global\n"); return -1; } + spin_lock_init(&iscsit_global->ts_bitmap_lock); mutex_init(&auth_id_lock); spin_lock_init(&sess_idr_lock); idr_init(&tiqn_idr); @@ -555,15 +556,11 @@ static int __init iscsi_target_init_module(void) if (ret < 0) goto out; - ret = iscsi_thread_set_init(); - if (ret < 0) + size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long); + iscsit_global->ts_bitmap = vzalloc(size); + if (!iscsit_global->ts_bitmap) { + pr_err("Unable to allocate iscsit_global->ts_bitmap\n"); goto configfs_out; - - if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) != - TARGET_THREAD_SET_COUNT) { - pr_err("iscsi_allocate_thread_sets() returned" - " unexpected value!\n"); - goto ts_out1; } lio_qr_cache = kmem_cache_create("lio_qr_cache", @@ -572,7 +569,7 @@ static int __init iscsi_target_init_module(void) if (!lio_qr_cache) { pr_err("nable to kmem_cache_create() for" " lio_qr_cache\n"); - goto ts_out2; + goto bitmap_out; } lio_dr_cache = kmem_cache_create("lio_dr_cache", @@ -617,10 +614,8 @@ dr_out: kmem_cache_destroy(lio_dr_cache); qr_out: kmem_cache_destroy(lio_qr_cache); -ts_out2: - iscsi_deallocate_thread_sets(); -ts_out1: - iscsi_thread_set_free(); +bitmap_out: + vfree(iscsit_global->ts_bitmap); configfs_out: iscsi_target_deregister_configfs(); out: @@ -630,8 +625,6 @@ out: static void __exit iscsi_target_cleanup_module(void) { - iscsi_deallocate_thread_sets(); - iscsi_thread_set_free(); iscsit_release_discovery_tpg(); iscsit_unregister_transport(&iscsi_target_transport); kmem_cache_destroy(lio_qr_cache); @@ -641,6 +634,7 @@ static void __exit iscsi_target_cleanup_module(void) iscsi_target_deregister_configfs(); + vfree(iscsit_global->ts_bitmap); kfree(iscsit_global); } @@ -3681,17 +3675,16 @@ static int iscsit_send_reject( void iscsit_thread_get_cpumask(struct iscsi_conn *conn) { - struct iscsi_thread_set *ts = conn->thread_set; int ord, cpu; /* - * thread_id is assigned from iscsit_global->ts_bitmap from - * within iscsi_thread_set.c:iscsi_allocate_thread_sets() + * bitmap_id is assigned from iscsit_global->ts_bitmap from + * within iscsit_start_kthreads() * - * Here we use thread_id to determine which CPU that this - * iSCSI connection's iscsi_thread_set will be scheduled to + * Here we use bitmap_id to determine which CPU that this + * iSCSI connection's RX/TX threads will be scheduled to * execute upon. */ - ord = ts->thread_id % cpumask_weight(cpu_online_mask); + ord = conn->bitmap_id % cpumask_weight(cpu_online_mask); for_each_online_cpu(cpu) { if (ord-- == 0) { cpumask_set_cpu(cpu, conn->conn_cpumask); @@ -3880,7 +3873,7 @@ check_rsp_state: switch (state) { case ISTATE_SEND_LOGOUTRSP: if (!iscsit_logout_post_handler(cmd, conn)) - goto restart; + return -ECONNRESET; /* fall through */ case ISTATE_SEND_STATUS: case ISTATE_SEND_ASYNCMSG: @@ -3908,8 +3901,6 @@ check_rsp_state: err: return -1; -restart: - return -EAGAIN; } static int iscsit_handle_response_queue(struct iscsi_conn *conn) @@ -3936,21 +3927,13 @@ static int iscsit_handle_response_queue(struct iscsi_conn *conn) int iscsi_target_tx_thread(void *arg) { int ret = 0; - struct iscsi_conn *conn; - struct iscsi_thread_set *ts = arg; + struct iscsi_conn *conn = arg; /* * Allow ourselves to be interrupted by SIGINT so that a * connection recovery / failure event can be triggered externally. */ allow_signal(SIGINT); -restart: - conn = iscsi_tx_thread_pre_handler(ts); - if (!conn) - goto out; - - ret = 0; - while (!kthread_should_stop()) { /* * Ensure that both TX and RX per connection kthreads @@ -3959,11 +3942,9 @@ restart: iscsit_thread_check_cpumask(conn, current, 1); wait_event_interruptible(conn->queues_wq, - !iscsit_conn_all_queues_empty(conn) || - ts->status == ISCSI_THREAD_SET_RESET); + !iscsit_conn_all_queues_empty(conn)); - if ((ts->status == ISCSI_THREAD_SET_RESET) || - signal_pending(current)) + if (signal_pending(current)) goto transport_err; get_immediate: @@ -3974,15 +3955,14 @@ get_immediate: ret = iscsit_handle_response_queue(conn); if (ret == 1) goto get_immediate; - else if (ret == -EAGAIN) - goto restart; + else if (ret == -ECONNRESET) + goto out; else if (ret < 0) goto transport_err; } transport_err: iscsit_take_action_for_connection_exit(conn); - goto restart; out: return 0; } @@ -4071,8 +4051,7 @@ int iscsi_target_rx_thread(void *arg) int ret; u8 buffer[ISCSI_HDR_LEN], opcode; u32 checksum = 0, digest = 0; - struct iscsi_conn *conn = NULL; - struct iscsi_thread_set *ts = arg; + struct iscsi_conn *conn = arg; struct kvec iov; /* * Allow ourselves to be interrupted by SIGINT so that a @@ -4080,11 +4059,6 @@ int iscsi_target_rx_thread(void *arg) */ allow_signal(SIGINT); -restart: - conn = iscsi_rx_thread_pre_handler(ts); - if (!conn) - goto out; - if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) { struct completion comp; int rc; @@ -4094,7 +4068,7 @@ restart: if (rc < 0) goto transport_err; - goto out; + goto transport_err; } while (!kthread_should_stop()) { @@ -4170,8 +4144,6 @@ transport_err: if (!signal_pending(current)) atomic_set(&conn->transport_failed, 1); iscsit_take_action_for_connection_exit(conn); - goto restart; -out: return 0; } @@ -4233,7 +4205,24 @@ int iscsit_close_connection( if (conn->conn_transport->transport_type == ISCSI_TCP) complete(&conn->conn_logout_comp); - iscsi_release_thread_set(conn); + if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) { + if (conn->tx_thread && + cmpxchg(&conn->tx_thread_active, true, false)) { + send_sig(SIGINT, conn->tx_thread, 1); + kthread_stop(conn->tx_thread); + } + } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) { + if (conn->rx_thread && + cmpxchg(&conn->rx_thread_active, true, false)) { + send_sig(SIGINT, conn->rx_thread, 1); + kthread_stop(conn->rx_thread); + } + } + + spin_lock(&iscsit_global->ts_bitmap_lock); + bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id, + get_order(1)); + spin_unlock(&iscsit_global->ts_bitmap_lock); iscsit_stop_timers_for_cmds(conn); iscsit_stop_nopin_response_timer(conn); @@ -4511,15 +4500,13 @@ static void iscsit_logout_post_handler_closesession( struct iscsi_conn *conn) { struct iscsi_session *sess = conn->sess; - - iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD); - iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD); + int sleep = cmpxchg(&conn->tx_thread_active, true, false); atomic_set(&conn->conn_logout_remove, 0); complete(&conn->conn_logout_comp); iscsit_dec_conn_usage_count(conn); - iscsit_stop_session(sess, 1, 1); + iscsit_stop_session(sess, sleep, sleep); iscsit_dec_session_usage_count(sess); target_put_session(sess->se_sess); } @@ -4527,13 +4514,12 @@ static void iscsit_logout_post_handler_closesession( static void iscsit_logout_post_handler_samecid( struct iscsi_conn *conn) { - iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD); - iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD); + int sleep = cmpxchg(&conn->tx_thread_active, true, false); atomic_set(&conn->conn_logout_remove, 0); complete(&conn->conn_logout_comp); - iscsit_cause_connection_reinstatement(conn, 1); + iscsit_cause_connection_reinstatement(conn, sleep); iscsit_dec_conn_usage_count(conn); } diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h index cbcff38..5dfd179 100644 --- a/drivers/target/iscsi/iscsi_target_core.h +++ b/drivers/target/iscsi/iscsi_target_core.h @@ -602,6 +602,11 @@ struct iscsi_conn { struct iscsi_session *sess; /* Pointer to thread_set in use for this conn's threads */ struct iscsi_thread_set *thread_set; + int bitmap_id; + int rx_thread_active; + struct task_struct *rx_thread; + int tx_thread_active; + struct task_struct *tx_thread; /* list_head for session connection list */ struct list_head conn_list; } ____cacheline_aligned; @@ -871,10 +876,12 @@ struct iscsit_global { /* Unique identifier used for the authentication daemon */ u32 auth_id; u32 inactive_ts; +#define ISCSIT_BITMAP_BITS 262144 /* Thread Set bitmap count */ int ts_bitmap_count; /* Thread Set bitmap pointer */ unsigned long *ts_bitmap; + spinlock_t ts_bitmap_lock; /* Used for iSCSI discovery session authentication */ struct iscsi_node_acl discovery_acl; struct iscsi_portal_group *discovery_tpg; diff --git a/drivers/target/iscsi/iscsi_target_erl0.c b/drivers/target/iscsi/iscsi_target_erl0.c index a0ae5fc..87f6872 100644 --- a/drivers/target/iscsi/iscsi_target_erl0.c +++ b/drivers/target/iscsi/iscsi_target_erl0.c @@ -860,7 +860,10 @@ void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn) } spin_unlock_bh(&conn->state_lock); - iscsi_thread_set_force_reinstatement(conn); + if (conn->tx_thread && conn->tx_thread_active) + send_sig(SIGINT, conn->tx_thread, 1); + if (conn->rx_thread && conn->rx_thread_active) + send_sig(SIGINT, conn->rx_thread, 1); sleep: wait_for_completion(&conn->conn_wait_rcfr_comp); @@ -885,10 +888,10 @@ void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep) return; } - if (iscsi_thread_set_force_reinstatement(conn) < 0) { - spin_unlock_bh(&conn->state_lock); - return; - } + if (conn->tx_thread && conn->tx_thread_active) + send_sig(SIGINT, conn->tx_thread, 1); + if (conn->rx_thread && conn->rx_thread_active) + send_sig(SIGINT, conn->rx_thread, 1); atomic_set(&conn->connection_reinstatement, 1); if (!sleep) { diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c index 713c0c1..3b55fe1 100644 --- a/drivers/target/iscsi/iscsi_target_login.c +++ b/drivers/target/iscsi/iscsi_target_login.c @@ -699,6 +699,51 @@ static void iscsi_post_login_start_timers(struct iscsi_conn *conn) iscsit_start_nopin_timer(conn); } +int iscsit_start_kthreads(struct iscsi_conn *conn) +{ + int ret = 0; + + spin_lock(&iscsit_global->ts_bitmap_lock); + conn->bitmap_id = bitmap_find_free_region(iscsit_global->ts_bitmap, + ISCSIT_BITMAP_BITS, get_order(1)); + spin_unlock(&iscsit_global->ts_bitmap_lock); + + if (conn->bitmap_id < 0) { + pr_err("bitmap_find_free_region() failed for" + " iscsit_start_kthreads()\n"); + return -ENOMEM; + } + + conn->tx_thread = kthread_run(iscsi_target_tx_thread, conn, + "%s", ISCSI_TX_THREAD_NAME); + if (IS_ERR(conn->tx_thread)) { + pr_err("Unable to start iscsi_target_tx_thread\n"); + ret = PTR_ERR(conn->tx_thread); + goto out_bitmap; + } + conn->tx_thread_active = true; + + conn->rx_thread = kthread_run(iscsi_target_rx_thread, conn, + "%s", ISCSI_RX_THREAD_NAME); + if (IS_ERR(conn->rx_thread)) { + pr_err("Unable to start iscsi_target_rx_thread\n"); + ret = PTR_ERR(conn->rx_thread); + goto out_tx; + } + conn->rx_thread_active = true; + + return 0; +out_tx: + kthread_stop(conn->tx_thread); + conn->tx_thread_active = false; +out_bitmap: + spin_lock(&iscsit_global->ts_bitmap_lock); + bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id, + get_order(1)); + spin_unlock(&iscsit_global->ts_bitmap_lock); + return ret; +} + int iscsi_post_login_handler( struct iscsi_np *np, struct iscsi_conn *conn, @@ -709,7 +754,7 @@ int iscsi_post_login_handler( struct se_session *se_sess = sess->se_sess; struct iscsi_portal_group *tpg = sess->tpg; struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; - struct iscsi_thread_set *ts; + int rc; iscsit_inc_conn_usage_count(conn); @@ -724,7 +769,6 @@ int iscsi_post_login_handler( /* * SCSI Initiator -> SCSI Target Port Mapping */ - ts = iscsi_get_thread_set(); if (!zero_tsih) { iscsi_set_session_parameters(sess->sess_ops, conn->param_list, 0); @@ -751,9 +795,11 @@ int iscsi_post_login_handler( sess->sess_ops->InitiatorName); spin_unlock_bh(&sess->conn_lock); - iscsi_post_login_start_timers(conn); + rc = iscsit_start_kthreads(conn); + if (rc) + return rc; - iscsi_activate_thread_set(conn, ts); + iscsi_post_login_start_timers(conn); /* * Determine CPU mask to ensure connection's RX and TX kthreads * are scheduled on the same CPU. @@ -810,8 +856,11 @@ int iscsi_post_login_handler( " iSCSI Target Portal Group: %hu\n", tpg->nsessions, tpg->tpgt); spin_unlock_bh(&se_tpg->session_lock); + rc = iscsit_start_kthreads(conn); + if (rc) + return rc; + iscsi_post_login_start_timers(conn); - iscsi_activate_thread_set(conn, ts); /* * Determine CPU mask to ensure connection's RX and TX kthreads * are scheduled on the same CPU.