Message ID | 20210223131820.53813-1-i.maximets@ovn.org |
---|---|
State | Accepted |
Headers | show |
Series | [ovs-dev] raft: Add 'stop-raft-rpc' failure test command. | expand |
On Tue, Feb 23, 2021 at 5:18 AM Ilya Maximets <i.maximets@ovn.org> wrote: > > This command will stop sending and receiving any RAFT-related > traffic or accepting new connections. Useful to simulate > network problems between cluster members. > > There is no unit test that uses it yet, but it's convenient for > manual testing. > > Signed-off-by: Ilya Maximets <i.maximets@ovn.org> > --- > ovsdb/raft.c | 31 +++++++++++++++++++++---------- > 1 file changed, 21 insertions(+), 10 deletions(-) > > diff --git a/ovsdb/raft.c b/ovsdb/raft.c > index 192f7f0a9..d2ff643b2 100644 > --- a/ovsdb/raft.c > +++ b/ovsdb/raft.c > @@ -75,7 +75,8 @@ enum raft_failure_test { > FT_CRASH_AFTER_SEND_EXEC_REQ, > FT_CRASH_AFTER_RECV_APPEND_REQ_UPDATE, > FT_DELAY_ELECTION, > - FT_DONT_SEND_VOTE_REQUEST > + FT_DONT_SEND_VOTE_REQUEST, > + FT_STOP_RAFT_RPC, > }; > static enum raft_failure_test failure_test; > > @@ -1474,6 +1475,10 @@ raft_send_add_server_request(struct raft *raft, struct raft_conn *conn) > static void > raft_conn_run(struct raft *raft, struct raft_conn *conn) > { > + if (failure_test == FT_STOP_RAFT_RPC) { > + return; > + } > + > jsonrpc_session_run(conn->js); > > unsigned int new_seqno = jsonrpc_session_get_seqno(conn->js); > @@ -1794,7 +1799,8 @@ static void > raft_open_conn(struct raft *raft, const char *address, const struct uuid *sid) > { > if (strcmp(address, raft->local_address) > - && !raft_find_conn_by_address(raft, address)) { > + && !raft_find_conn_by_address(raft, address) > + && failure_test != FT_STOP_RAFT_RPC) { > raft_add_conn(raft, jsonrpc_session_open(address, true), sid, false); > } > } > @@ -1870,7 +1876,7 @@ raft_run(struct raft *raft) > free(paddr); > } > > - if (raft->listener) { > + if (raft->listener && failure_test != FT_STOP_RAFT_RPC) { > struct stream *stream; > int error = pstream_accept(raft->listener, &stream); > if (!error) { > @@ -1995,7 +2001,7 @@ raft_run(struct raft *raft) > static void > raft_wait_session(struct jsonrpc_session *js) > { > - if (js) { > + if (js && failure_test != FT_STOP_RAFT_RPC) { > jsonrpc_session_wait(js); > jsonrpc_session_recv_wait(js); > } > @@ -2012,10 +2018,12 @@ raft_wait(struct raft *raft) > > raft_waiters_wait(raft); > > - if (raft->listener) { > - pstream_wait(raft->listener); > - } else { > - poll_timer_wait_until(raft->listen_backoff); > + if (failure_test != FT_STOP_RAFT_RPC) { > + if (raft->listener) { > + pstream_wait(raft->listener); > + } else { > + poll_timer_wait_until(raft->listen_backoff); > + } > } > > struct raft_conn *conn; > @@ -4361,8 +4369,9 @@ raft_send_to_conn_at(struct raft *raft, const union raft_rpc *rpc, > struct raft_conn *conn, int line_number) > { > log_rpc(rpc, "-->", conn, line_number); > - return !jsonrpc_session_send( > - conn->js, raft_rpc_to_jsonrpc(&raft->cid, &raft->sid, rpc)); > + return failure_test == FT_STOP_RAFT_RPC > + || !jsonrpc_session_send( > + conn->js, raft_rpc_to_jsonrpc(&raft->cid, &raft->sid, rpc)); > } > > static bool > @@ -4856,6 +4865,8 @@ raft_unixctl_failure_test(struct unixctl_conn *conn OVS_UNUSED, > } > } else if (!strcmp(test, "dont-send-vote-request")) { > failure_test = FT_DONT_SEND_VOTE_REQUEST; > + } else if (!strcmp(test, "stop-raft-rpc")) { > + failure_test = FT_STOP_RAFT_RPC; > } else if (!strcmp(test, "clear")) { > failure_test = FT_NO_TEST; > unixctl_command_reply(conn, "test dismissed"); > -- > 2.26.2 > Acked-by: Han Zhou <hzhou@ovn.org>
On 2/25/21 8:16 AM, Han Zhou wrote: > > > On Tue, Feb 23, 2021 at 5:18 AM Ilya Maximets <i.maximets@ovn.org <mailto:i.maximets@ovn.org>> wrote: >> >> This command will stop sending and receiving any RAFT-related >> traffic or accepting new connections. Useful to simulate >> network problems between cluster members. >> >> There is no unit test that uses it yet, but it's convenient for >> manual testing. >> >> Signed-off-by: Ilya Maximets <i.maximets@ovn.org <mailto:i.maximets@ovn.org>> >> --- >> ovsdb/raft.c | 31 +++++++++++++++++++++---------- >> 1 file changed, 21 insertions(+), 10 deletions(-) > > Acked-by: Han Zhou <hzhou@ovn.org <mailto:hzhou@ovn.org>> Thanks! Applied to master. Best regards, Ilya Maximets.
diff --git a/ovsdb/raft.c b/ovsdb/raft.c index 192f7f0a9..d2ff643b2 100644 --- a/ovsdb/raft.c +++ b/ovsdb/raft.c @@ -75,7 +75,8 @@ enum raft_failure_test { FT_CRASH_AFTER_SEND_EXEC_REQ, FT_CRASH_AFTER_RECV_APPEND_REQ_UPDATE, FT_DELAY_ELECTION, - FT_DONT_SEND_VOTE_REQUEST + FT_DONT_SEND_VOTE_REQUEST, + FT_STOP_RAFT_RPC, }; static enum raft_failure_test failure_test; @@ -1474,6 +1475,10 @@ raft_send_add_server_request(struct raft *raft, struct raft_conn *conn) static void raft_conn_run(struct raft *raft, struct raft_conn *conn) { + if (failure_test == FT_STOP_RAFT_RPC) { + return; + } + jsonrpc_session_run(conn->js); unsigned int new_seqno = jsonrpc_session_get_seqno(conn->js); @@ -1794,7 +1799,8 @@ static void raft_open_conn(struct raft *raft, const char *address, const struct uuid *sid) { if (strcmp(address, raft->local_address) - && !raft_find_conn_by_address(raft, address)) { + && !raft_find_conn_by_address(raft, address) + && failure_test != FT_STOP_RAFT_RPC) { raft_add_conn(raft, jsonrpc_session_open(address, true), sid, false); } } @@ -1870,7 +1876,7 @@ raft_run(struct raft *raft) free(paddr); } - if (raft->listener) { + if (raft->listener && failure_test != FT_STOP_RAFT_RPC) { struct stream *stream; int error = pstream_accept(raft->listener, &stream); if (!error) { @@ -1995,7 +2001,7 @@ raft_run(struct raft *raft) static void raft_wait_session(struct jsonrpc_session *js) { - if (js) { + if (js && failure_test != FT_STOP_RAFT_RPC) { jsonrpc_session_wait(js); jsonrpc_session_recv_wait(js); } @@ -2012,10 +2018,12 @@ raft_wait(struct raft *raft) raft_waiters_wait(raft); - if (raft->listener) { - pstream_wait(raft->listener); - } else { - poll_timer_wait_until(raft->listen_backoff); + if (failure_test != FT_STOP_RAFT_RPC) { + if (raft->listener) { + pstream_wait(raft->listener); + } else { + poll_timer_wait_until(raft->listen_backoff); + } } struct raft_conn *conn; @@ -4361,8 +4369,9 @@ raft_send_to_conn_at(struct raft *raft, const union raft_rpc *rpc, struct raft_conn *conn, int line_number) { log_rpc(rpc, "-->", conn, line_number); - return !jsonrpc_session_send( - conn->js, raft_rpc_to_jsonrpc(&raft->cid, &raft->sid, rpc)); + return failure_test == FT_STOP_RAFT_RPC + || !jsonrpc_session_send( + conn->js, raft_rpc_to_jsonrpc(&raft->cid, &raft->sid, rpc)); } static bool @@ -4856,6 +4865,8 @@ raft_unixctl_failure_test(struct unixctl_conn *conn OVS_UNUSED, } } else if (!strcmp(test, "dont-send-vote-request")) { failure_test = FT_DONT_SEND_VOTE_REQUEST; + } else if (!strcmp(test, "stop-raft-rpc")) { + failure_test = FT_STOP_RAFT_RPC; } else if (!strcmp(test, "clear")) { failure_test = FT_NO_TEST; unixctl_command_reply(conn, "test dismissed");
This command will stop sending and receiving any RAFT-related traffic or accepting new connections. Useful to simulate network problems between cluster members. There is no unit test that uses it yet, but it's convenient for manual testing. Signed-off-by: Ilya Maximets <i.maximets@ovn.org> --- ovsdb/raft.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-)