[ovs-dev,v2,4/9] ovsdb raft: Support commands that are required for testing failure scenarios.

Message ID 1555111588-79659-4-git-send-email-hzhou8@ebay.com
State New
Headers show
Series
  • [ovs-dev,v2,1/9] ovsdb.at: Move ovsdb macros from ovsdb.at to ovsdb-macros.at.
Related show

Commit Message

Han Zhou April 12, 2019, 11:26 p.m.
From: Han Zhou <hzhou8@ebay.com>

Added unix commands cluster/... for ovsdb raft, which will be used in a future
patch to test more fine-grained failure scenarios. The commands either causes
a node to crash at certain point, or manipulate the election timer so that
we can control the election process to elect a new leader we desired for the
test cases.

Signed-off-by: Han Zhou <hzhou8@ebay.com>
---
 ovsdb/raft.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

Patch

diff --git a/ovsdb/raft.c b/ovsdb/raft.c
index 31e9e72..abcc9c0 100644
--- a/ovsdb/raft.c
+++ b/ovsdb/raft.c
@@ -63,6 +63,20 @@  enum raft_role {
     RAFT_LEADER
 };
 
+/* Flags for unit tests. */
+enum raft_failure_test {
+    FT_NO_TEST,
+    FT_CRASH_BEFORE_SEND_APPEND_REQ,
+    FT_CRASH_AFTER_SEND_APPEND_REQ,
+    FT_CRASH_BEFORE_SEND_EXEC_REP,
+    FT_CRASH_AFTER_SEND_EXEC_REP,
+    FT_CRASH_BEFORE_SEND_EXEC_REQ,
+    FT_CRASH_AFTER_SEND_EXEC_REQ,
+    FT_CRASH_AFTER_RECV_APPEND_REQ_UPDATE,
+    FT_DELAY_ELECTION
+};
+static enum raft_failure_test failure_test;
+
 /* A connection between this Raft server and another one. */
 struct raft_conn {
     struct ovs_list list_node;  /* In struct raft's 'conns' list. */
@@ -871,6 +885,10 @@  raft_reset_election_timer(struct raft *raft)
     unsigned int duration = (ELECTION_BASE_MSEC
                              + random_range(ELECTION_RANGE_MSEC));
     raft->election_base = time_msec();
+    if (failure_test == FT_DELAY_ELECTION) {
+        /* Slow down this node so that it won't win the next election. */
+        duration += ELECTION_BASE_MSEC;
+    }
     raft->election_timeout = raft->election_base + duration;
 }
 
@@ -1962,6 +1980,9 @@  raft_command_initiate(struct raft *raft,
 
     raft_waiter_create(raft, RAFT_W_ENTRY, true)->entry.index = cmd->index;
 
+    if (failure_test == FT_CRASH_BEFORE_SEND_APPEND_REQ) {
+        ovs_fatal(0, "Raft test: crash before sending append_request.");
+    }
     /* Write to remote logs. */
     struct raft_server *s;
     HMAP_FOR_EACH (s, hmap_node, &raft->servers) {
@@ -1970,6 +1991,9 @@  raft_command_initiate(struct raft *raft,
             s->next_index++;
         }
     }
+    if (failure_test == FT_CRASH_AFTER_SEND_APPEND_REQ) {
+        ovs_fatal(0, "Raft test: crash after sending append_request.");
+    }
     raft_reset_ping_timer(raft);
 
     return cmd;
@@ -2014,10 +2038,18 @@  raft_command_execute__(struct raft *raft,
                 .result = eid,
             }
         };
+        if (failure_test == FT_CRASH_BEFORE_SEND_EXEC_REQ) {
+            ovs_fatal(0, "Raft test: crash before sending "
+                      "execute_command_request");
+        }
         if (!raft_send(raft, &rpc)) {
             /* Couldn't send command, so it definitely failed. */
             return raft_command_create_completed(RAFT_CMD_NOT_LEADER);
         }
+        if (failure_test == FT_CRASH_AFTER_SEND_EXEC_REQ) {
+            ovs_fatal(0, "Raft test: crash after sending "
+                      "execute_command_request");
+        }
 
         struct raft_command *cmd = raft_command_create_incomplete(raft, 0);
         cmd->timestamp = time_msec();
@@ -2731,6 +2763,10 @@  raft_handle_append_entries(struct raft *raft,
         }
     }
 
+    if (failure_test == FT_CRASH_AFTER_RECV_APPEND_REQ_UPDATE) {
+        ovs_fatal(0, "Raft test: crash after receiving append_request with "
+                  "update.");
+    }
     /* Figure 3.1: "Append any entries not already in the log." */
     struct ovsdb_error *error = NULL;
     bool any_written = false;
@@ -3939,6 +3975,9 @@  raft_send_execute_command_reply(struct raft *raft,
                                 enum raft_command_status status,
                                 uint64_t commit_index)
 {
+    if (failure_test == FT_CRASH_BEFORE_SEND_EXEC_REP) {
+        ovs_fatal(0, "Raft test: crash before sending execute_command_reply");
+    }
     union raft_rpc rpc = {
         .execute_command_reply = {
             .common = {
@@ -3951,6 +3990,9 @@  raft_send_execute_command_reply(struct raft *raft,
         },
     };
     raft_send(raft, &rpc);
+    if (failure_test == FT_CRASH_AFTER_SEND_EXEC_REP) {
+        ovs_fatal(0, "Raft test: crash after sending execute_command_reply.");
+    }
 }
 
 static enum raft_command_status
@@ -4387,6 +4429,45 @@  raft_unixctl_kick(struct unixctl_conn *conn, int argc OVS_UNUSED,
 }
 
 static void
+raft_unixctl_failure_test(struct unixctl_conn *conn OVS_UNUSED,
+                          int argc OVS_UNUSED, const char *argv[],
+                          void *aux OVS_UNUSED)
+{
+    const char *test = argv[1];
+    if (!strcmp(test, "crash-before-sending-append-request")) {
+        failure_test = FT_CRASH_BEFORE_SEND_APPEND_REQ;
+    } else if (!strcmp(test, "crash-after-sending-append-request")) {
+        failure_test = FT_CRASH_AFTER_SEND_APPEND_REQ;
+    } else if (!strcmp(test, "crash-before-sending-execute-command-reply")) {
+        failure_test = FT_CRASH_BEFORE_SEND_EXEC_REP;
+    } else if (!strcmp(test, "crash-after-sending-execute-command-reply")) {
+        failure_test = FT_CRASH_AFTER_SEND_EXEC_REP;
+    } else if (!strcmp(test, "crash-before-sending-execute-command-request")) {
+        failure_test = FT_CRASH_BEFORE_SEND_EXEC_REQ;
+    } else if (!strcmp(test, "crash-after-sending-execute-command-request")) {
+        failure_test = FT_CRASH_AFTER_SEND_EXEC_REQ;
+    } else if (!strcmp(test, "crash-after-receiving-append-request-update")) {
+        failure_test = FT_CRASH_AFTER_RECV_APPEND_REQ_UPDATE;
+    } else if (!strcmp(test, "delay-election")) {
+        failure_test = FT_DELAY_ELECTION;
+        struct raft *raft;
+        HMAP_FOR_EACH (raft, hmap_node, &all_rafts) {
+            if (raft->role == RAFT_FOLLOWER) {
+                raft_reset_election_timer(raft);
+            }
+        }
+    } else if (!strcmp(test, "clear")) {
+        failure_test = FT_NO_TEST;
+        unixctl_command_reply(conn, "test dismissed");
+        return;
+    } else {
+        unixctl_command_reply_error(conn, "unknown test scenario");
+        return;
+    }
+    unixctl_command_reply(conn, "test engaged");
+}
+
+static void
 raft_init(void)
 {
     static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
@@ -4403,5 +4484,7 @@  raft_init(void)
                              raft_unixctl_leave, NULL);
     unixctl_command_register("cluster/kick", "DB SERVER", 2, 2,
                              raft_unixctl_kick, NULL);
+    unixctl_command_register("cluster/failure-test", "FAILURE SCENARIO", 1, 1,
+                             raft_unixctl_failure_test, NULL);
     ovsthread_once_done(&once);
 }