[ovs-dev,v4,5/5] ovsdb-idl.c: Fast resync from server when connection reset.

Message ID 1551374120-44287-6-git-send-email-hzhou8@ebay.com
State New
Headers show
Series
  • Fast OVSDB resync after restart or fail-over.
Related show

Commit Message

Han Zhou Feb. 28, 2019, 5:15 p.m.
From: Han Zhou <hzhou8@ebay.com>

Use monitor_cond_since to request changes after last version of local
data when connection to server is reset, without clearing the local
data. It falls back to clearing and repopulating all the data when
the requested id cannot be fulfilled by the server.

Test result at ovn-scale-test environment using clustered mode:
- 1K HVs (ovsdb clients)
- 10K lports

Without the patch it took 30+ min for the SB ovsdb-server to calm down
and HVs to stablize the connectin and finish syncing data.

With the patch there were no noticible CPU spike of SB ovsdb-server,
and all HVs were in sync with SB within 1 min, which is the probe
interval set in this test (so it took at most 1 min for HVs to notice
the TCP connection reset and reconnect and resync finished immediately
after that).

Reported-at:
https://mail.openvswitch.org/pipermail/ovs-discuss/2018-September/047457.html
Signed-off-by: Han Zhou <hzhou8@ebay.com>
---
 lib/ovsdb-idl.c | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

Patch

diff --git a/lib/ovsdb-idl.c b/lib/ovsdb-idl.c
index 5d86046..8cfb201 100644
--- a/lib/ovsdb-idl.c
+++ b/lib/ovsdb-idl.c
@@ -216,6 +216,9 @@  struct ovsdb_idl_db {
     bool has_lock;              /* Has db server told us we have the lock? */
     bool is_lock_contended;     /* Has db server told us we can't get lock? */
     struct json *lock_request_id; /* JSON-RPC ID of in-flight lock request. */
+
+    /* Last db txn id, used for fast resync through monitor_cond_since */
+    struct uuid last_id;
 };
 
 static void ovsdb_idl_db_track_clear(struct ovsdb_idl_db *);
@@ -2070,9 +2073,8 @@  ovsdb_idl_send_monitor_request(struct ovsdb_idl *idl, struct ovsdb_idl_db *db,
             break;
         case OVSDB_IDL_MM_MONITOR_COND_SINCE:
             method = "monitor_cond_since";
-            struct uuid last_id = UUID_ZERO;
             struct json *json_last_id = json_string_create_nocopy(
-                    xasprintf(UUID_FMT, UUID_ARGS(&last_id)));
+                    xasprintf(UUID_FMT, UUID_ARGS(&db->last_id)));
             json_array_add(params, json_last_id);
             break;
         default:
@@ -2100,6 +2102,7 @@  ovsdb_idl_db_parse_monitor_reply(struct ovsdb_idl_db *db,
 {
     db->change_seqno++;
     const struct json *table_updates = result;
+    bool clear_db = true;
     if (method == OVSDB_IDL_MM_MONITOR_COND_SINCE) {
         if (result->type != JSON_ARRAY || result->array.n != 3) {
             struct ovsdb_error *error = ovsdb_syntax_error(result, NULL,
@@ -2109,10 +2112,25 @@  ovsdb_idl_db_parse_monitor_reply(struct ovsdb_idl_db *db,
             return;
         }
 
-        table_updates = result->array.elems[2];
+        bool found = json_boolean(result->array.elems[0]);
+        if (found) {
+            clear_db = false;
+        }
 
+        const char *last_id = json_string(result->array.elems[1]);
+        if (!uuid_from_string(&db->last_id, last_id)) {
+            struct ovsdb_error *error = ovsdb_syntax_error(result, NULL,
+                                     "Last-id %s is not in UUID format.",
+                                     last_id);
+            log_parse_update_error(error);
+            return;
+        }
+
+        table_updates = result->array.elems[2];
+    }
+    if (clear_db) {
+        ovsdb_idl_db_clear(db);
     }
-    ovsdb_idl_db_clear(db);
     ovsdb_idl_db_parse_update(db, table_updates, method);
 }
 
@@ -2155,6 +2173,14 @@  ovsdb_idl_db_parse_update_rpc(struct ovsdb_idl_db *db,
     struct json *table_updates = params->array.elems[1];
     if (!strcmp(msg->method, "update3")) {
         table_updates = params->array.elems[2];
+        const char *last_id = json_string(params->array.elems[1]);
+        if (!uuid_from_string(&db->last_id, last_id)) {
+            struct ovsdb_error *error = ovsdb_syntax_error(params, NULL,
+                                     "Last-id %s is not in UUID format.",
+                                     last_id);
+            log_parse_update_error(error);
+            return false;
+        }
     }
     ovsdb_idl_db_parse_update(db, table_updates, mm);
     return true;