@@ -110,6 +110,8 @@ lib_libopenvswitch_la_SOURCES = \
lib/json.c \
lib/jsonrpc.c \
lib/jsonrpc.h \
+ lib/keepalive.c \
+ lib/keepalive.h \
lib/lacp.c \
lib/lacp.h \
lib/latch.h \
@@ -32,6 +32,7 @@
#include "dirs.h"
#include "fatal-signal.h"
+#include "keepalive.h"
#include "netdev-dpdk.h"
#include "openvswitch/dynamic-string.h"
#include "openvswitch/vlog.h"
@@ -477,6 +478,28 @@ dpdk_init(const struct smap *ovs_other_config)
}
}
+int
+dpdk_ka_init(void)
+{
+ struct keepalive_shm *ka_shm = get_ka_shm();
+ if (!ka_shm) {
+ VLOG_ERR("SHM uninitialized? keepalive initialization aborted.");
+ return -1;
+ }
+
+ /* Initialize keepalive subsystem */
+ if ((rte_global_keepalive_info =
+ rte_keepalive_create(&dpdk_failcore_cb, ka_shm)) == NULL) {
+ VLOG_ERR("Keepalive initialization failed.");
+ return -1;
+ } else {
+ rte_keepalive_register_relay_callback(rte_global_keepalive_info,
+ dpdk_ka_update_core_state, ka_shm);
+ }
+
+ return 0;
+}
+
const char *
dpdk_get_vhost_sock_dir(void)
{
@@ -37,6 +37,7 @@ struct smap;
struct rte_keepalive *rte_global_keepalive_info;
void dpdk_init(const struct smap *ovs_other_config);
+int dpdk_ka_init(void);
void dpdk_set_lcore_id(unsigned cpu);
const char *dpdk_get_vhost_sock_dir(void);
new file mode 100644
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2014, 2015, 2016, 2017 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "dpdk.h"
+#include "keepalive.h"
+#include "lib/vswitch-idl.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(keepalive);
+
+static bool keepalive_enable = false; /* KeepAlive disabled by default */
+static bool ka_init_status = ka_init_failure; /* KeepAlive initialization */
+static uint32_t keepalive_timer_interval; /* keepalive timer interval */
+
+static const char *keepalive_shm_blk = NULL;
+struct keepalive_shm *ka_shm = NULL;
+
+/* Return the Keepalive shared memory block name. */
+static inline const char *
+get_ka_shm_blk(void)
+{
+ return keepalive_shm_blk;
+}
+
+inline struct keepalive_shm *
+get_ka_shm(void)
+{
+ return ka_shm;
+}
+
+void
+ka_set_pmd_state_ts(unsigned core_id, enum keepalive_state state,
+ uint64_t last_alive)
+{
+ ka_shm->core_state[core_id] = state;
+ ka_shm->core_last_seen_times[core_id] = last_alive;
+}
+
+/* Retrieve and return the keepalive timer interval from OVSDB. */
+static uint32_t
+get_ka_timer_interval(const struct smap *ovs_other_config OVS_UNUSED)
+{
+#define OVS_KEEPALIVE_TIMEOUT 1000 /* Default timeout set to 1000ms */
+ uint32_t ka_interval;
+
+ /* Timer granularity in milliseconds
+ * Defaults to OVS_KEEPALIVE_TIMEOUT(ms) if not set */
+ ka_interval = smap_get_int(ovs_other_config, "keepalive-interval",
+ OVS_KEEPALIVE_TIMEOUT);
+
+ VLOG_INFO("Keepalive timer interval set to %"PRIu32" (ms)\n", ka_interval);
+ return ka_interval;
+}
+
+static const char *
+get_ka_shm_block(const struct smap *ovs_other_config OVS_UNUSED)
+{
+/* Shared mem block. */
+#define OVS_KEEPALIVE_SHM_NAME /dpdk_keepalive_shm_name
+ keepalive_shm_blk = smap_get(ovs_other_config, "keepalive-shm-name");
+ if (!keepalive_shm_blk) {
+ keepalive_shm_blk = OVS_STRINGIZE(OVS_KEEPALIVE_SHM_NAME);
+ }
+
+ VLOG_INFO("KeepAlive shared memory block: %s\n", keepalive_shm_blk);
+ return keepalive_shm_blk;
+}
+
+/* Create POSIX Shared memory object and initialize the core states. */
+static
+struct keepalive_shm *keepalive_shm_create(void)
+{
+ int fd;
+ int coreid;
+ struct keepalive_shm *ka_shm;
+ char ka_shmblk[40];
+
+ sprintf(ka_shmblk, "%s", get_ka_shm_blk());
+ if (shm_unlink(ka_shmblk) == -1 && errno != ENOENT) {
+ VLOG_ERR("Error unlinking stale %s \n", ka_shmblk);
+ }
+
+ if ((fd = shm_open(ka_shmblk,
+ O_CREAT | O_TRUNC | O_RDWR, 0666)) < 0) {
+ VLOG_WARN("Failed to open %s as SHM \n", ka_shmblk);
+ } else if (ftruncate(fd, sizeof(struct keepalive_shm)) != 0) {
+ VLOG_WARN("Failed to resize SHM \n");
+ } else {
+ ka_shm = (struct keepalive_shm *) mmap(
+ 0, sizeof(struct keepalive_shm),
+ PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ close(fd);
+ if (ka_shm == MAP_FAILED) {
+ VLOG_WARN("Failed to mmap SHM \n");
+ } else {
+ memset(ka_shm, 0, sizeof(struct keepalive_shm));
+
+ /* Mark all cores to 'not present' */
+ for (coreid = 0; coreid < KEEPALIVE_MAXCORES; coreid++) {
+ ka_shm->core_state[coreid] = KA_STATE_UNUSED;
+ ka_shm->core_last_seen_times[coreid] = 0;
+ }
+
+ return ka_shm;
+ }
+ }
+ return NULL;
+}
+
+static int
+ka_init__(void)
+{
+#ifdef DPDK_NETDEV
+ return dpdk_ka_init();
+#else
+ return -1;
+#endif
+}
+
+void
+ka_init(const struct smap *ovs_other_config)
+{
+ if (ka_init_status || !ovs_other_config) {
+ return;
+ }
+
+ static struct ovsthread_once once_enable = OVSTHREAD_ONCE_INITIALIZER;
+ if (ovsthread_once_start(&once_enable)) {
+ if (smap_get_bool(ovs_other_config, "enable-keepalive", false)) {
+ keepalive_enable = true;
+ VLOG_INFO("OvS Keepalive enabled.");
+
+ keepalive_timer_interval =
+ get_ka_timer_interval(ovs_other_config);
+ keepalive_shm_blk = get_ka_shm_block(ovs_other_config);
+
+ /* Create shared memory block */
+ if ((ka_shm = keepalive_shm_create()) != NULL) {
+ int err = ka_init__();
+ if (!err) {
+ VLOG_INFO("OvS Keepalive - initialized.");
+ ka_init_status = ka_init_success;
+ }
+ } else {
+ VLOG_ERR("keepalive_shm_create() failed.");
+ }
+ } else {
+ VLOG_INFO("OvS Keepalive disabled.");
+ }
+
+ ovsthread_once_done(&once_enable);
+ }
+}
new file mode 100644
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2016 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef KEEPALIVE_H
+#define KEEPALIVE_H
+
+#include <stdint.h>
+#ifdef DPDK_NETDEV
+#include <rte_keepalive.h>
+#define KEEPALIVE_MAXCORES RTE_KEEPALIVE_MAXCORES
+#else
+#define KEEPALIVE_MAXCORES 128
+#endif /* DPDK_NETDEV */
+
+struct smap;
+
+enum keepalive_state {
+ KA_STATE_UNUSED = 0,
+ KA_STATE_ALIVE = 1,
+ KA_STATE_MISSING = 4,
+ KA_STATE_DEAD = 2,
+ KA_STATE_GONE = 3,
+ KA_STATE_DOZING = 5,
+ KA_STATE_SLEEP = 6,
+ KA_STATE_CHECK = 7
+};
+
+struct keepalive_shm {
+ enum keepalive_state core_state[KEEPALIVE_MAXCORES];
+
+ /* Last seen timestamp of the core */
+ uint64_t core_last_seen_times[KEEPALIVE_MAXCORES];
+
+ /* Store pmd thread tid */
+ pid_t thread_id[KEEPALIVE_MAXCORES];
+};
+
+enum keepalive_status {
+ ka_init_failure = 0,
+ ka_init_success
+};
+
+void ka_init(const struct smap *);
+struct keepalive_shm *get_ka_shm(void);
+void ka_set_pmd_state_ts(unsigned, enum keepalive_state, uint64_t);
+
+#endif /* keepalive.h */
@@ -32,12 +32,14 @@
#include <rte_mbuf.h>
#include <rte_meter.h>
#include <rte_virtio_net.h>
+#include <rte_keepalive.h>
#include "dirs.h"
#include "dp-packet.h"
#include "dpdk.h"
#include "dpif-netdev.h"
#include "fatal-signal.h"
+#include "keepalive.h"
#include "netdev-provider.h"
#include "netdev-vport.h"
#include "odp-util.h"
@@ -48,8 +50,9 @@
#include "ovs-numa.h"
#include "ovs-thread.h"
#include "ovs-rcu.h"
-#include "packets.h"
#include "openvswitch/shash.h"
+#include "packets.h"
+#include "process.h"
#include "smap.h"
#include "sset.h"
#include "unaligned.h"
@@ -567,6 +570,79 @@ dpdk_mp_put(struct dpdk_mp *dmp)
ovs_mutex_unlock(&dpdk_mp_mutex);
}
+/* Callback function invoked on heartbeat miss. Verify if it is genuine
+ * heartbeat miss or a false positive and log the message accordingly.
+ */
+void
+dpdk_failcore_cb(void *ptr_data, const int core_id)
+{
+ struct keepalive_shm *ka_shm = (struct keepalive_shm *)ptr_data;
+
+ if (ka_shm) {
+ int pstate;
+ uint32_t tid = ka_shm->thread_id[core_id];
+ int err = get_process_status(tid, &pstate);
+
+ if (!err) {
+ switch (pstate) {
+
+ case ACTIVE_STATE:
+ VLOG_INFO_RL(&rl,"False positive, pmd tid[%"PRIu32"] alive\n",
+ tid);
+ break;
+ case STOPPED_STATE:
+ case TRACED_STATE:
+ case DEFUNC_STATE:
+ case UNINTERRUPTIBLE_SLEEP_STATE:
+ VLOG_WARN_RL(&rl,
+ "PMD tid[%"PRIu32"] on core[%d] is unresponsive\n",
+ tid, core_id);
+ break;
+ default:
+ VLOG_DBG("%s: The process state: %d\n", __FUNCTION__, pstate);
+ OVS_NOT_REACHED();
+ }
+ }
+ }
+}
+
+/* Update the core state in shared memory.
+ *
+ * This function shall be invoked periodically to write the core status and
+ * last seen timestamp of the cores in to shared memory block.
+ */
+void
+dpdk_ka_update_core_state(void *ptr_data, const int core_id,
+ const enum rte_keepalive_state core_state, uint64_t last_alive)
+{
+ struct keepalive_shm *ka_shm = (struct keepalive_shm *)ptr_data;
+ if (!ka_shm) {
+ VLOG_ERR_RL(&rl, "KeepAlive: Invalid shared memory block\n");
+ return;
+ }
+
+ VLOG_DBG_RL(&rl,
+ "TS(%lu):CORE%d, old state:%d, current_state:%d\n",
+ (unsigned long)time(NULL),core_id,ka_shm->core_state[core_id],
+ core_state);
+
+ switch (core_state) {
+ case RTE_KA_STATE_ALIVE:
+ case RTE_KA_STATE_MISSING:
+ ka_set_pmd_state_ts(core_id, KA_STATE_ALIVE, last_alive);
+ break;
+ case RTE_KA_STATE_DOZING:
+ case RTE_KA_STATE_SLEEP:
+ case RTE_KA_STATE_DEAD:
+ case RTE_KA_STATE_GONE:
+ ka_set_pmd_state_ts(core_id, core_state, last_alive);
+ break;
+ case RTE_KA_STATE_UNUSED:
+ ka_set_pmd_state_ts(core_id, KA_STATE_UNUSED, 0);
+ break;
+ }
+}
+
/* Tries to allocate new mempool on requested_socket_id with
* mbuf size corresponding to requested_mtu.
* On success new configuration will be applied.
@@ -18,15 +18,20 @@
#define NETDEV_DPDK_H
#include <config.h>
+#include <stdint.h>
#include "openvswitch/compiler.h"
struct dp_packet;
+enum rte_keepalive_state;
#ifdef DPDK_NETDEV
void netdev_dpdk_register(void);
void free_dpdk_buf(struct dp_packet *);
+void dpdk_failcore_cb(void *, const int);
+void dpdk_ka_update_core_state(void *ptr, const int,
+ const enum rte_keepalive_state, uint64_t);
#else
@@ -1,6 +1,6 @@
{"name": "Open_vSwitch",
- "version": "7.15.0",
- "cksum": "544856471 23228",
+ "version": "7.16.0",
+ "cksum": "2916438977 23364",
"tables": {
"Open_vSwitch": {
"columns": {
@@ -28,6 +28,9 @@
"statistics": {
"type": {"key": "string", "value": "string", "min": 0, "max": "unlimited"},
"ephemeral": true},
+ "keepalive": {
+ "type": {"key": "string", "value": "string", "min": 0, "max": "unlimited"},
+ "ephemeral": true},
"ovs_version": {
"type": {"key": {"type": "string"},
"min": 0, "max": 1}},
@@ -531,6 +531,65 @@
</p>
</column>
</group>
+
+ <group title="Keepalive">
+ <p>
+ The <code>keepalive</code> column contains key-value pairs that
+ report health of datapath cores in Open vSwitch. These are updated
+ periodically (based on the keepalive-interval).
+ </p>
+
+ <column name="other_config" key="enable-keepalive"
+ type='{"type": "boolean"}'>
+ Keepalive is disabled by default to avoid overhead in the common
+ case when heartbeat monitoring is not useful. Set this value to
+ <code>true</code> to enable keepalive <ref column="keepalive"/>
+ column or to <code>false</code> to explicitly disable it.
+ </column>
+
+ <column name="other_config" key="keepalive-interval"
+ type='{"type": "integer", "minInteger": 1}'>
+ <p>
+ Specifies the keepalive interval value.
+ </p>
+ <p>
+ If not specified, this will be set to 1000 milliseconds (default
+ value). Changing this value requires restarting the daemon.
+ </p>
+ </column>
+
+ <column name="other_config" key="keepalive-shm-name"
+ type='{"type": "string"}'>
+ <p>
+ Specifies the keepalive shared memory block name.
+ </p>
+ <p>
+ If not specified, shared memory block named "keepalive_shm_name"
+ (default name) is created. Changing this value requires restarting
+ the daemon.
+ </p>
+ </column>
+
+ <column name="keepalive" key="CORE_ID">
+ <p>
+ One such key-value pair, with <code>ID</code> replaced by the
+ core id, will exist for each active PMD thread. The value is a
+ comma-separated list of status of PMD core and last seen timestamp
+ of PMD thread. In respective order, these values are:
+ </p>
+
+ <ol>
+ <li>Status of PMD core. Valid values include ALIVE, MISSING, DEAD,
+ GONE, DOZING, SLEEPING.</li>
+ <li>Last seen timestamp of the PMD core.</li>
+ </ol>
+
+ <p>
+ This is only valid for OvS-DPDK Datapath and only PMD threads status
+ is implemented.
+ </p>
+ </column>
+ </group>
</group>
<group title="Version Reporting">
This commit introduces the initial keepalive support by adding 'keepalive' module and also helper and initialization functions that will be invoked by later commits. This commit adds new ovsdb column "keepalive". It shows the overall datapath status and the health of the cores running datapath threads. For eg: To enable keepalive feature. 'ovs-vsctl --no-wait set Open_vSwitch . other_config:enable-keepalive=true' To set timer interval of 5000ms for monitoring packet processing cores; 'ovs-vsctl --no-wait set Open_vSwitch . \ other_config:keepalive-interval="5000" To set shared memory block name where the events shall be updated 'ovs-vsctl --no-wait set Open_vSwitch . other_config:keepalive-shm-name="/ovs_keepalive_shm_name"' Signed-off-by: Bhanuprakash Bodireddy <bhanuprakash.bodireddy@intel.com> --- lib/automake.mk | 2 + lib/dpdk.c | 23 ++++++ lib/dpdk.h | 1 + lib/keepalive.c | 173 +++++++++++++++++++++++++++++++++++++++++++++ lib/keepalive.h | 60 ++++++++++++++++ lib/netdev-dpdk.c | 78 +++++++++++++++++++- lib/netdev-dpdk.h | 5 ++ vswitchd/vswitch.ovsschema | 7 +- vswitchd/vswitch.xml | 59 ++++++++++++++++ 9 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 lib/keepalive.c create mode 100644 lib/keepalive.h