@@ -110,6 +110,8 @@ lib_libopenvswitch_la_SOURCES = \
lib/json.c \
lib/jsonrpc.c \
lib/jsonrpc.h \
+ lib/keepalive.c \
+ lib/keepalive.h \
lib/lacp.c \
lib/lacp.h \
lib/latch.h \
@@ -32,6 +32,7 @@
#include "dirs.h"
#include "fatal-signal.h"
+#include "keepalive.h"
#include "netdev-dpdk.h"
#include "openvswitch/dynamic-string.h"
#include "openvswitch/vlog.h"
@@ -477,6 +478,28 @@ dpdk_init(const struct smap *ovs_other_config)
}
}
+int
+dpdk_ka_init(void)
+{
+ struct keepalive_shm *ka_shm = get_ka_shm();
+ if (!ka_shm) {
+ VLOG_ERR("SHM uninitialized? keepalive initialization aborted.");
+ return -1;
+ }
+
+ /* Initialize keepalive subsystem */
+ if ((rte_global_keepalive_info =
+ rte_keepalive_create(&dpdk_failcore_cb, ka_shm)) == NULL) {
+ VLOG_ERR("Keepalive initialization failed.");
+ return -1;
+ } else {
+ rte_keepalive_register_relay_callback(rte_global_keepalive_info,
+ dpdk_ka_update_core_state, ka_shm);
+ }
+
+ return 0;
+}
+
const char *
dpdk_get_vhost_sock_dir(void)
{
@@ -37,6 +37,7 @@ struct smap;
struct rte_keepalive *rte_global_keepalive_info;
void dpdk_init(const struct smap *ovs_other_config);
+int dpdk_ka_init(void);
void dpdk_set_lcore_id(unsigned cpu);
const char *dpdk_get_vhost_sock_dir(void);
new file mode 100644
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2014, 2015, 2016, 2017 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "dpdk.h"
+#include "keepalive.h"
+#include "lib/vswitch-idl.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(keepalive);
+
+static bool keepalive_enable = false; /* KeepAlive disabled by default */
+static bool ka_init_status = ka_init_failure; /* KeepAlive initialization */
+static uint32_t keepalive_timer_interval; /* keepalive timer interval */
+
+static const char *keepalive_shm_blk = NULL;
+struct keepalive_shm *ka_shm = NULL;
+
+/* Return the Keepalive shared memory block name. */
+static inline const char *
+get_ka_shm_blk(void)
+{
+ return keepalive_shm_blk;
+}
+
+inline struct keepalive_shm *
+get_ka_shm(void)
+{
+ return ka_shm;
+}
+
+/* Retrieve and return the keepalive timer interval from OVSDB. */
+static uint32_t
+get_ka_timer_interval(const struct smap *ovs_other_config OVS_UNUSED)
+{
+#define OVS_KEEPALIVE_TIMEOUT 100 /* Default timeout set to 100ms */
+ uint32_t ka_interval;
+
+ /* Timer granularity in milliseconds
+ * Defaults to OVS_KEEPALIVE_TIMEOUT(ms) if not set */
+ ka_interval = smap_get_int(ovs_other_config, "keepalive-interval",
+ OVS_KEEPALIVE_TIMEOUT);
+
+ VLOG_INFO("Keepalive timer interval set to %"PRIu32" (ms)\n", ka_interval);
+ return ka_interval;
+}
+
+static const char *
+get_ka_shm_block(const struct smap *ovs_other_config OVS_UNUSED)
+{
+/* Shared mem block. */
+#define OVS_KEEPALIVE_SHM_NAME /dpdk_keepalive_shm_name
+ keepalive_shm_blk = smap_get(ovs_other_config, "keepalive-shm-name");
+ if (!keepalive_shm_blk) {
+ keepalive_shm_blk = OVS_STRINGIZE(OVS_KEEPALIVE_SHM_NAME);
+ }
+
+ VLOG_INFO("KeepAlive shared memory block: %s\n", keepalive_shm_blk);
+ return keepalive_shm_blk;
+}
+
+/* Create POSIX Shared memory object and initialize the core states. */
+static
+struct keepalive_shm *keepalive_shm_create(void)
+{
+ int fd;
+ int coreid;
+ struct keepalive_shm *ka_shm;
+ char ka_shmblk[40];
+
+ sprintf(ka_shmblk, "%s", get_ka_shm_blk());
+ if (shm_unlink(ka_shmblk) == -1 && errno != ENOENT) {
+ VLOG_ERR("Error unlinking stale %s \n", ka_shmblk);
+ }
+
+ if ((fd = shm_open(ka_shmblk,
+ O_CREAT | O_TRUNC | O_RDWR, 0666)) < 0) {
+ VLOG_WARN("Failed to open %s as SHM \n", ka_shmblk);
+ } else if (ftruncate(fd, sizeof(struct keepalive_shm)) != 0) {
+ VLOG_WARN("Failed to resize SHM \n");
+ } else {
+ ka_shm = (struct keepalive_shm *) mmap(
+ 0, sizeof(struct keepalive_shm),
+ PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ close(fd);
+ if (ka_shm == MAP_FAILED) {
+ VLOG_WARN("Failed to mmap SHM \n");
+ } else {
+ memset(ka_shm, 0, sizeof(struct keepalive_shm));
+
+ /* Mark all cores to 'not present' */
+ for (coreid = 0; coreid < KEEPALIVE_MAXCORES; coreid++) {
+ ka_shm->core_state[coreid] = KA_STATE_UNUSED;
+ ka_shm->core_last_seen_times[coreid] = 0;
+ }
+
+ return ka_shm;
+ }
+ }
+ return NULL;
+}
+
+static int
+ka_init__(void)
+{
+#ifdef DPDK_NETDEV
+ return dpdk_ka_init();
+#else
+ return -1;
+#endif
+}
+
+void
+ka_init(const struct smap *ovs_other_config)
+{
+ if (ka_init_status || !ovs_other_config) {
+ return;
+ }
+
+ if (smap_get_bool(ovs_other_config, "enable-keepalive", false)) {
+ static struct ovsthread_once once_enable = OVSTHREAD_ONCE_INITIALIZER;
+
+ if (ovsthread_once_start(&once_enable)) {
+ keepalive_enable = true;
+ VLOG_INFO("OvS Keepalive enabled.");
+
+ keepalive_timer_interval =
+ get_ka_timer_interval(ovs_other_config);
+ keepalive_shm_blk = get_ka_shm_block(ovs_other_config);
+
+ /* Create shared memory block */
+ if ((ka_shm = keepalive_shm_create()) != NULL) {
+ int err = ka_init__();
+ if (!err) {
+ VLOG_INFO("OvS Keepalive - initialized.");
+ ka_init_status = ka_init_success;
+ }
+ } else {
+ VLOG_ERR("keepalive_shm_create() failed.");
+ }
+ ovsthread_once_done(&once_enable);
+ }
+ }
+}
new file mode 100644
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2016 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef KEEPALIVE_H
+#define KEEPALIVE_H
+
+#include <stdint.h>
+#ifdef DPDK_NETDEV
+#include <rte_keepalive.h>
+#define KEEPALIVE_MAXCORES RTE_KEEPALIVE_MAXCORES
+#else
+#define KEEPALIVE_MAXCORES 128
+#endif /* DPDK_NETDEV */
+
+struct smap;
+
+enum keepalive_state {
+ KA_STATE_UNUSED = 0,
+ KA_STATE_ALIVE = 1,
+ KA_STATE_MISSING = 4,
+ KA_STATE_DEAD = 2,
+ KA_STATE_GONE = 3,
+ KA_STATE_DOZING = 5,
+ KA_STATE_SLEEP = 6,
+ KA_STATE_CHECK = 7
+};
+
+struct keepalive_shm {
+ enum keepalive_state core_state[KEEPALIVE_MAXCORES];
+
+ /* Last seen timestamp of the core */
+ uint64_t core_last_seen_times[KEEPALIVE_MAXCORES];
+
+ /* Store pmd thread tid */
+ pid_t thread_id[KEEPALIVE_MAXCORES];
+};
+
+enum keepalive_status {
+ ka_init_failure = 0,
+ ka_init_success
+};
+
+void ka_init(const struct smap *);
+struct keepalive_shm *get_ka_shm(void);
+
+#endif /* keepalive.h */
@@ -32,12 +32,14 @@
#include <rte_mbuf.h>
#include <rte_meter.h>
#include <rte_virtio_net.h>
+#include <rte_keepalive.h>
#include "dirs.h"
#include "dp-packet.h"
#include "dpdk.h"
#include "dpif-netdev.h"
#include "fatal-signal.h"
+#include "keepalive.h"
#include "netdev-provider.h"
#include "netdev-vport.h"
#include "odp-util.h"
@@ -48,8 +50,9 @@
#include "ovs-numa.h"
#include "ovs-thread.h"
#include "ovs-rcu.h"
-#include "packets.h"
#include "openvswitch/shash.h"
+#include "packets.h"
+#include "process.h"
#include "smap.h"
#include "sset.h"
#include "unaligned.h"
@@ -567,6 +570,81 @@ dpdk_mp_put(struct dpdk_mp *dmp)
ovs_mutex_unlock(&dpdk_mp_mutex);
}
+/* Callback function invoked on heartbeat miss. Verify if it is genuine
+ * heartbeat miss or a false positive and log the message accordingly.
+ */
+void
+dpdk_failcore_cb(void *ptr_data, const int core_id)
+{
+ struct keepalive_shm *ka_shm = (struct keepalive_shm *)ptr_data;
+
+ if (ka_shm) {
+ int pstate;
+ uint32_t tid = ka_shm->thread_id[core_id];
+ int err = get_process_status(tid, &pstate);
+
+ if (!err) {
+ switch (pstate) {
+
+ case ACTIVE_STATE:
+ VLOG_INFO_RL(&rl,"False positive, pmd tid[%"PRIu32"] alive\n",
+ tid);
+ break;
+ case STOPPED_STATE:
+ case TRACED_STATE:
+ case DEFUNC_STATE:
+ case UNINTERRUPTIBLE_SLEEP_STATE:
+ VLOG_WARN_RL(&rl,
+ "PMD tid[%"PRIu32"] on core[%d] is unresponsive\n",
+ tid, core_id);
+ break;
+ default:
+ VLOG_DBG("%s: The process state: %d\n", __FUNCTION__, pstate);
+ OVS_NOT_REACHED();
+ }
+ }
+ }
+}
+
+/* Update the core state in shared memory.
+ *
+ * This function shall be invoked periodically to write the core status and
+ * last seen timestamp of the cores in to shared memory block.
+ */
+void
+dpdk_ka_update_core_state(void *ptr_data, const int core_id,
+ const enum rte_keepalive_state core_state, uint64_t last_alive)
+{
+ struct keepalive_shm *ka_shm = (struct keepalive_shm *)ptr_data;
+ if (!ka_shm) {
+ VLOG_ERR_RL(&rl, "KeepAlive: Invalid shared memory block\n");
+ return;
+ }
+
+ VLOG_DBG_RL(&rl,
+ "TS(%lu):CORE%d, old state:%d, current_state:%d\n",
+ (unsigned long)time(NULL),core_id,ka_shm->core_state[core_id],
+ core_state);
+
+ switch (core_state) {
+ case RTE_KA_STATE_ALIVE:
+ case RTE_KA_STATE_MISSING:
+ ka_shm->core_state[core_id] = KA_STATE_ALIVE;
+ ka_shm->core_last_seen_times[core_id] = last_alive;
+ break;
+ case RTE_KA_STATE_DOZING:
+ case RTE_KA_STATE_SLEEP:
+ case RTE_KA_STATE_DEAD:
+ case RTE_KA_STATE_GONE:
+ ka_shm->core_state[core_id] = core_state;
+ ka_shm->core_last_seen_times[core_id] = last_alive;
+ break;
+ case RTE_KA_STATE_UNUSED:
+ ka_shm->core_state[core_id] = KA_STATE_UNUSED;
+ break;
+ }
+}
+
/* Tries to allocate new mempool on requested_socket_id with
* mbuf size corresponding to requested_mtu.
* On success new configuration will be applied.
@@ -18,15 +18,20 @@
#define NETDEV_DPDK_H
#include <config.h>
+#include <stdint.h>
#include "openvswitch/compiler.h"
struct dp_packet;
+enum rte_keepalive_state;
#ifdef DPDK_NETDEV
void netdev_dpdk_register(void);
void free_dpdk_buf(struct dp_packet *);
+void dpdk_failcore_cb(void *, const int);
+void dpdk_ka_update_core_state(void *ptr, const int,
+ const enum rte_keepalive_state, uint64_t);
#else
This commit introduces the initial keepalive support by adding 'keepalive' module and also helper and initialization functions that will be invoked by later commits. Signed-off-by: Bhanuprakash Bodireddy <bhanuprakash.bodireddy@intel.com> --- lib/automake.mk | 2 + lib/dpdk.c | 23 ++++++++ lib/dpdk.h | 1 + lib/keepalive.c | 163 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/keepalive.h | 59 ++++++++++++++++++++ lib/netdev-dpdk.c | 80 ++++++++++++++++++++++++++- lib/netdev-dpdk.h | 5 ++ 7 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 lib/keepalive.c create mode 100644 lib/keepalive.h