@@ -213,6 +213,8 @@ lib_libopenvswitch_la_SOURCES = \
lib/object-collection.h \
lib/odp-execute.c \
lib/odp-execute.h \
+ lib/odp-execute-private.c \
+ lib/odp-execute-private.h \
lib/odp-util.c \
lib/odp-util.h \
lib/ofp-actions.c \
@@ -1675,6 +1675,10 @@ create_dpif_netdev(struct dp_netdev *dp)
dpif->dp = dp;
dpif->last_port_seq = seq_read(dp->port_seq);
+ /* Called once at initialization time. This handles setting up the state
+ * of the actions functions at init time. */
+ odp_execute_init();
+
return &dpif->dpif;
}
new file mode 100644
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2022 Intel.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "dpdk.h"
+#include "dp-packet.h"
+#include "odp-execute-private.h"
+#include "odp-netlink.h"
+#include "odp-util.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(odp_execute_impl);
+
+static struct odp_execute_action_impl action_impls[] = {
+ [ACTION_IMPL_SCALAR] = {
+ .available = false,
+ .name = "scalar",
+ .init_func = NULL,
+ },
+};
+
+static void
+action_impl_copy_funcs(struct odp_execute_action_impl *src,
+ const struct odp_execute_action_impl *dst)
+{
+ for (int i = 0; i < __OVS_ACTION_ATTR_MAX; i++) {
+ atomic_store_relaxed(&src->funcs[i], dst->funcs[i]);
+ }
+}
+
+void
+odp_execute_action_init(void)
+{
+ /* Each impl's function array is initialized to reflect the scalar
+ * implementation. This simplifies adding optimized implementations,
+ * as the autovalidator can always compare all actions.
+ *
+ * Below will check if impl is available and copies the scalar functions
+ * to all other implementations.
+ */
+ for (int i = 0; i < ACTION_IMPL_MAX; i++) {
+ bool avail = true;
+
+ if (action_impls[i].init_func) {
+ /* Return zero is success, non-zero means error. */
+ avail = (action_impls[i].init_func(&action_impls[i]) == 0);
+ }
+
+ action_impls[i].available = avail;
+
+ if (i != ACTION_IMPL_SCALAR) {
+ action_impl_copy_funcs(&action_impls[i],
+ &action_impls[ACTION_IMPL_SCALAR]);
+ }
+
+ if (action_impls[i].available == true) {
+ action_impls[i].init_func(&action_impls[i]);
+ }
+
+ VLOG_INFO("Action implementation %s (available: %s)\n",
+ action_impls[i].name, avail ? "available" : "not available");
+ }
+}
new file mode 100644
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2022 Intel.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ODP_EXTRACT_PRIVATE
+#define ODP_EXTRACT_PRIVATE 1
+
+#include "dp-packet.h"
+#include "odp-execute.h"
+#include "odp-netlink.h"
+#include "ovs-atomic.h"
+
+/* Forward declaration for typedef. */
+struct odp_execute_action_impl;
+
+/* Typedef for an initialization function that can initialize each
+ * implementation, checking requirements such as CPU ISA.
+ */
+typedef int (*odp_execute_action_init_func)
+ (struct odp_execute_action_impl *self);
+
+/* Structure represents an implementation of the odp actions. */
+struct odp_execute_action_impl {
+ /* When set, the CPU ISA required for this implementation is available
+ * and the implementation can be used.
+ */
+ bool available;
+
+ /* Name of the implementation. */
+ const char *name;
+
+ /* Function is used to detect if this CPU has the ISA required
+ * to run the optimized action implementation and if available, initializes
+ * the implementation for use.
+ */
+ odp_execute_action_init_func init_func;
+
+ /* An array of callback functions, one for each action. */
+ ATOMIC(odp_execute_action_cb) funcs[__OVS_ACTION_ATTR_MAX];
+};
+
+/* Order of Actions implementations. */
+enum odp_execute_action_impl_idx {
+ ACTION_IMPL_SCALAR,
+ /* See ACTION_IMPL_BEGIN below, for "first to-be-validated" impl.
+ * Do not change the autovalidator position in this list without updating
+ * the define below.
+ */
+
+ ACTION_IMPL_MAX,
+};
+
+/* Index to start verifying implementations from. */
+BUILD_ASSERT_DECL(ACTION_IMPL_SCALAR == 0);
+
+/* Odp execute init handles setting up the state of the actions functions at
+ * initialization time. It cannot return errors, as it must always succeed in
+ * initializing the scalar/generic codepath.
+ */
+void odp_execute_action_init(void);
+
+#endif /* ODP_EXTRACT_PRIVATE */
@@ -17,6 +17,7 @@
#include <config.h>
#include "odp-execute.h"
+#include "odp-execute-private.h"
#include <sys/types.h>
#include <netinet/in.h>
#include <arpa/inet.h>
@@ -833,6 +834,23 @@ requires_datapath_assistance(const struct nlattr *a)
return false;
}
+/* The active function pointers on the datapath. ISA optimized implementations
+ * are enabled by plugging them into this static arary, which is consulted when
+ * applying actions on the datapath.
+ */
+static struct odp_execute_action_impl actions_active_impl;
+
+void
+odp_execute_init(void)
+{
+ static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
+ if (ovsthread_once_start(&once)) {
+ odp_execute_action_init();
+ ovsthread_once_done(&once);
+ }
+}
+
+
/* Executes all of the 'actions_len' bytes of datapath actions in 'actions' on
* the packets in 'batch'. If 'steal' is true, possibly modifies and
* definitely free the packets in 'batch', otherwise leaves 'batch' unchanged.
@@ -857,14 +875,12 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
NL_ATTR_FOR_EACH_UNSAFE (a, left, actions, actions_len) {
int type = nl_attr_type(a);
+ enum ovs_action_attr attr_type = (enum ovs_action_attr) type;
bool last_action = (left <= NLA_ALIGN(a->nla_len));
+ bool should_steal = steal && last_action;
if (requires_datapath_assistance(a)) {
if (dp_execute_action) {
- /* Allow 'dp_execute_action' to steal the packet data if we do
- * not need it any more. */
- bool should_steal = steal && last_action;
-
dp_execute_action(dp, batch, a, should_steal);
if (last_action || dp_packet_batch_is_empty(batch)) {
@@ -879,8 +895,20 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
continue;
}
- switch ((enum ovs_action_attr) type) {
+ /* If type is set in the active actions implementation, call the
+ * function-pointer and continue to the next action.
+ */
+ if (actions_active_impl.funcs[attr_type] &&
+ attr_type <= OVS_ACTION_ATTR_MAX) {
+ actions_active_impl.funcs[attr_type](batch, a);
+ continue;
+ }
+
+ /* If the action was not handled by the active function pointers above,
+ * process them by switching on the type below.
+ */
+ switch (attr_type) {
case OVS_ACTION_ATTR_HASH: {
const struct ovs_action_hash *hash_act = nl_attr_get(a);
@@ -1094,6 +1122,9 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
case __OVS_ACTION_ATTR_MAX:
OVS_NOT_REACHED();
}
+
+ /* Do not add any generic processing here, as it won't be executed when
+ * an ISA-specific action implementation exists. */
}
dp_packet_delete_batch(batch, steal);
@@ -28,6 +28,13 @@ struct dp_packet;
struct pkt_metadata;
struct dp_packet_batch;
+
+/* Called once at initialization time. */
+void odp_execute_init(void);
+
+typedef void (*odp_execute_action_cb)(struct dp_packet_batch *batch,
+ const struct nlattr *action);
+
typedef void (*odp_execute_cb)(void *dp, struct dp_packet_batch *batch,
const struct nlattr *action, bool should_steal);