@@ -38,6 +38,12 @@ BUILD_ASSERT_DECL(offsetof(struct dp_packet, l3_ofs) +
MEMBER_SIZEOF(struct dp_packet, l3_ofs) ==
offsetof(struct dp_packet, l4_ofs));
+BUILD_ASSERT_DECL(offsetof(struct ovs_key_ethernet, eth_src) +
+ MEMBER_SIZEOF(struct ovs_key_ethernet, eth_src) ==
+ offsetof(struct ovs_key_ethernet, eth_dst));
+
+static struct odp_execute_action_impl avx512_impl;
+
/* Adjust the size of the l2 portion of the dp_packet, updating the l2
* pointer and the layer offsets. The function will broadcast resize_by_bytes
* across a register and uses a kmask to identify which lanes should be
@@ -144,6 +150,61 @@ action_avx512_push_vlan(struct dp_packet_batch *batch, const struct nlattr *a)
}
}
+/* This function will load the contents of eth_header into a 128-bit wide
+ * register. Then an 8-byte shuffle is required to shuffle both key and
+ * mask to match the layout of the eth_header struct. A bitwise ANDNOT and OR
+ * is performed on the entire header and results are stored back. */
+static void
+action_avx512_eth_set_addrs(struct dp_packet_batch *batch,
+ const struct nlattr *a)
+{
+ a = nl_attr_get(a);
+ const struct ovs_key_ethernet *key = nl_attr_get(a);
+ const struct ovs_key_ethernet *mask = get_mask(a, struct ovs_key_ethernet);
+ struct dp_packet *packet;
+
+ __m128i v_src = _mm_loadu_si128((void *) key);
+ __m128i v_mask = _mm_loadu_si128((void *) mask);
+
+ DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+
+ struct eth_header *eh = dp_packet_eth(packet);
+
+ if (!eh) {
+ continue;
+ }
+
+ static const uint8_t eth_shuffle[16] = {
+ 6, 7, 8, 9, 10, 11, 0, 1,
+ 2, 3, 4, 5, 12, 13, 14, 15
+ };
+
+ __m128i v_dst = _mm_loadu_si128((void *) eh);
+ __m128i v_shuf = _mm_loadu_si128((void *) eth_shuffle);
+
+ v_src = _mm_shuffle_epi8(v_src, v_shuf);
+ v_mask = _mm_shuffle_epi8(v_mask, v_shuf);
+
+ __m128i dst_masked = _mm_andnot_si128(v_mask, v_dst);
+ __m128i res = _mm_or_si128(v_src, dst_masked);
+
+ __m128i res_blend = _mm_blend_epi16(v_dst, res, 0x3F);
+ _mm_storeu_si128((void *) eh, res_blend);
+ }
+}
+
+static void
+action_avx512_set_masked(struct dp_packet_batch *batch OVS_UNUSED,
+ const struct nlattr *a)
+{
+ a = nl_attr_get(a);
+ enum ovs_key_attr attr_type = nl_attr_type(a);
+
+ if (avx512_impl.set_masked_funcs[attr_type]) {
+ avx512_impl.set_masked_funcs[attr_type](batch, a);
+ }
+}
+
/* Probe functions to check ISA requirements. */
static bool
avx512_isa_probe(void)
@@ -176,6 +237,14 @@ action_avx512_init(struct odp_execute_action_impl *self)
* are identified by OVS_ACTION_ATTR_*. */
self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_avx512_pop_vlan;
self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_avx512_push_vlan;
+ self->funcs[OVS_ACTION_ATTR_SET_MASKED] = action_avx512_set_masked;
+
+ /* Set function pointers that need a 2nd-level function. SET_MASKED action
+ * requires further processing for action type. Note that 2nd level items
+ * are identified by OVS_KEY_ATTR_*. */
+ self->set_masked_funcs[OVS_KEY_ATTR_ETHERNET] =
+ action_avx512_eth_set_addrs;
+ avx512_impl = *self;
return 0;
}
@@ -29,6 +29,8 @@
VLOG_DEFINE_THIS_MODULE(odp_execute_impl);
static int active_action_impl_index;
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+static struct odp_execute_action_impl autoval_impl;
+static bool set_masked = false;
static struct odp_execute_action_impl action_impls[] = {
[ACTION_IMPL_AUTOVALIDATOR] = {
@@ -59,6 +61,11 @@ action_impl_copy_funcs(struct odp_execute_action_impl *src,
for (int i = 0; i < __OVS_ACTION_ATTR_MAX; i++) {
atomic_store_relaxed(&src->funcs[i], dst->funcs[i]);
}
+
+ for (uint32_t i = 0; i < __OVS_KEY_ATTR_MAX; i++) {
+ atomic_store_relaxed(&src->set_masked_funcs[i],
+ dst->set_masked_funcs[i]);
+ }
}
int
@@ -135,19 +142,36 @@ action_autoval_generic(struct dp_packet_batch *batch, const struct nlattr *a)
bool failed = false;
int type = nl_attr_type(a);
enum ovs_action_attr attr_type = (enum ovs_action_attr) type;
+ enum ovs_key_attr key_attr_type = (enum ovs_key_attr) type;
+
+ if (attr_type == OVS_ACTION_ATTR_SET_MASKED) {
+ set_masked = true;
+ const struct nlattr *key = nl_attr_get(a);
+ key_attr_type = nl_attr_type(key);
+ }
+
struct odp_execute_action_impl *scalar = &action_impls[ACTION_IMPL_SCALAR];
struct dp_packet_batch good_batch;
dp_packet_batch_clone(&good_batch, batch);
- scalar->funcs[attr_type](&good_batch, a);
+ if (!set_masked) {
+ scalar->funcs[attr_type](&good_batch, a);
+ } else {
+ scalar->set_masked_funcs[key_attr_type](&good_batch, a);
+ }
for (int impl = ACTION_IMPL_BEGIN; impl < ACTION_IMPL_MAX; impl++) {
/* Clone original batch and execute implementation under test. */
struct dp_packet_batch test_batch;
dp_packet_batch_clone(&test_batch, batch);
- action_impls[impl].funcs[attr_type](&test_batch, a);
+
+ if (!set_masked) {
+ action_impls[impl].funcs[attr_type](&test_batch, a);
+ } else {
+ action_impls[impl].set_masked_funcs[key_attr_type](&test_batch, a);
+ }
/* Loop over implementations, checking each one. */
for (int pidx = 0; pidx < batch->count; pidx++) {
@@ -200,7 +224,26 @@ action_autoval_generic(struct dp_packet_batch *batch, const struct nlattr *a)
dp_packet_delete_batch(&good_batch, 1);
/* Apply the action to the original batch for continued processing. */
- scalar->funcs[attr_type](batch, a);
+ if (!set_masked) {
+ scalar->funcs[attr_type](batch, a);
+ } else {
+ scalar->set_masked_funcs[key_attr_type](batch, a);
+ }
+
+ set_masked = false;
+}
+
+static void
+action_set_masked_init(struct dp_packet_batch *batch OVS_UNUSED,
+ const struct nlattr *a)
+{
+ const struct nlattr *type = nl_attr_get(a);
+ enum ovs_key_attr attr_type = nl_attr_type(type);
+
+ if (autoval_impl.set_masked_funcs[attr_type]) {
+ set_masked = true;
+ autoval_impl.set_masked_funcs[attr_type](batch, a);
+ }
}
int
@@ -210,6 +253,13 @@ action_autoval_init(struct odp_execute_action_impl *self)
* are identified by OVS_ACTION_ATTR_*. */
self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_autoval_generic;
self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_autoval_generic;
+ self->funcs[OVS_ACTION_ATTR_SET_MASKED] = action_set_masked_init;
+
+ /* Set function pointers that need a 2nd-level function. SET_MASKED action
+ * requires further processing for action type. Note that 2nd level items
+ * are identified by OVS_KEY_ATTR_*. */
+ self->set_masked_funcs[OVS_KEY_ATTR_ETHERNET] = action_autoval_generic;
+ autoval_impl = *self;
return 0;
}
@@ -49,6 +49,10 @@ struct odp_execute_action_impl {
/* An array of callback functions, one for each action. */
ATOMIC(odp_execute_action_cb) funcs[__OVS_ACTION_ATTR_MAX];
+
+ /* An array of callback functions, one for each action type. */
+ ATOMIC(odp_execute_action_cb) set_masked_funcs[__OVS_KEY_ATTR_MAX];
+
};
/* Order of Actions implementations. */
@@ -561,8 +561,6 @@ odp_execute_set_action(struct dp_packet *packet, const struct nlattr *a)
}
}
-#define get_mask(a, type) ((const type *)(const void *)(a + 1) + 1)
-
static void
odp_execute_masked_set_action(struct dp_packet *packet,
const struct nlattr *a)
@@ -582,11 +580,6 @@ odp_execute_masked_set_action(struct dp_packet *packet,
| (md->pkt_mark & ~*get_mask(a, uint32_t));
break;
- case OVS_KEY_ATTR_ETHERNET:
- odp_eth_set_addrs(packet, nl_attr_get(a),
- get_mask(a, struct ovs_key_ethernet));
- break;
-
case OVS_KEY_ATTR_NSH: {
odp_set_nsh(packet, a, true);
break;
@@ -669,6 +662,8 @@ odp_execute_masked_set_action(struct dp_packet *packet,
case OVS_KEY_ATTR_TCP_FLAGS:
case OVS_KEY_ATTR_TUNNEL_INFO:
case __OVS_KEY_ATTR_MAX:
+ /* The following action types are handled by the scalar implementation. */
+ case OVS_KEY_ATTR_ETHERNET:
default:
OVS_NOT_REACHED();
}
@@ -834,6 +829,12 @@ requires_datapath_assistance(const struct nlattr *a)
return false;
}
+/* The active function pointers on the datapath. ISA optimized implementations
+ * are enabled by plugging them into this static arary, which is consulted when
+ * applying actions on the datapath.
+ */
+static struct odp_execute_action_impl actions_active_impl;
+
static void
action_pop_vlan(struct dp_packet_batch *batch,
const struct nlattr *a OVS_UNUSED)
@@ -856,6 +857,36 @@ action_push_vlan(struct dp_packet_batch *batch, const struct nlattr *a)
}
}
+static void
+action_set_masked(struct dp_packet_batch *batch, const struct nlattr *a)
+{
+ struct dp_packet *packet;
+
+ const struct nlattr *key = nl_attr_get(a);
+ enum ovs_key_attr key_type = nl_attr_type(key);
+
+ if (actions_active_impl.set_masked_funcs[key_type]) {
+ actions_active_impl.set_masked_funcs[key_type](batch, a);
+ } else {
+ a = nl_attr_get(a);
+ DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+ odp_execute_masked_set_action(packet, a);
+ }
+ }
+}
+
+static void
+action_mod_eth(struct dp_packet_batch *batch, const struct nlattr *a)
+{
+ a = nl_attr_get(a);
+ struct dp_packet *packet;
+
+ DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+ odp_eth_set_addrs(packet, nl_attr_get(a),
+ get_mask(a, struct ovs_key_ethernet));
+ }
+}
+
/* Implementation of the scalar actions impl init function. Build up the
* array of func ptrs here.
*/
@@ -866,16 +897,17 @@ odp_action_scalar_init(struct odp_execute_action_impl *self)
* are identified by OVS_ACTION_ATTR_*. */
self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_pop_vlan;
self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_push_vlan;
+ self->funcs[OVS_ACTION_ATTR_SET_MASKED] = action_set_masked;
+
+ /* Set function pointers that need a 2nd-level function. SET_MASKED action
+ * requires further processing for action type. Note that 2nd level items
+ * are identified by OVS_KEY_ATTR_*. */
+ self->set_masked_funcs[OVS_KEY_ATTR_ETHERNET] = action_mod_eth;
+ actions_active_impl = *self;
return 0;
}
-/* The active function pointers on the datapath. ISA optimized implementations
- * are enabled by plugging them into this static arary, which is consulted when
- * applying actions on the datapath.
- */
-static struct odp_execute_action_impl actions_active_impl;
-
void
odp_execute_init(void)
{
@@ -1028,12 +1060,6 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
}
break;
- case OVS_ACTION_ATTR_SET_MASKED:
- DP_PACKET_BATCH_FOR_EACH(i, packet, batch) {
- odp_execute_masked_set_action(packet, nl_attr_get(a));
- }
- break;
-
case OVS_ACTION_ATTR_SAMPLE:
DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
odp_execute_sample(dp, packet, steal && last_action, a,
@@ -1160,6 +1186,7 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
/* The following actions are handled by the scalar implementation. */
case OVS_ACTION_ATTR_POP_VLAN:
case OVS_ACTION_ATTR_PUSH_VLAN:
+ case OVS_ACTION_ATTR_SET_MASKED:
OVS_NOT_REACHED();
}
@@ -50,4 +50,7 @@ void odp_execute_actions(void *dp, struct dp_packet_batch *batch,
bool steal,
const struct nlattr *actions, size_t actions_len,
odp_execute_cb dp_execute_action);
+
+#define get_mask(a, type) ((const type *)(const void *)(a + 1) + 1)
+
#endif
This commit includes infrastructure changes for enabling set_masked_X actions and also adds support for the AVX512 implementation of the eth_set_addrs action. Signed-off-by: Emma Finn <emma.finn@intel.com> --- lib/odp-execute-avx512.c | 69 +++++++++++++++++++++++++++++++++++++++ lib/odp-execute-private.c | 56 +++++++++++++++++++++++++++++-- lib/odp-execute-private.h | 4 +++ lib/odp-execute.c | 65 +++++++++++++++++++++++++----------- lib/odp-execute.h | 3 ++ 5 files changed, 175 insertions(+), 22 deletions(-)