[ovs-dev,v13,12/12] dpcls-avx512: Enable avx512 vector popcount instruction.

Message ID	20210617161825.94741-13-cian.ferriter@intel.com
State	Changes Requested
Headers	show Return-Path: <ovs-dev-bounces@openvswitch.org> IronPort-SDR: JFSVvlHZ7RqoqvwqUZe9EN4CcEa0sjT/L6gYVBt9Vp4pBo386hFBjF0af/vIaYjqsqA+pz0QYV wFtHNAyaHkBw== IronPort-SDR: 8x/QgcUhqnjGkgIizpJYtPo7mVIAmEjY9icXDHjSwgwa2L3M0isIdNH1WZNM3yCSeW8id41HHA xpn7qZBH1Z+A== From: Cian Ferriter <cian.ferriter@intel.com> To: ovs-dev@openvswitch.org Date: Thu, 17 Jun 2021 17:18:25 +0100 Message-Id: <20210617161825.94741-13-cian.ferriter@intel.com> In-Reply-To: <20210617161825.94741-1-cian.ferriter@intel.com> References: <20210617161825.94741-1-cian.ferriter@intel.com> MIME-Version: 1.0 Cc: i.maximets@ovn.org Subject: [ovs-dev] [v13 12/12] dpcls-avx512: Enable avx512 vector popcount instruction. Precedence: list Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ovs-dev-bounces@openvswitch.org Sender: "dev" <ovs-dev-bounces@openvswitch.org>
Series	DPIF Framework + Optimizations \| expand [ovs-dev,v13,00/12] DPIF Framework + Optimizations [ovs-dev,v13,01/12] dpif-netdev: Refactor to multiple header files. [ovs-dev,v13,02/12] dpif-netdev: Split HWOL out to own header file. [ovs-dev,v13,03/12] dpif-netdev: Add function pointer for netdev input. [ovs-dev,v13,04/12] dpif-avx512: Add ISA implementation of dpif. [ovs-dev,v13,05/12] dpif-netdev: Add command to switch dpif implementation. [ovs-dev,v13,06/12] dpif-netdev: Add command to get dpif implementations. [ovs-dev,v13,07/12] dpif-netdev: Add a partial HWOL PMD statistic. [ovs-dev,v13,08/12] dpif-netdev-unixctl.man: Document subtable-lookup-* CMDs [ovs-dev,v13,09/12] dpif-netdev/dpcls-avx512: Enable 16 block processing. [ovs-dev,v13,10/12] dpif-netdev/dpcls: Specialize more subtable signatures. [ovs-dev,v13,11/12] dpdk: Cache result of CPU ISA checks. [ovs-dev,v13,12/12] dpcls-avx512: Enable avx512 vector popcount instruction.

diff --git a/NEWS b/NEWS index f03bfeb5d..bc1db7948 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,9 @@ Post-v2.15.0 * Enable AVX512 optimized DPCLS to search subtables with larger miniflows. * Add more specialized DPCLS subtables to cover common rules, enhancing the lookup performance. + * Enable the AVX512 DPCLS implementation to use VPOPCNT instruction if the + CPU supports it. This enhances performance by using the native vpopcount + instructions, instead of the emulated version of vpopcount. - ovs-ctl: * New option '--no-record-hostname' to disable hostname configuration in ovsdb on startup. diff --git a/lib/dpdk.c b/lib/dpdk.c index c883a4b8b..a9494a40f 100644 --- a/lib/dpdk.c +++ b/lib/dpdk.c @@ -655,6 +655,7 @@ dpdk_get_cpu_has_isa(const char *arch, const char *feature) #if __x86_64__ /* CPU flags only defined for the architecture that support it. */ CHECK_CPU_FEATURE(feature, "avx512f", RTE_CPUFLAG_AVX512F); + CHECK_CPU_FEATURE(feature, "avx512vpopcntdq", RTE_CPUFLAG_AVX512VPOPCNTDQ); CHECK_CPU_FEATURE(feature, "bmi2", RTE_CPUFLAG_BMI2); #endif diff --git a/lib/dpif-netdev-lookup-avx512-gather.c b/lib/dpif-netdev-lookup-avx512-gather.c index 0b51ef9dc..bc359dc4a 100644 --- a/lib/dpif-netdev-lookup-avx512-gather.c +++ b/lib/dpif-netdev-lookup-avx512-gather.c @@ -53,6 +53,15 @@ VLOG_DEFINE_THIS_MODULE(dpif_lookup_avx512_gather); + +/* Wrapper function required to enable ISA. */ +static inline __m512i +__attribute__((__target__("avx512vpopcntdq"))) +_mm512_popcnt_epi64_wrapper(__m512i v_in) +{ + return _mm512_popcnt_epi64(v_in); +} + static inline __m512i _mm512_popcnt_epi64_manual(__m512i v_in) { @@ -131,6 +140,7 @@ netdev_rule_matches_key(const struct dpcls_rule *rule, * pkt_mf_u0_pop: population count of bits in u0 of the packet. * zero_mask: bitmask of lanes to zero as packet doesn't have mf bits set. * u64_lanes_mask: bitmask of lanes to process. + * use_vpop: compile-time constant indicating if VPOPCNT instruction allowed. */ static inline ALWAYS_INLINE __m512i avx512_blocks_gather(__m512i v_u0, @@ -141,7 +151,8 @@ avx512_blocks_gather(__m512i v_u0, __mmask64 u1_bcast_msk, const uint64_t pkt_mf_u0_pop, __mmask64 zero_mask, - __mmask64 u64_lanes_mask) + __mmask64 u64_lanes_mask, + const uint32_t use_vpop) { /* Suggest to compiler to load tbl blocks ahead of gather(). */ __m512i v_tbl_blocks = _mm512_maskz_loadu_epi64(u64_lanes_mask, @@ -155,8 +166,15 @@ avx512_blocks_gather(__m512i v_u0, tbl_mf_masks); __m512i v_masks = _mm512_and_si512(v_pkt_bits, v_tbl_masks); - /* Manual AVX512 popcount for u64 lanes. */ - __m512i v_popcnts = _mm512_popcnt_epi64_manual(v_masks); + /* Calculate AVX512 popcount for u64 lanes using the native instruction + * if available, or using emulation if not available. + */ + __m512i v_popcnts; + if (use_vpop) { + v_popcnts = _mm512_popcnt_epi64_wrapper(v_masks); + } else { + v_popcnts = _mm512_popcnt_epi64_manual(v_masks); + } /* Add popcounts and offset for u1 bits. */ __m512i v_idx_u0_offset = _mm512_maskz_set1_epi64(u1_bcast_msk, @@ -181,7 +199,8 @@ avx512_lookup_impl(struct dpcls_subtable *subtable, const struct netdev_flow_key *keys[], struct dpcls_rule **rules, const uint32_t bit_count_u0, - const uint32_t bit_count_u1) + const uint32_t bit_count_u1, + const uint32_t use_vpop) { OVS_ALIGNED_VAR(CACHE_LINE_SIZE)uint64_t block_cache[BLOCKS_CACHE_SIZE]; uint32_t hashes[NETDEV_MAX_BURST]; @@ -233,7 +252,8 @@ avx512_lookup_impl(struct dpcls_subtable *subtable, u1_bcast_mask, pkt_mf_u0_pop, zero_mask, - bit_count_total_mask); + bit_count_total_mask, + use_vpop); _mm512_storeu_si512(&block_cache[i * MF_BLOCKS_PER_PACKET], v_blocks); if (bit_count_total > 8) { @@ -254,7 +274,8 @@ avx512_lookup_impl(struct dpcls_subtable *subtable, u1_bcast_mask_gt8, pkt_mf_u0_pop, zero_mask_gt8, - bit_count_gt8_mask); + bit_count_gt8_mask, + use_vpop); _mm512_storeu_si512(&block_cache[(i * MF_BLOCKS_PER_PACKET) + 8], v_blocks_gt8); } @@ -303,7 +324,11 @@ avx512_lookup_impl(struct dpcls_subtable *subtable, return found_map; } -/* Expand out specialized functions with U0 and U1 bit attributes. */ +/* Expand out specialized functions with U0 and U1 bit attributes. As the + * AVX512 vpopcnt instruction is not supported on all AVX512 capable CPUs, + * create two functions for each miniflow signature. This allows the runtime + * CPU detection in probe() to select the ideal implementation. + */ #define DECLARE_OPTIMIZED_LOOKUP_FUNCTION(U0, U1) \ static uint32_t \ dpcls_avx512_gather_mf_##U0##_##U1(struct dpcls_subtable *subtable, \ @@ -311,7 +336,20 @@ avx512_lookup_impl(struct dpcls_subtable *subtable, const struct netdev_flow_key *keys[], \ struct dpcls_rule **rules) \ { \ - return avx512_lookup_impl(subtable, keys_map, keys, rules, U0, U1); \ + const uint32_t use_vpop = 0; \ + return avx512_lookup_impl(subtable, keys_map, keys, rules, \ + U0, U1, use_vpop); \ + } \ + \ + static uint32_t __attribute__((__target__("avx512vpopcntdq"))) \ + dpcls_avx512_gather_mf_##U0##_##U1##_vpop(struct dpcls_subtable *subtable,\ + uint32_t keys_map, \ + const struct netdev_flow_key *keys[], \ + struct dpcls_rule **rules) \ + { \ + const uint32_t use_vpop = 1; \ + return avx512_lookup_impl(subtable, keys_map, keys, rules, \ + U0, U1, use_vpop); \ } \ DECLARE_OPTIMIZED_LOOKUP_FUNCTION(9, 4) @@ -321,11 +359,18 @@ DECLARE_OPTIMIZED_LOOKUP_FUNCTION(5, 1) DECLARE_OPTIMIZED_LOOKUP_FUNCTION(4, 1) DECLARE_OPTIMIZED_LOOKUP_FUNCTION(4, 0) -/* Check if a specialized function is valid for the required subtable. */ -#define CHECK_LOOKUP_FUNCTION(U0, U1) \ +/* Check if a specialized function is valid for the required subtable. + * The use_vpop variable is used to decide if the VPOPCNT instruction can be + * used or not. + */ +#define CHECK_LOOKUP_FUNCTION(U0, U1, use_vpop) \ ovs_assert((U0 + U1) <= (NUM_U64_IN_ZMM_REG * 2)); \ if (!f && u0_bits == U0 && u1_bits == U1) { \ - f = dpcls_avx512_gather_mf_##U0##_##U1; \ + if (use_vpop) { \ + f = dpcls_avx512_gather_mf_##U0##_##U1##_vpop; \ + } else { \ + f = dpcls_avx512_gather_mf_##U0##_##U1; \ + } \ } static uint32_t @@ -333,9 +378,11 @@ dpcls_avx512_gather_mf_any(struct dpcls_subtable *subtable, uint32_t keys_map, const struct netdev_flow_key *keys[], struct dpcls_rule **rules) { + const uint32_t use_vpop = 0; return avx512_lookup_impl(subtable, keys_map, keys, rules, subtable->mf_bits_set_unit0, - subtable->mf_bits_set_unit1); + subtable->mf_bits_set_unit1, + use_vpop); } dpcls_subtable_lookup_func @@ -349,12 +396,14 @@ dpcls_subtable_avx512_gather_probe(uint32_t u0_bits, uint32_t u1_bits) return NULL; } - CHECK_LOOKUP_FUNCTION(9, 4); - CHECK_LOOKUP_FUNCTION(9, 1); - CHECK_LOOKUP_FUNCTION(5, 3); - CHECK_LOOKUP_FUNCTION(5, 1); - CHECK_LOOKUP_FUNCTION(4, 1); - CHECK_LOOKUP_FUNCTION(4, 0); + int use_vpop = dpdk_get_cpu_has_isa("x86_64", "avx512vpopcntdq"); + + CHECK_LOOKUP_FUNCTION(9, 4, use_vpop); + CHECK_LOOKUP_FUNCTION(9, 1, use_vpop); + CHECK_LOOKUP_FUNCTION(5, 3, use_vpop); + CHECK_LOOKUP_FUNCTION(5, 1, use_vpop); + CHECK_LOOKUP_FUNCTION(4, 1, use_vpop); + CHECK_LOOKUP_FUNCTION(4, 0, use_vpop); /* Check if the _any looping version of the code can perform this miniflow * lookup. Performance gain may be less pronounced due to non-specialized

[ovs-dev,v13,12/12] dpcls-avx512: Enable avx512 vector popcount instruction.

Commit Message

Comments

Patch