From patchwork Mon Jun 3 09:51:16 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yanqin Wei X-Patchwork-Id: 1109172 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=openvswitch.org (client-ip=140.211.169.12; helo=mail.linuxfoundation.org; envelope-from=ovs-dev-bounces@openvswitch.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=arm.com Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 45HVhH5m5fz9s4V for ; Mon, 3 Jun 2019 19:51:59 +1000 (AEST) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id 859FBD8D; Mon, 3 Jun 2019 09:51:29 +0000 (UTC) X-Original-To: dev@openvswitch.org Delivered-To: ovs-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id F38A5C79 for ; Mon, 3 Jun 2019 09:51:27 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from foss.arm.com (usa-sjc-mx-foss1.foss.arm.com [217.140.101.70]) by smtp1.linuxfoundation.org (Postfix) with ESMTP id 8B9D5A3 for ; Mon, 3 Jun 2019 09:51:27 +0000 (UTC) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 37DAB374; Mon, 3 Jun 2019 02:51:27 -0700 (PDT) Received: from net-arm-thunderx2.shanghai.arm.com (net-arm-thunderx2.shanghai.arm.com [10.169.40.40]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 377A13F690; Mon, 3 Jun 2019 02:51:26 -0700 (PDT) From: Yanqin Wei To: dev@openvswitch.org Date: Mon, 3 Jun 2019 17:51:16 +0800 Message-Id: <1559555477-36363-2-git-send-email-Yanqin.Wei@arm.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1559555477-36363-1-git-send-email-Yanqin.Wei@arm.com> References: <1559555477-36363-1-git-send-email-Yanqin.Wei@arm.com> X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on smtp1.linux-foundation.org Cc: nd@arm.com, Malvika.Gupta@arm.com, Gavin.Hu@arm.com Subject: [ovs-dev] [PATCH v4 1/2] dpif-netdev: add EMC entries counter X-BeenThere: ovs-dev@openvswitch.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: ovs-dev-bounces@openvswitch.org Errors-To: ovs-dev-bounces@openvswitch.org Implement entries counter for EMC. It could be used for improvement for EMC lookup. Signed-off-by: Yanqin Wei Reviewed-by: Gavin Hu --- lib/dpif-netdev.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 5a6f2ab..c74cc02 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -197,6 +197,7 @@ struct emc_entry { struct emc_cache { struct emc_entry entries[EM_FLOW_HASH_ENTRIES]; int sweep_idx; /* For emc_cache_slow_sweep(). */ + uint32_t counter; }; struct smc_bucket { @@ -826,7 +827,7 @@ static int dpif_netdev_xps_get_tx_qid(const struct dp_netdev_pmd_thread *pmd, struct tx_port *tx); static inline bool emc_entry_alive(struct emc_entry *ce); -static void emc_clear_entry(struct emc_entry *ce); +static void emc_clear_entry(struct emc_cache *cache, struct emc_entry *ce); static void smc_clear_entry(struct smc_bucket *b, int idx); static void dp_netdev_request_reconfigure(struct dp_netdev *dp); @@ -840,6 +841,7 @@ emc_cache_init(struct emc_cache *flow_cache) { int i; + flow_cache->counter = 0; flow_cache->sweep_idx = 0; for (i = 0; i < ARRAY_SIZE(flow_cache->entries); i++) { flow_cache->entries[i].flow = NULL; @@ -872,8 +874,9 @@ emc_cache_uninit(struct emc_cache *flow_cache) { int i; + flow_cache->counter = 0; for (i = 0; i < ARRAY_SIZE(flow_cache->entries); i++) { - emc_clear_entry(&flow_cache->entries[i]); + emc_clear_entry(flow_cache, &flow_cache->entries[i]); } } @@ -904,7 +907,7 @@ emc_cache_slow_sweep(struct emc_cache *flow_cache) struct emc_entry *entry = &flow_cache->entries[flow_cache->sweep_idx]; if (!emc_entry_alive(entry)) { - emc_clear_entry(entry); + emc_clear_entry(flow_cache,entry); } flow_cache->sweep_idx = (flow_cache->sweep_idx + 1) & EM_FLOW_HASH_MASK; } @@ -2771,25 +2774,28 @@ emc_entry_alive(struct emc_entry *ce) } static void -emc_clear_entry(struct emc_entry *ce) +emc_clear_entry(struct emc_cache *cache, struct emc_entry *ce) { if (ce->flow) { dp_netdev_flow_unref(ce->flow); ce->flow = NULL; + cache->counter--; } } static inline void -emc_change_entry(struct emc_entry *ce, struct dp_netdev_flow *flow, - const struct netdev_flow_key *key) +emc_change_entry(struct emc_cache *cache, struct emc_entry *ce, + struct dp_netdev_flow *flow, const struct netdev_flow_key *key) { if (ce->flow != flow) { if (ce->flow) { dp_netdev_flow_unref(ce->flow); + cache->counter--; } if (dp_netdev_flow_ref(flow)) { ce->flow = flow; + cache->counter++; } else { ce->flow = NULL; } @@ -2809,7 +2815,7 @@ emc_insert(struct emc_cache *cache, const struct netdev_flow_key *key, EMC_FOR_EACH_POS_WITH_HASH(cache, current_entry, key->hash) { if (netdev_flow_key_equal(¤t_entry->key, key)) { /* We found the entry with the 'mf' miniflow */ - emc_change_entry(current_entry, flow, NULL); + emc_change_entry(cache,current_entry, flow, NULL); return; } @@ -2825,7 +2831,7 @@ emc_insert(struct emc_cache *cache, const struct netdev_flow_key *key, /* We didn't find the miniflow in the cache. * The 'to_be_replaced' entry is where the new flow will be stored */ - emc_change_entry(to_be_replaced, flow, key); + emc_change_entry(cache,to_be_replaced, flow, key); } static inline void From patchwork Mon Jun 3 09:51:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yanqin Wei X-Patchwork-Id: 1109174 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=openvswitch.org (client-ip=140.211.169.12; helo=mail.linuxfoundation.org; envelope-from=ovs-dev-bounces@openvswitch.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=arm.com Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 45HVhs5J5Dz9s4Y for ; Mon, 3 Jun 2019 19:52:29 +1000 (AEST) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id 2E40AD91; Mon, 3 Jun 2019 09:51:32 +0000 (UTC) X-Original-To: dev@openvswitch.org Delivered-To: ovs-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id 60990D89 for ; Mon, 3 Jun 2019 09:51:30 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from foss.arm.com (foss.arm.com [217.140.101.70]) by smtp1.linuxfoundation.org (Postfix) with ESMTP id CBD245D3 for ; Mon, 3 Jun 2019 09:51:29 +0000 (UTC) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 97CE915AB; Mon, 3 Jun 2019 02:51:29 -0700 (PDT) Received: from net-arm-thunderx2.shanghai.arm.com (net-arm-thunderx2.shanghai.arm.com [10.169.40.40]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 8EE113F690; Mon, 3 Jun 2019 02:51:28 -0700 (PDT) From: Yanqin Wei To: dev@openvswitch.org Date: Mon, 3 Jun 2019 17:51:17 +0800 Message-Id: <1559555477-36363-3-git-send-email-Yanqin.Wei@arm.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1559555477-36363-1-git-send-email-Yanqin.Wei@arm.com> References: <1559555477-36363-1-git-send-email-Yanqin.Wei@arm.com> X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on smtp1.linux-foundation.org Cc: nd@arm.com, Malvika.Gupta@arm.com, Gavin.Hu@arm.com Subject: [ovs-dev] [PATCH v4 2/2] dpif-netdev: dfc_process performance optimization by prefetching EMC entry. X-BeenThere: ovs-dev@openvswitch.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: ovs-dev-bounces@openvswitch.org Errors-To: ovs-dev-bounces@openvswitch.org It is observed that the throughput of medium number of flows(9-8191) is worse than low number of flows(1-8) in the EMC NIC2NIC test. It is because CPU cache-miss increasing in EMC lookup. Each flow need load at least one EMC entry to CPU L1 cache(several cache lines) and compare it with packet miniflow. This patch improves it by prefetching EMC entry in advance. Hash value can be obtained from dpdk rss hash, so this step can be advanced ahead of miniflow_extract() and prefetch EMC entry there. By testing on several kinds of cpu with 32K L1 cache(x86-64 and arm64), prefetch start to improve performance from 8~10 flows onwards. In order to benefit most modern CPUs, the minimum threshold is set to 20. The max threshold is set to EM_FLOW_HASH_ENTRIES-1 because entry prefetching become negative in huge number of flows. So this patch prefetch one EMC cache line only when EMC counter is 20-8191, which could ensure no side effect in all cases. Performance test was run in some arm and x86 platform. Medium number of flow case achieved around 2-3% improvement in RFC2544 test in x86 and arm. High number of flows(>8191) also benifit during EMC insertion, so it acheives around 2% improvement in 100k flows RFC2544 test. And low number of flows has almost no performance impact. Signed-off-by: Yanqin Wei Reviewed-by: Gavin Hu --- lib/dpif-netdev.c | 66 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 23 deletions(-) mode change 100644 => 100755 lib/dpif-netdev.c diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c old mode 100644 new mode 100755 index c74cc02..dc2ad64 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -189,6 +189,9 @@ struct netdev_flow_key { #define DEFAULT_EM_FLOW_INSERT_MIN (UINT32_MAX / \ DEFAULT_EM_FLOW_INSERT_INV_PROB) +/* Prefetch minimum threshold*/ +#define EMC_PREFETCH_MIN_THRESHOLD 10 + struct emc_entry { struct dp_netdev_flow *flow; struct netdev_flow_key key; /* key.hash used for emc hash value. */ @@ -215,6 +218,11 @@ struct dfc_cache { struct smc_cache smc_cache; }; +/* Prefetch in case of [EMC_PREFETCH_THRESHOLD,EM_FLOW_HASH_ENTRIES) entries*/ +#define EMC_PREFETCH_IN_RANGE(DFC_CACHE) \ + ((DFC_CACHE)->emc_cache.counter >= EMC_PREFETCH_MIN_THRESHOLD \ + && (DFC_CACHE)->emc_cache.counter < EM_FLOW_HASH_ENTRIES) + /* Iterate in the exact match cache through every entry that might contain a * miniflow with hash 'HASH'. */ #define EMC_FOR_EACH_POS_WITH_HASH(EMC, CURRENT_ENTRY, HASH) \ @@ -6172,41 +6180,41 @@ dp_netdev_upcall(struct dp_netdev_pmd_thread *pmd, struct dp_packet *packet_, } static inline uint32_t -dpif_netdev_packet_get_rss_hash_orig_pkt(struct dp_packet *packet, - const struct miniflow *mf) +dpif_netdev_packet_get_5tuple_hash(struct dp_packet *packet, + const struct miniflow *mf, + bool account_recirc_id) { - uint32_t hash; + uint32_t hash, recirc_depth; - if (OVS_LIKELY(dp_packet_rss_valid(packet))) { - hash = dp_packet_get_rss_hash(packet); - } else { - hash = miniflow_hash_5tuple(mf, 0); - dp_packet_set_rss_hash(packet, hash); + hash = miniflow_hash_5tuple(mf, 0); + + if (account_recirc_id) { + /* The RSS hash must account for the recirculation depth to avoid + * collisions in the exact match cache */ + recirc_depth = *recirc_depth_get_unsafe(); + hash = hash_finish(hash, recirc_depth); } + dp_packet_set_rss_hash(packet, hash); return hash; } static inline uint32_t dpif_netdev_packet_get_rss_hash(struct dp_packet *packet, - const struct miniflow *mf) + bool account_recirc_id) { uint32_t hash, recirc_depth; - if (OVS_LIKELY(dp_packet_rss_valid(packet))) { - hash = dp_packet_get_rss_hash(packet); - } else { - hash = miniflow_hash_5tuple(mf, 0); - dp_packet_set_rss_hash(packet, hash); - } + hash = dp_packet_get_rss_hash(packet); - /* The RSS hash must account for the recirculation depth to avoid - * collisions in the exact match cache */ - recirc_depth = *recirc_depth_get_unsafe(); - if (OVS_UNLIKELY(recirc_depth)) { + if (account_recirc_id) { + /* The RSS hash must account for the recirculation depth to avoid + * collisions in the exact match cache */ + recirc_depth = *recirc_depth_get_unsafe(); hash = hash_finish(hash, recirc_depth); dp_packet_set_rss_hash(packet, hash); } + return hash; } @@ -6396,6 +6404,8 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, bool smc_enable_db; size_t map_cnt = 0; bool batch_enable = true; + bool rss_valid; + bool prefetch_emc = cur_min && EMC_PREFETCH_IN_RANGE(cache); atomic_read_relaxed(&pmd->dp->smc_enable_db, &smc_enable_db); pmd_perf_update_counter(&pmd->perf_stats, @@ -6442,12 +6452,22 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd, } } + rss_valid = dp_packet_rss_valid(packet); + if (rss_valid) { + key->hash = dpif_netdev_packet_get_rss_hash(packet, md_is_valid); + if (prefetch_emc) { + OVS_PREFETCH(&cache->emc_cache.entries[key->hash + & EM_FLOW_HASH_MASK]); + } + } + miniflow_extract(packet, &key->mf); key->len = 0; /* Not computed yet. */ - key->hash = - (md_is_valid == false) - ? dpif_netdev_packet_get_rss_hash_orig_pkt(packet, &key->mf) - : dpif_netdev_packet_get_rss_hash(packet, &key->mf); + + if (!rss_valid) { + key->hash = dpif_netdev_packet_get_5tuple_hash(packet, &key->mf, + md_is_valid); + } /* If EMC is disabled skip emc_lookup */ flow = (cur_min != 0) ? emc_lookup(&cache->emc_cache, key) : NULL;