[ovs-dev,RFC,v7,08/13] netdev-dpdk: Serialise non-pmds mbufs' alloc/free.

Message ID 1527094048-105084-9-git-send-email-tiago.lam@intel.com
State Superseded
Headers show
Series
  • Support multi-segment mbufs
Related show

Commit Message

Tiago Lam May 23, 2018, 4:47 p.m.
A new mutex, 'nonpmd_mp_mutex', has been introduced to serialise
allocation and free operations by non-pmd threads on a given mempool.
This mutex is wrapped by dpdk_buf_alloc() and free_dpdk_buf(), which are
used by dp_packet_resize__() in a lter commit to allocate or free mbufs,
as needed.

Signed-off-by: Tiago Lam <tiago.lam@intel.com>
---
Note that another version of this mutex was removed in commit 1166b0d82
("netdev-dpdk: Remove useless nonpmd_mempool_mutex."), since "non-pmd
threads access to netdev is already serialized with "non_pmd_mutex" in
dpif-netdev".  However, it needs to be checked if operations that result
in calls to dp_packet_resize__() (and thus allocate and free mbufs from
the mempool), can happen outside of dpif_netdev_execute(). Otherwise
this commit is indeed "useless".

 lib/netdev-dpdk.c | 37 ++++++++++++++++++++++++++++++++++++-
 lib/netdev-dpdk.h |  3 +++
 2 files changed, 39 insertions(+), 1 deletion(-)

Patch

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 8f44691..f8caca8 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -294,6 +294,10 @@  static struct ovs_mutex dpdk_mp_mutex OVS_ACQ_AFTER(dpdk_mutex)
 static struct ovs_list dpdk_mp_free_list OVS_GUARDED_BY(dpdk_mp_mutex)
     = OVS_LIST_INITIALIZER(&dpdk_mp_free_list);
 
+/* This mutex must be used by non pmd threads when allocating or freeing
+ * mbufs through mempools. */
+static struct ovs_mutex nonpmd_mp_mutex = OVS_MUTEX_INITIALIZER;
+
 /* Wrapper for a mempool released but not yet freed. */
 struct dpdk_mp {
      struct rte_mempool *mp;
@@ -459,6 +463,8 @@  struct netdev_rxq_dpdk {
     dpdk_port_t port_id;
 };
 
+static bool dpdk_thread_is_pmd(void);
+
 static void netdev_dpdk_destruct(struct netdev *netdev);
 static void netdev_dpdk_vhost_destruct(struct netdev *netdev);
 
@@ -492,6 +498,12 @@  dpdk_buf_size(int mtu)
                      NETDEV_DPDK_MBUF_ALIGN);
 }
 
+static bool
+dpdk_thread_is_pmd(void)
+{
+     return rte_lcore_id() != NON_PMD_CORE_ID;
+}
+
 /* Allocates an area of 'sz' bytes from DPDK.  The memory is zero'ed.
  *
  * Unlike xmalloc(), this function can return NULL on failure. */
@@ -501,12 +513,35 @@  dpdk_rte_mzalloc(size_t sz)
     return rte_zmalloc(OVS_VPORT_DPDK, sz, OVS_CACHE_LINE_SIZE);
 }
 
+struct dp_packet *
+dpdk_buf_alloc(struct rte_mempool *mp)
+{
+    if (!dpdk_thread_is_pmd()) {
+        ovs_mutex_lock(&nonpmd_mp_mutex);
+    }
+
+    struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mp);
+
+    if (!dpdk_thread_is_pmd()) {
+        ovs_mutex_unlock(&nonpmd_mp_mutex);
+    }
+
+    return (struct dp_packet *) mbuf;
+}
+
 void
 free_dpdk_buf(struct dp_packet *p)
 {
-    struct rte_mbuf *pkt = (struct rte_mbuf *) p;
+    if (!dpdk_thread_is_pmd()) {
+        ovs_mutex_lock(&nonpmd_mp_mutex);
+    }
 
+    struct rte_mbuf *pkt = (struct rte_mbuf *) p;
     rte_pktmbuf_free(pkt);
+
+    if (!dpdk_thread_is_pmd()) {
+        ovs_mutex_unlock(&nonpmd_mp_mutex);
+    }
 }
 
 static void
diff --git a/lib/netdev-dpdk.h b/lib/netdev-dpdk.h
index b7d02a7..e627553 100644
--- a/lib/netdev-dpdk.h
+++ b/lib/netdev-dpdk.h
@@ -24,8 +24,11 @@ 
 struct dp_packet;
 
 #ifdef DPDK_NETDEV
+struct rte_mempool;
 
 void netdev_dpdk_register(void);
+struct dp_packet *
+dpdk_buf_alloc(struct rte_mempool *mp);
 void free_dpdk_buf(struct dp_packet *);
 
 #else