[ovs-dev,v8,04/13] netdev-dpdk: Serialise non-pmds mbufs' alloc/free.

Message ID 1528734090-220990-5-git-send-email-tiago.lam@intel.com
State New
Headers show
Series
  • Support multi-segment mbufs
Related show

Commit Message

Lam, Tiago June 11, 2018, 4:21 p.m.
A new mutex, 'nonpmd_mp_mutex', has been introduced to serialise
allocation and free operations by non-pmd threads on a given mempool.

free_dpdk_buf() has been modified to make use of the introduced mutex.

Signed-off-by: Tiago Lam <tiago.lam@intel.com>
---
 lib/netdev-dpdk.c | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

Comments

Eelco Chaudron June 18, 2018, 11:28 a.m. | #1
On 11 Jun 2018, at 18:21, Tiago Lam wrote:

> A new mutex, 'nonpmd_mp_mutex', has been introduced to serialise
> allocation and free operations by non-pmd threads on a given mempool.
>

Can you explain why we need the mutex here? Can't see any reason why 
rte_pktmbuf_free() needs to be protected for non-pmd threads?

> free_dpdk_buf() has been modified to make use of the introduced mutex.
>
> Signed-off-by: Tiago Lam <tiago.lam@intel.com>
> ---
>  lib/netdev-dpdk.c | 21 ++++++++++++++++++++-
>  1 file changed, 20 insertions(+), 1 deletion(-)
>
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index f546507..efd7c20 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -294,6 +294,10 @@ static struct ovs_mutex dpdk_mp_mutex 
> OVS_ACQ_AFTER(dpdk_mutex)
>  static struct ovs_list dpdk_mp_free_list 
> OVS_GUARDED_BY(dpdk_mp_mutex)
>      = OVS_LIST_INITIALIZER(&dpdk_mp_free_list);
>
> +/* This mutex must be used by non pmd threads when allocating or 
> freeing
> + * mbufs through mempools. */
> +static struct ovs_mutex nonpmd_mp_mutex = OVS_MUTEX_INITIALIZER;
> +
>  /* Wrapper for a mempool released but not yet freed. */
>  struct dpdk_mp {
>       struct rte_mempool *mp;
> @@ -461,6 +465,8 @@ struct netdev_rxq_dpdk {
>      dpdk_port_t port_id;
>  };
>
> +static bool dpdk_thread_is_pmd(void);
> +
>  static void netdev_dpdk_destruct(struct netdev *netdev);
>  static void netdev_dpdk_vhost_destruct(struct netdev *netdev);
>
> @@ -494,6 +500,12 @@ dpdk_buf_size(int mtu)
>                       NETDEV_DPDK_MBUF_ALIGN);
>  }
>
> +static bool
> +dpdk_thread_is_pmd(void)
> +{
> +     return rte_lcore_id() != NON_PMD_CORE_ID;

Will this continue to work with newer DPDK versions? I've not looked at 
it in detail, but I did notice that the vhost threads in the newer DPDK 
now get created with rte_ctrl_thread_create() and does some lcore 
mangling.

> +}
> +
>  /* Allocates an area of 'sz' bytes from DPDK.  The memory is zero'ed.
>   *
>   * Unlike xmalloc(), this function can return NULL on failure. */
> @@ -506,9 +518,16 @@ dpdk_rte_mzalloc(size_t sz)
>  void
>  free_dpdk_buf(struct dp_packet *p)
>  {
> -    struct rte_mbuf *pkt = (struct rte_mbuf *) p;
> +    if (!dpdk_thread_is_pmd()) {
> +        ovs_mutex_lock(&nonpmd_mp_mutex);
> +    }
>
> +    struct rte_mbuf *pkt = (struct rte_mbuf *) p;

Should we not use a container_of macro here?

>      rte_pktmbuf_free(pkt);
> +
> +    if (!dpdk_thread_is_pmd()) {
> +        ovs_mutex_unlock(&nonpmd_mp_mutex);
> +    }
>  }
>
>  static void
> -- 
> 2.7.4

Patch

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index f546507..efd7c20 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -294,6 +294,10 @@  static struct ovs_mutex dpdk_mp_mutex OVS_ACQ_AFTER(dpdk_mutex)
 static struct ovs_list dpdk_mp_free_list OVS_GUARDED_BY(dpdk_mp_mutex)
     = OVS_LIST_INITIALIZER(&dpdk_mp_free_list);
 
+/* This mutex must be used by non pmd threads when allocating or freeing
+ * mbufs through mempools. */
+static struct ovs_mutex nonpmd_mp_mutex = OVS_MUTEX_INITIALIZER;
+
 /* Wrapper for a mempool released but not yet freed. */
 struct dpdk_mp {
      struct rte_mempool *mp;
@@ -461,6 +465,8 @@  struct netdev_rxq_dpdk {
     dpdk_port_t port_id;
 };
 
+static bool dpdk_thread_is_pmd(void);
+
 static void netdev_dpdk_destruct(struct netdev *netdev);
 static void netdev_dpdk_vhost_destruct(struct netdev *netdev);
 
@@ -494,6 +500,12 @@  dpdk_buf_size(int mtu)
                      NETDEV_DPDK_MBUF_ALIGN);
 }
 
+static bool
+dpdk_thread_is_pmd(void)
+{
+     return rte_lcore_id() != NON_PMD_CORE_ID;
+}
+
 /* Allocates an area of 'sz' bytes from DPDK.  The memory is zero'ed.
  *
  * Unlike xmalloc(), this function can return NULL on failure. */
@@ -506,9 +518,16 @@  dpdk_rte_mzalloc(size_t sz)
 void
 free_dpdk_buf(struct dp_packet *p)
 {
-    struct rte_mbuf *pkt = (struct rte_mbuf *) p;
+    if (!dpdk_thread_is_pmd()) {
+        ovs_mutex_lock(&nonpmd_mp_mutex);
+    }
 
+    struct rte_mbuf *pkt = (struct rte_mbuf *) p;
     rte_pktmbuf_free(pkt);
+
+    if (!dpdk_thread_is_pmd()) {
+        ovs_mutex_unlock(&nonpmd_mp_mutex);
+    }
 }
 
 static void