| Message ID | 1530200511-77186-5-git-send-email-tiago.lam@intel.com |
|---|---|
| State | Superseded |
| Delegated to: | Ian Stokes |
| Headers | show |
| Series | Support multi-segment mbufs | expand |
On 28 Jun 2018, at 17:41, Tiago Lam wrote: > A new mutex, 'nonpmd_mp_mutex', has been introduced to serialise > allocation and free operations by non-pmd threads on a given mempool. > > free_dpdk_buf() has been modified to make use of the introduced mutex. > > Signed-off-by: Tiago Lam <tiago.lam@intel.com> > --- > lib/netdev-dpdk.c | 30 +++++++++++++++++++++++++++--- > 1 file changed, 27 insertions(+), 3 deletions(-) > > diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c > index c5c6aab..5c62d85 100644 > --- a/lib/netdev-dpdk.c > +++ b/lib/netdev-dpdk.c > @@ -294,6 +294,16 @@ static struct ovs_mutex dpdk_mp_mutex > OVS_ACQ_AFTER(dpdk_mutex) > static struct ovs_list dpdk_mp_free_list > OVS_GUARDED_BY(dpdk_mp_mutex) > = OVS_LIST_INITIALIZER(&dpdk_mp_free_list); > > +/* This mutex must be used by non pmd threads when allocating or > freeing > + * mbufs through mempools, when outside of the `non_pmd_mutex` mutex, > in struct > + * dp_netdev. > + * The reason, as pointed out in the "Known Issues" section in DPDK's > EAL docs, > + * is that the implementation on which mempool is based off is > non-preemptable. > + * Since non-pmds may end up not being pinned this could lead to the > preemption > + * between non-pmds performing operations on the same mempool, which > could lead > + * to memory corruption. */ > +static struct ovs_mutex nonpmd_mp_mutex = OVS_MUTEX_INITIALIZER; > + > /* Wrapper for a mempool released but not yet freed. */ > struct dpdk_mp { > struct rte_mempool *mp; > @@ -462,6 +472,8 @@ struct netdev_rxq_dpdk { > dpdk_port_t port_id; > }; > > +static bool dpdk_thread_is_pmd(void); > + > static void netdev_dpdk_destruct(struct netdev *netdev); > static void netdev_dpdk_vhost_destruct(struct netdev *netdev); > > @@ -495,6 +507,12 @@ dpdk_buf_size(int mtu) > NETDEV_DPDK_MBUF_ALIGN); > } > > +static bool > +dpdk_thread_is_pmd(void) > +{ > + return rte_lcore_id() != NON_PMD_CORE_ID; > +} > + > /* Allocates an area of 'sz' bytes from DPDK. The memory is zero'ed. > * > * Unlike xmalloc(), this function can return NULL on failure. */ > @@ -505,11 +523,17 @@ dpdk_rte_mzalloc(size_t sz) > } > > void > -free_dpdk_buf(struct dp_packet *p) > +free_dpdk_buf(struct dp_packet *packet) > { > - struct rte_mbuf *pkt = (struct rte_mbuf *) p; > + if (!dpdk_thread_is_pmd()) { > + ovs_mutex_lock(&nonpmd_mp_mutex); > + } > > - rte_pktmbuf_free(pkt); > + rte_pktmbuf_free((struct rte_mbuf *) packet); Rather than casting it, it should be &packet->mbuf, some one might change the order in the dp_packet structure. > + > + if (!dpdk_thread_is_pmd()) { > + ovs_mutex_unlock(&nonpmd_mp_mutex); > + } > } > > static void > -- > 2.7.4
On 03/07/2018 10:49, Eelco Chaudron wrote: > > > On 28 Jun 2018, at 17:41, Tiago Lam wrote: > >> A new mutex, 'nonpmd_mp_mutex', has been introduced to serialise >> allocation and free operations by non-pmd threads on a given mempool. >> >> free_dpdk_buf() has been modified to make use of the introduced mutex. >> >> Signed-off-by: Tiago Lam <tiago.lam@intel.com> >> --- >> lib/netdev-dpdk.c | 30 +++++++++++++++++++++++++++--- >> 1 file changed, 27 insertions(+), 3 deletions(-) >> >> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c >> index c5c6aab..5c62d85 100644 >> --- a/lib/netdev-dpdk.c >> +++ b/lib/netdev-dpdk.c >> @@ -294,6 +294,16 @@ static struct ovs_mutex dpdk_mp_mutex >> OVS_ACQ_AFTER(dpdk_mutex) >> static struct ovs_list dpdk_mp_free_list >> OVS_GUARDED_BY(dpdk_mp_mutex) >> = OVS_LIST_INITIALIZER(&dpdk_mp_free_list); >> >> +/* This mutex must be used by non pmd threads when allocating or >> freeing >> + * mbufs through mempools, when outside of the `non_pmd_mutex` mutex, >> in struct >> + * dp_netdev. >> + * The reason, as pointed out in the "Known Issues" section in DPDK's >> EAL docs, >> + * is that the implementation on which mempool is based off is >> non-preemptable. >> + * Since non-pmds may end up not being pinned this could lead to the >> preemption >> + * between non-pmds performing operations on the same mempool, which >> could lead >> + * to memory corruption. */ >> +static struct ovs_mutex nonpmd_mp_mutex = OVS_MUTEX_INITIALIZER; >> + >> /* Wrapper for a mempool released but not yet freed. */ >> struct dpdk_mp { >> struct rte_mempool *mp; >> @@ -462,6 +472,8 @@ struct netdev_rxq_dpdk { >> dpdk_port_t port_id; >> }; >> >> +static bool dpdk_thread_is_pmd(void); >> + >> static void netdev_dpdk_destruct(struct netdev *netdev); >> static void netdev_dpdk_vhost_destruct(struct netdev *netdev); >> >> @@ -495,6 +507,12 @@ dpdk_buf_size(int mtu) >> NETDEV_DPDK_MBUF_ALIGN); >> } >> >> +static bool >> +dpdk_thread_is_pmd(void) >> +{ >> + return rte_lcore_id() != NON_PMD_CORE_ID; >> +} >> + >> /* Allocates an area of 'sz' bytes from DPDK. The memory is zero'ed. >> * >> * Unlike xmalloc(), this function can return NULL on failure. */ >> @@ -505,11 +523,17 @@ dpdk_rte_mzalloc(size_t sz) >> } >> >> void >> -free_dpdk_buf(struct dp_packet *p) >> +free_dpdk_buf(struct dp_packet *packet) >> { >> - struct rte_mbuf *pkt = (struct rte_mbuf *) p; >> + if (!dpdk_thread_is_pmd()) { >> + ovs_mutex_lock(&nonpmd_mp_mutex); >> + } >> >> - rte_pktmbuf_free(pkt); >> + rte_pktmbuf_free((struct rte_mbuf *) packet); > > Rather than casting it, it should be &packet->mbuf, some one might > change the order in the dp_packet structure. > Missed this one. I'll do this for v2, thanks. Tiago.
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index c5c6aab..5c62d85 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -294,6 +294,16 @@ static struct ovs_mutex dpdk_mp_mutex OVS_ACQ_AFTER(dpdk_mutex) static struct ovs_list dpdk_mp_free_list OVS_GUARDED_BY(dpdk_mp_mutex) = OVS_LIST_INITIALIZER(&dpdk_mp_free_list); +/* This mutex must be used by non pmd threads when allocating or freeing + * mbufs through mempools, when outside of the `non_pmd_mutex` mutex, in struct + * dp_netdev. + * The reason, as pointed out in the "Known Issues" section in DPDK's EAL docs, + * is that the implementation on which mempool is based off is non-preemptable. + * Since non-pmds may end up not being pinned this could lead to the preemption + * between non-pmds performing operations on the same mempool, which could lead + * to memory corruption. */ +static struct ovs_mutex nonpmd_mp_mutex = OVS_MUTEX_INITIALIZER; + /* Wrapper for a mempool released but not yet freed. */ struct dpdk_mp { struct rte_mempool *mp; @@ -462,6 +472,8 @@ struct netdev_rxq_dpdk { dpdk_port_t port_id; }; +static bool dpdk_thread_is_pmd(void); + static void netdev_dpdk_destruct(struct netdev *netdev); static void netdev_dpdk_vhost_destruct(struct netdev *netdev); @@ -495,6 +507,12 @@ dpdk_buf_size(int mtu) NETDEV_DPDK_MBUF_ALIGN); } +static bool +dpdk_thread_is_pmd(void) +{ + return rte_lcore_id() != NON_PMD_CORE_ID; +} + /* Allocates an area of 'sz' bytes from DPDK. The memory is zero'ed. * * Unlike xmalloc(), this function can return NULL on failure. */ @@ -505,11 +523,17 @@ dpdk_rte_mzalloc(size_t sz) } void -free_dpdk_buf(struct dp_packet *p) +free_dpdk_buf(struct dp_packet *packet) { - struct rte_mbuf *pkt = (struct rte_mbuf *) p; + if (!dpdk_thread_is_pmd()) { + ovs_mutex_lock(&nonpmd_mp_mutex); + } - rte_pktmbuf_free(pkt); + rte_pktmbuf_free((struct rte_mbuf *) packet); + + if (!dpdk_thread_is_pmd()) { + ovs_mutex_unlock(&nonpmd_mp_mutex); + } } static void
A new mutex, 'nonpmd_mp_mutex', has been introduced to serialise allocation and free operations by non-pmd threads on a given mempool. free_dpdk_buf() has been modified to make use of the introduced mutex. Signed-off-by: Tiago Lam <tiago.lam@intel.com> --- lib/netdev-dpdk.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-)