diff mbox series

[ovs-dev,v2,2/3] system-dpdk: use optimum hugepages for dpdk tests

Message ID 7bcb3150fa36e23de78d725bd91718c344ed896f.1589605823.git.gmuthukr@redhat.com
State New
Headers show
Series system-dpdk: add userspace-tso tests | expand

Commit Message

Gowrishankar Muthukrishnan May 16, 2020, 5:53 a.m. UTC
Today we need 1GB from hugepages for running dpdk tests (i.e
512MB for ovs-vswitchd including phy ports and 512MB for testpmd
app). This patch optimize the usage as:
  - 1GB for dpdk tests including phy ports, vhu ports and testpmd
  - 512MB for dpdk tests including vhu ports and testpmd

Signed-off-by: Gowrishankar Muthukrishnan <gmuthukr@redhat.com>
---
v2:
 - fixed hunk to reflect intended changes.

---
 tests/system-dpdk-macros.at | 19 +++++++++++++++++++
 tests/system-dpdk.at        | 10 ++++++----
 2 files changed, 25 insertions(+), 4 deletions(-)

Comments

Flavio Leitner May 18, 2020, 6:25 p.m. UTC | #1
On Sat, May 16, 2020 at 11:23:52AM +0530, Gowrishankar Muthukrishnan wrote:
> Today we need 1GB from hugepages for running dpdk tests (i.e
> 512MB for ovs-vswitchd including phy ports and 512MB for testpmd
> app). This patch optimize the usage as:
>   - 1GB for dpdk tests including phy ports, vhu ports and testpmd
>   - 512MB for dpdk tests including vhu ports and testpmd
> 
> Signed-off-by: Gowrishankar Muthukrishnan <gmuthukr@redhat.com>
> ---
> v2:
>  - fixed hunk to reflect intended changes.
> 
> ---
>  tests/system-dpdk-macros.at | 19 +++++++++++++++++++
>  tests/system-dpdk.at        | 10 ++++++----
>  2 files changed, 25 insertions(+), 4 deletions(-)
> 
> diff --git a/tests/system-dpdk-macros.at b/tests/system-dpdk-macros.at
> index d3a3aea..e5ac4f4 100644
> --- a/tests/system-dpdk-macros.at
> +++ b/tests/system-dpdk-macros.at
> @@ -1,3 +1,18 @@
> +# OVS_DPDK_SET_SOCKET_MEM()
> +#
> +# Set optimum memory required for tests.
> +#
> +m4_define([OVS_DPDK_SET_SOCKET_MEM],
> +  [
> +   AT_CHECK([$DPDK_DIR/usertools/dpdk-devbind.py -s],[],[stdout])
> +   AT_CHECK([sed -n '/DPDK/,/kernel/{//!p;}' stdout|sed '/^=/d'],[],[stdout])
> +   AT_CHECK([egrep -c '^....:..:..\..' stdout],[ignore],[stdout])
> +   AS_IF([test $(cat stdout) -eq 0],[echo 64 > SOCKET_MEM],
> +         [echo $((64+512)) > SOCKET_MEM])
> +
> +])
> +
> +
>  # OVS_DPDK_PRE_CHECK()
>  #
>  # Check prerequisites for DPDK tests. Following settings are checked:
> @@ -57,6 +72,10 @@ m4_define([OVS_DPDK_START],
>  
>     dnl Enable DPDK functionality
>     AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true])
> +   OVS_DPDK_SET_SOCKET_MEM()
> +   AT_CHECK([lscpu], [], [stdout])
> +   AT_CHECK([cat stdout | grep "NUMA node(s)" | awk -v m=$(cat SOCKET_MEM) '{c=1; while (c++<$(3)) {printf m","}; print m}' > NUMA_NODE])
> +   AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$(cat NUMA_NODE)])
>  
>     dnl Start ovs-vswitchd.
>     AT_CHECK([ovs-vswitchd --detach --no-chdir --pidfile --log-file -vvconn -vofproto_dpif -vunixctl], [0], [stdout], [stderr])
> diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at
> index 01ac970..b032c5b 100644
> --- a/tests/system-dpdk.at
> +++ b/tests/system-dpdk.at
> @@ -93,7 +93,7 @@ OVS_DPDK_START()
>  
>  dnl Find number of sockets
>  AT_CHECK([lscpu], [], [stdout])
> -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE])
> +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE])

Why 448?


>  dnl Add userspace bridge and attach it to OVS
>  AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev])
> @@ -118,7 +118,8 @@ ADD_VETH(tap1, ns2, br10, "172.31.110.12/24")
>  
>  dnl Execute testpmd in background
>  on_exit "pkill -f -x -9 'tail -f /dev/null'"
> -tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\
> +tail -f /dev/null | testpmd --no-pci \
> +           --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \
>             --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostuser0" \
>             --vdev="net_tap0,iface=tap0" --file-prefix page0 \
>             --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuser0.log 2>&1 &
> @@ -172,7 +173,7 @@ OVS_DPDK_START()
>  
>  dnl Find number of sockets
>  AT_CHECK([lscpu], [], [stdout])
> -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE])
> +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE])
>  
>  dnl Add userspace bridge and attach it to OVS
>  AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev])
> @@ -196,7 +197,8 @@ ADD_VETH(tap1, ns2, br10, "172.31.110.12/24")
>  
>  dnl Execute testpmd in background
>  on_exit "pkill -f -x -9 'tail -f /dev/null'"
> -tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\
> +tail -f /dev/null | testpmd --no-pci \
> +           --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \
>             --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostclient0,server=1" \
>             --vdev="net_tap0,iface=tap0" --file-prefix page0 \
>             --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuserclient0.log 2>&1 &
> -- 
> 1.8.3.1
> 
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Gowrishankar Muthukrishnan May 18, 2020, 6:53 p.m. UTC | #2
>
> >  dnl Find number of sockets
> >  AT_CHECK([lscpu], [], [stdout])
> > -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while
> (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE])
> > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while
> (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE])
>
> Why 448?
>
> When ovs dpdk is run with dpdk compatible drivers, its socket mem is set
to (64MB + 512MB) per socket, otherwise only 64MB
(for eg using dpdk virt devices) as far as the current tests require.

Testpmd app would need around ~338 MB (n_mbuf=155456, mbuf_size=2176) in
its mempool, we would need in total:
64 MB (+ 512 MB) + 338 MB. I just rounded up the requirement for testpmd to
448 MB so that, we either configure 512 MB
or 1GB in hugepage per socket.

Thanks.
Flavio Leitner May 21, 2020, 4:33 a.m. UTC | #3
On Tue, May 19, 2020 at 12:23:40AM +0530, Gowrishankar Muthukrishnan wrote:
> >
> > >  dnl Find number of sockets
> > >  AT_CHECK([lscpu], [], [stdout])
> > > -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while
> > (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE])
> > > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while
> > (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE])
> >
> > Why 448?
> >
> > When ovs dpdk is run with dpdk compatible drivers, its socket mem is set
> to (64MB + 512MB) per socket, otherwise only 64MB
> (for eg using dpdk virt devices) as far as the current tests require.
> 
> Testpmd app would need around ~338 MB (n_mbuf=155456, mbuf_size=2176) in
> its mempool, we would need in total:
> 64 MB (+ 512 MB) + 338 MB. I just rounded up the requirement for testpmd to
> 448 MB so that, we either configure 512 MB
> or 1GB in hugepage per socket.

Can you add a small comment saying that?
Ilya Maximets May 21, 2020, 2:27 p.m. UTC | #4
On 5/16/20 7:53 AM, Gowrishankar Muthukrishnan wrote:
> Today we need 1GB from hugepages for running dpdk tests (i.e
> 512MB for ovs-vswitchd including phy ports and 512MB for testpmd
> app). This patch optimize the usage as:
>   - 1GB for dpdk tests including phy ports, vhu ports and testpmd
>   - 512MB for dpdk tests including vhu ports and testpmd

I think, we need to just drop all the memory configurations for OVS and
testpmd.  Modern DPDK memory allocator will allocate as much memory as
we need dynamically.  Configuration only complicates things.

Best regards, Ilya Maximets.
diff mbox series

Patch

diff --git a/tests/system-dpdk-macros.at b/tests/system-dpdk-macros.at
index d3a3aea..e5ac4f4 100644
--- a/tests/system-dpdk-macros.at
+++ b/tests/system-dpdk-macros.at
@@ -1,3 +1,18 @@ 
+# OVS_DPDK_SET_SOCKET_MEM()
+#
+# Set optimum memory required for tests.
+#
+m4_define([OVS_DPDK_SET_SOCKET_MEM],
+  [
+   AT_CHECK([$DPDK_DIR/usertools/dpdk-devbind.py -s],[],[stdout])
+   AT_CHECK([sed -n '/DPDK/,/kernel/{//!p;}' stdout|sed '/^=/d'],[],[stdout])
+   AT_CHECK([egrep -c '^....:..:..\..' stdout],[ignore],[stdout])
+   AS_IF([test $(cat stdout) -eq 0],[echo 64 > SOCKET_MEM],
+         [echo $((64+512)) > SOCKET_MEM])
+
+])
+
+
 # OVS_DPDK_PRE_CHECK()
 #
 # Check prerequisites for DPDK tests. Following settings are checked:
@@ -57,6 +72,10 @@  m4_define([OVS_DPDK_START],
 
    dnl Enable DPDK functionality
    AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true])
+   OVS_DPDK_SET_SOCKET_MEM()
+   AT_CHECK([lscpu], [], [stdout])
+   AT_CHECK([cat stdout | grep "NUMA node(s)" | awk -v m=$(cat SOCKET_MEM) '{c=1; while (c++<$(3)) {printf m","}; print m}' > NUMA_NODE])
+   AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$(cat NUMA_NODE)])
 
    dnl Start ovs-vswitchd.
    AT_CHECK([ovs-vswitchd --detach --no-chdir --pidfile --log-file -vvconn -vofproto_dpif -vunixctl], [0], [stdout], [stderr])
diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at
index 01ac970..b032c5b 100644
--- a/tests/system-dpdk.at
+++ b/tests/system-dpdk.at
@@ -93,7 +93,7 @@  OVS_DPDK_START()
 
 dnl Find number of sockets
 AT_CHECK([lscpu], [], [stdout])
-AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE])
+AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE])
 
 dnl Add userspace bridge and attach it to OVS
 AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev])
@@ -118,7 +118,8 @@  ADD_VETH(tap1, ns2, br10, "172.31.110.12/24")
 
 dnl Execute testpmd in background
 on_exit "pkill -f -x -9 'tail -f /dev/null'"
-tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\
+tail -f /dev/null | testpmd --no-pci \
+           --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \
            --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostuser0" \
            --vdev="net_tap0,iface=tap0" --file-prefix page0 \
            --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuser0.log 2>&1 &
@@ -172,7 +173,7 @@  OVS_DPDK_START()
 
 dnl Find number of sockets
 AT_CHECK([lscpu], [], [stdout])
-AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE])
+AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE])
 
 dnl Add userspace bridge and attach it to OVS
 AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev])
@@ -196,7 +197,8 @@  ADD_VETH(tap1, ns2, br10, "172.31.110.12/24")
 
 dnl Execute testpmd in background
 on_exit "pkill -f -x -9 'tail -f /dev/null'"
-tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\
+tail -f /dev/null | testpmd --no-pci \
+           --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \
            --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostclient0,server=1" \
            --vdev="net_tap0,iface=tap0" --file-prefix page0 \
            --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuserclient0.log 2>&1 &