| Message ID | 7bcb3150fa36e23de78d725bd91718c344ed896f.1589605823.git.gmuthukr@redhat.com |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | system-dpdk: add userspace-tso tests | expand |
On Sat, May 16, 2020 at 11:23:52AM +0530, Gowrishankar Muthukrishnan wrote: > Today we need 1GB from hugepages for running dpdk tests (i.e > 512MB for ovs-vswitchd including phy ports and 512MB for testpmd > app). This patch optimize the usage as: > - 1GB for dpdk tests including phy ports, vhu ports and testpmd > - 512MB for dpdk tests including vhu ports and testpmd > > Signed-off-by: Gowrishankar Muthukrishnan <gmuthukr@redhat.com> > --- > v2: > - fixed hunk to reflect intended changes. > > --- > tests/system-dpdk-macros.at | 19 +++++++++++++++++++ > tests/system-dpdk.at | 10 ++++++---- > 2 files changed, 25 insertions(+), 4 deletions(-) > > diff --git a/tests/system-dpdk-macros.at b/tests/system-dpdk-macros.at > index d3a3aea..e5ac4f4 100644 > --- a/tests/system-dpdk-macros.at > +++ b/tests/system-dpdk-macros.at > @@ -1,3 +1,18 @@ > +# OVS_DPDK_SET_SOCKET_MEM() > +# > +# Set optimum memory required for tests. > +# > +m4_define([OVS_DPDK_SET_SOCKET_MEM], > + [ > + AT_CHECK([$DPDK_DIR/usertools/dpdk-devbind.py -s],[],[stdout]) > + AT_CHECK([sed -n '/DPDK/,/kernel/{//!p;}' stdout|sed '/^=/d'],[],[stdout]) > + AT_CHECK([egrep -c '^....:..:..\..' stdout],[ignore],[stdout]) > + AS_IF([test $(cat stdout) -eq 0],[echo 64 > SOCKET_MEM], > + [echo $((64+512)) > SOCKET_MEM]) > + > +]) > + > + > # OVS_DPDK_PRE_CHECK() > # > # Check prerequisites for DPDK tests. Following settings are checked: > @@ -57,6 +72,10 @@ m4_define([OVS_DPDK_START], > > dnl Enable DPDK functionality > AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true]) > + OVS_DPDK_SET_SOCKET_MEM() > + AT_CHECK([lscpu], [], [stdout]) > + AT_CHECK([cat stdout | grep "NUMA node(s)" | awk -v m=$(cat SOCKET_MEM) '{c=1; while (c++<$(3)) {printf m","}; print m}' > NUMA_NODE]) > + AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$(cat NUMA_NODE)]) > > dnl Start ovs-vswitchd. > AT_CHECK([ovs-vswitchd --detach --no-chdir --pidfile --log-file -vvconn -vofproto_dpif -vunixctl], [0], [stdout], [stderr]) > diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at > index 01ac970..b032c5b 100644 > --- a/tests/system-dpdk.at > +++ b/tests/system-dpdk.at > @@ -93,7 +93,7 @@ OVS_DPDK_START() > > dnl Find number of sockets > AT_CHECK([lscpu], [], [stdout]) > -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE]) Why 448? > dnl Add userspace bridge and attach it to OVS > AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > @@ -118,7 +118,8 @@ ADD_VETH(tap1, ns2, br10, "172.31.110.12/24") > > dnl Execute testpmd in background > on_exit "pkill -f -x -9 'tail -f /dev/null'" > -tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ > +tail -f /dev/null | testpmd --no-pci \ > + --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \ > --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostuser0" \ > --vdev="net_tap0,iface=tap0" --file-prefix page0 \ > --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuser0.log 2>&1 & > @@ -172,7 +173,7 @@ OVS_DPDK_START() > > dnl Find number of sockets > AT_CHECK([lscpu], [], [stdout]) > -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE]) > > dnl Add userspace bridge and attach it to OVS > AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > @@ -196,7 +197,8 @@ ADD_VETH(tap1, ns2, br10, "172.31.110.12/24") > > dnl Execute testpmd in background > on_exit "pkill -f -x -9 'tail -f /dev/null'" > -tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ > +tail -f /dev/null | testpmd --no-pci \ > + --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \ > --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostclient0,server=1" \ > --vdev="net_tap0,iface=tap0" --file-prefix page0 \ > --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuserclient0.log 2>&1 & > -- > 1.8.3.1 > > _______________________________________________ > dev mailing list > dev@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev
> > > dnl Find number of sockets > > AT_CHECK([lscpu], [], [stdout]) > > -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while > (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) > > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while > (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE]) > > Why 448? > > When ovs dpdk is run with dpdk compatible drivers, its socket mem is set to (64MB + 512MB) per socket, otherwise only 64MB (for eg using dpdk virt devices) as far as the current tests require. Testpmd app would need around ~338 MB (n_mbuf=155456, mbuf_size=2176) in its mempool, we would need in total: 64 MB (+ 512 MB) + 338 MB. I just rounded up the requirement for testpmd to 448 MB so that, we either configure 512 MB or 1GB in hugepage per socket. Thanks.
On Tue, May 19, 2020 at 12:23:40AM +0530, Gowrishankar Muthukrishnan wrote: > > > > > dnl Find number of sockets > > > AT_CHECK([lscpu], [], [stdout]) > > > -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while > > (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) > > > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while > > (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE]) > > > > Why 448? > > > > When ovs dpdk is run with dpdk compatible drivers, its socket mem is set > to (64MB + 512MB) per socket, otherwise only 64MB > (for eg using dpdk virt devices) as far as the current tests require. > > Testpmd app would need around ~338 MB (n_mbuf=155456, mbuf_size=2176) in > its mempool, we would need in total: > 64 MB (+ 512 MB) + 338 MB. I just rounded up the requirement for testpmd to > 448 MB so that, we either configure 512 MB > or 1GB in hugepage per socket. Can you add a small comment saying that?
On 5/16/20 7:53 AM, Gowrishankar Muthukrishnan wrote: > Today we need 1GB from hugepages for running dpdk tests (i.e > 512MB for ovs-vswitchd including phy ports and 512MB for testpmd > app). This patch optimize the usage as: > - 1GB for dpdk tests including phy ports, vhu ports and testpmd > - 512MB for dpdk tests including vhu ports and testpmd I think, we need to just drop all the memory configurations for OVS and testpmd. Modern DPDK memory allocator will allocate as much memory as we need dynamically. Configuration only complicates things. Best regards, Ilya Maximets.
diff --git a/tests/system-dpdk-macros.at b/tests/system-dpdk-macros.at index d3a3aea..e5ac4f4 100644 --- a/tests/system-dpdk-macros.at +++ b/tests/system-dpdk-macros.at @@ -1,3 +1,18 @@ +# OVS_DPDK_SET_SOCKET_MEM() +# +# Set optimum memory required for tests. +# +m4_define([OVS_DPDK_SET_SOCKET_MEM], + [ + AT_CHECK([$DPDK_DIR/usertools/dpdk-devbind.py -s],[],[stdout]) + AT_CHECK([sed -n '/DPDK/,/kernel/{//!p;}' stdout|sed '/^=/d'],[],[stdout]) + AT_CHECK([egrep -c '^....:..:..\..' stdout],[ignore],[stdout]) + AS_IF([test $(cat stdout) -eq 0],[echo 64 > SOCKET_MEM], + [echo $((64+512)) > SOCKET_MEM]) + +]) + + # OVS_DPDK_PRE_CHECK() # # Check prerequisites for DPDK tests. Following settings are checked: @@ -57,6 +72,10 @@ m4_define([OVS_DPDK_START], dnl Enable DPDK functionality AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true]) + OVS_DPDK_SET_SOCKET_MEM() + AT_CHECK([lscpu], [], [stdout]) + AT_CHECK([cat stdout | grep "NUMA node(s)" | awk -v m=$(cat SOCKET_MEM) '{c=1; while (c++<$(3)) {printf m","}; print m}' > NUMA_NODE]) + AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$(cat NUMA_NODE)]) dnl Start ovs-vswitchd. AT_CHECK([ovs-vswitchd --detach --no-chdir --pidfile --log-file -vvconn -vofproto_dpif -vunixctl], [0], [stdout], [stderr]) diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at index 01ac970..b032c5b 100644 --- a/tests/system-dpdk.at +++ b/tests/system-dpdk.at @@ -93,7 +93,7 @@ OVS_DPDK_START() dnl Find number of sockets AT_CHECK([lscpu], [], [stdout]) -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE]) dnl Add userspace bridge and attach it to OVS AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) @@ -118,7 +118,8 @@ ADD_VETH(tap1, ns2, br10, "172.31.110.12/24") dnl Execute testpmd in background on_exit "pkill -f -x -9 'tail -f /dev/null'" -tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ +tail -f /dev/null | testpmd --no-pci \ + --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \ --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostuser0" \ --vdev="net_tap0,iface=tap0" --file-prefix page0 \ --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuser0.log 2>&1 & @@ -172,7 +173,7 @@ OVS_DPDK_START() dnl Find number of sockets AT_CHECK([lscpu], [], [stdout]) -AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "448,"}; print "448"}' > NUMA_NODE]) dnl Add userspace bridge and attach it to OVS AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) @@ -196,7 +197,8 @@ ADD_VETH(tap1, ns2, br10, "172.31.110.12/24") dnl Execute testpmd in background on_exit "pkill -f -x -9 'tail -f /dev/null'" -tail -f /dev/null | testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ +tail -f /dev/null | testpmd --no-pci \ + --socket-mem="$(cat NUMA_NODE)" --socket-limit="$(cat NUMA_NODE)" \ --vdev="net_virtio_user,path=$OVS_RUNDIR/dpdkvhostclient0,server=1" \ --vdev="net_tap0,iface=tap0" --file-prefix page0 \ --single-file-segments -- -a >$OVS_RUNDIR/testpmd-dpdkvhostuserclient0.log 2>&1 &
Today we need 1GB from hugepages for running dpdk tests (i.e 512MB for ovs-vswitchd including phy ports and 512MB for testpmd app). This patch optimize the usage as: - 1GB for dpdk tests including phy ports, vhu ports and testpmd - 512MB for dpdk tests including vhu ports and testpmd Signed-off-by: Gowrishankar Muthukrishnan <gmuthukr@redhat.com> --- v2: - fixed hunk to reflect intended changes. --- tests/system-dpdk-macros.at | 19 +++++++++++++++++++ tests/system-dpdk.at | 10 ++++++---- 2 files changed, 25 insertions(+), 4 deletions(-)