| Message ID | 1564697253-37992-10-git-send-email-yihung.wei@gmail.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Support zone-based conntrack timeout policy | expand |
Thanks for the patch I see the test is much improved now from V1 and passes - thanks Ideally, tests should be associated with some code for context It could be folded into patch 8 On Thu, Aug 1, 2019 at 3:12 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > This patch adds a system traffic test to verify the zone-based conntrack > timeout feature. The test uses ovs-vsctl commands to configure > the customized ICMP and UDP timeout on zone 5 to a shorter period. > It then injects ICMP and UDP traffic to conntrack, and checks if the > corresponding conntrack entry expires after the predefined timeout. > > Signed-off-by: Yi-Hung Wei <yihung.wei@gmail.com> > --- > tests/system-kmod-macros.at | 25 +++++++++++++++ > tests/system-traffic.at | 66 > ++++++++++++++++++++++++++++++++++++++++ > tests/system-userspace-macros.at | 26 ++++++++++++++++ > 3 files changed, 117 insertions(+) > > diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at > index 554a61e9bd95..1bc6f246f426 100644 > --- a/tests/system-kmod-macros.at > +++ b/tests/system-kmod-macros.at > @@ -100,6 +100,15 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP], > # > m4_define([CHECK_CONNTRACK_NAT]) > > +# CHECK_CONNTRACK_TIMEOUT() > +# > +# Perform requirements checks for running conntrack customized timeout > tests. > +# > +m4_define([CHECK_CONNTRACK_TIMEOUT], > +[ > + AT_SKIP_IF([! cat /boot/config-$(uname -r) | grep > NF_CONNTRACK_TIMEOUT | grep '=y' > /dev/null]) > +]) > + > # CHECK_CT_DPIF_PER_ZONE_LIMIT() > # > # Perform requirements checks for running ovs-dpctl > ct-[set|get|del]-limits per > @@ -185,3 +194,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL], > sublevel=$(uname -r | sed -e 's/\./ /g' | awk '{print $ 2}') > AT_SKIP_IF([ ! ( test $version -lt $1 || ( test $version -eq $1 && > test $sublevel -lt $2 ) || test $version -gt $3 || ( test $version -eq $3 > && test $sublevel -gt $4 ) ) ]) > ]) > + > +# VSCTL_ADD_DATAPATH_TABLE() > +# > +# Create system datapath table "system" for kernel tests in ovsdb > +m4_define([VSCTL_ADD_DATAPATH_TABLE], > +[ > + AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- > set Open_vSwitch . datapaths:"system"=@m], [0], [stdout]) > +]) > + > +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters]) > +# > +# Add zone based timeout policy to kernel datapath > +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY], > +[ > + AT_CHECK([ovs-vsctl add-zone-tp system $1], [0], [stdout]) > +]) > diff --git a/tests/system-traffic.at b/tests/system-traffic.at > index 1a04199dcfe9..f4ac8a8f2c06 100644 > --- a/tests/system-traffic.at > +++ b/tests/system-traffic.at > @@ -3179,6 +3179,72 @@ NXST_FLOW reply: > OVS_TRAFFIC_VSWITCHD_STOP > AT_CLEANUP > > +AT_SETUP([conntrack - zone-based timeout policy]) > +CHECK_CONNTRACK() > +CHECK_CONNTRACK_TIMEOUT() > +OVS_TRAFFIC_VSWITCHD_START() > + > +ADD_NAMESPACES(at_ns0, at_ns1) > + > +ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24") > +ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") > + > +AT_DATA([flows.txt], [dnl > +priority=1,action=drop > +priority=10,arp,action=normal > +priority=100,in_port=1,ip,action=ct(zone=5, table=1) > +priority=100,in_port=2,ip,action=ct(zone=5, table=1) > +table=1,in_port=2,ip,ct_state=+trk+est,action=1 > +table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit,zone=5),2 > +table=1,in_port=1,ip,ct_state=+trk+est,action=2 > +]) > + > +AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) > + > +dnl Test with default timeout > +dnl The default udp_single and icmp_first timeouts are 30 seconds in > +dnl kernel DP, and 60 seconds in userspace DP. > + > +dnl Send ICMP and UDP traffic > +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | > FORMAT_PING], [0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 > packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 > actions=resubmit(,0)"]) > + > +sleep 4 > + > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], > [0], [dnl > > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > +]) > + > +AT_CHECK([ovs-appctl dpctl/flush-conntrack]) > + > +dnl Shorten the udp_single and icmp_first timeout in zone 5 > +VSCTL_ADD_DATAPATH_TABLE() > +VSCTL_ADD_ZONE_TIMEOUT_POLICY([zone=5 udp_single=3 icmp_first=3]) > + > +dnl Send ICMP and UDP traffic > +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | > FORMAT_PING], [0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 > packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 > actions=resubmit(,0)"]) > + > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], > [0], [dnl > > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > +]) > + > +dnl Wait until the timeout expire. > +dnl We intend to wait a bit longer, because conntrack does not recycle > the entry right after it is expired. > +sleep 4 > + > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > +]) > + > +OVS_TRAFFIC_VSWITCHD_STOP > +AT_CLEANUP > + > AT_BANNER([conntrack - L7]) > > AT_SETUP([conntrack - IPv4 HTTP]) > diff --git a/tests/system-userspace-macros.at b/tests/ > system-userspace-macros.at > index 9d5f3bf419d3..8950a4de7287 100644 > --- a/tests/system-userspace-macros.at > +++ b/tests/system-userspace-macros.at > @@ -98,6 +98,16 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP]) > # > m4_define([CHECK_CONNTRACK_NAT]) > > +# CHECK_CONNTRACK_TIMEOUT() > +# > +# Perform requirements checks for running conntrack customized timeout > tests. > +* The userspace datapath does not support this feature yet. > +# > +m4_define([CHECK_CONNTRACK_TIMEOUT], > +[ > + AT_SKIP_IF([:]) > +]) > + > # CHECK_CT_DPIF_PER_ZONE_LIMIT() > # > # Perform requirements checks for running ovs-dpctl > ct-[set|get|del]-limits per > @@ -295,3 +305,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL], > [ > AT_SKIP_IF([:]) > ]) > + > +# VSCTL_ADD_DATAPATH_TABLE() > +# > +# Create datapath table "netdev" for userspace tests in ovsdb > +m4_define([VSCTL_ADD_DATAPATH_TABLE], > +[ > + AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- > set Open_vSwitch . datapaths:"netdev"=@m], [0], [stdout]) > +]) > + > +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters]) > +# > +# Add zone based timeout policy to userspace datapath > +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY], > +[ > + AT_CHECK([ovs-vsctl add-zone-tp netdev $1], [0], [stdout]) > +]) > -- > 2.7.4 > > _______________________________________________ > dev mailing list > dev@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev >
On Mon, Aug 5, 2019 at 9:03 PM Darrell Ball <dlu998@gmail.com> wrote: > Thanks for the patch > > I see the test is much improved now from V1 and passes - thanks > > Ideally, tests should be associated with some code for context > It could be folded into patch 8 > I did some more testing and found a similar problem as in V1. This test can be run successfully once and then fails after that. Maybe you want to look into that. It is probably related to: dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf . nfnetlink_cttimeout 16384 1 . Darrell > > > On Thu, Aug 1, 2019 at 3:12 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > >> This patch adds a system traffic test to verify the zone-based conntrack >> timeout feature. The test uses ovs-vsctl commands to configure >> the customized ICMP and UDP timeout on zone 5 to a shorter period. >> It then injects ICMP and UDP traffic to conntrack, and checks if the >> corresponding conntrack entry expires after the predefined timeout. >> >> Signed-off-by: Yi-Hung Wei <yihung.wei@gmail.com> >> --- >> tests/system-kmod-macros.at | 25 +++++++++++++++ >> tests/system-traffic.at | 66 >> ++++++++++++++++++++++++++++++++++++++++ >> tests/system-userspace-macros.at | 26 ++++++++++++++++ >> 3 files changed, 117 insertions(+) >> >> diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at >> index 554a61e9bd95..1bc6f246f426 100644 >> --- a/tests/system-kmod-macros.at >> +++ b/tests/system-kmod-macros.at >> @@ -100,6 +100,15 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP], >> # >> m4_define([CHECK_CONNTRACK_NAT]) >> >> +# CHECK_CONNTRACK_TIMEOUT() >> +# >> +# Perform requirements checks for running conntrack customized timeout >> tests. >> +# >> +m4_define([CHECK_CONNTRACK_TIMEOUT], >> +[ >> + AT_SKIP_IF([! cat /boot/config-$(uname -r) | grep >> NF_CONNTRACK_TIMEOUT | grep '=y' > /dev/null]) >> +]) >> + >> # CHECK_CT_DPIF_PER_ZONE_LIMIT() >> # >> # Perform requirements checks for running ovs-dpctl >> ct-[set|get|del]-limits per >> @@ -185,3 +194,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL], >> sublevel=$(uname -r | sed -e 's/\./ /g' | awk '{print $ 2}') >> AT_SKIP_IF([ ! ( test $version -lt $1 || ( test $version -eq $1 && >> test $sublevel -lt $2 ) || test $version -gt $3 || ( test $version -eq $3 >> && test $sublevel -gt $4 ) ) ]) >> ]) >> + >> +# VSCTL_ADD_DATAPATH_TABLE() >> +# >> +# Create system datapath table "system" for kernel tests in ovsdb >> +m4_define([VSCTL_ADD_DATAPATH_TABLE], >> +[ >> + AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- >> set Open_vSwitch . datapaths:"system"=@m], [0], [stdout]) >> +]) >> + >> +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters]) >> +# >> +# Add zone based timeout policy to kernel datapath >> +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY], >> +[ >> + AT_CHECK([ovs-vsctl add-zone-tp system $1], [0], [stdout]) >> +]) >> diff --git a/tests/system-traffic.at b/tests/system-traffic.at >> index 1a04199dcfe9..f4ac8a8f2c06 100644 >> --- a/tests/system-traffic.at >> +++ b/tests/system-traffic.at >> @@ -3179,6 +3179,72 @@ NXST_FLOW reply: >> OVS_TRAFFIC_VSWITCHD_STOP >> AT_CLEANUP >> >> +AT_SETUP([conntrack - zone-based timeout policy]) >> +CHECK_CONNTRACK() >> +CHECK_CONNTRACK_TIMEOUT() >> +OVS_TRAFFIC_VSWITCHD_START() >> + >> +ADD_NAMESPACES(at_ns0, at_ns1) >> + >> +ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24") >> +ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") >> + >> +AT_DATA([flows.txt], [dnl >> +priority=1,action=drop >> +priority=10,arp,action=normal >> +priority=100,in_port=1,ip,action=ct(zone=5, table=1) >> +priority=100,in_port=2,ip,action=ct(zone=5, table=1) >> +table=1,in_port=2,ip,ct_state=+trk+est,action=1 >> +table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit,zone=5),2 >> +table=1,in_port=1,ip,ct_state=+trk+est,action=2 >> +]) >> + >> +AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) >> + >> +dnl Test with default timeout >> +dnl The default udp_single and icmp_first timeouts are 30 seconds in >> +dnl kernel DP, and 60 seconds in userspace DP. >> + >> +dnl Send ICMP and UDP traffic >> +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | >> FORMAT_PING], [0], [dnl >> +3 packets transmitted, 3 received, 0% packet loss, time 0ms >> +]) >> +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 >> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 >> actions=resubmit(,0)"]) >> + >> +sleep 4 >> + >> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], >> [0], [dnl >> >> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 >> >> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 >> +]) >> + >> +AT_CHECK([ovs-appctl dpctl/flush-conntrack]) >> + >> +dnl Shorten the udp_single and icmp_first timeout in zone 5 >> +VSCTL_ADD_DATAPATH_TABLE() >> +VSCTL_ADD_ZONE_TIMEOUT_POLICY([zone=5 udp_single=3 icmp_first=3]) >> + >> +dnl Send ICMP and UDP traffic >> +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | >> FORMAT_PING], [0], [dnl >> +3 packets transmitted, 3 received, 0% packet loss, time 0ms >> +]) >> +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 >> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 >> actions=resubmit(,0)"]) >> + >> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], >> [0], [dnl >> >> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 >> >> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 >> +]) >> + >> +dnl Wait until the timeout expire. >> +dnl We intend to wait a bit longer, because conntrack does not recycle >> the entry right after it is expired. >> +sleep 4 >> + >> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], >> [dnl >> +]) >> + >> +OVS_TRAFFIC_VSWITCHD_STOP >> +AT_CLEANUP >> + >> AT_BANNER([conntrack - L7]) >> >> AT_SETUP([conntrack - IPv4 HTTP]) >> diff --git a/tests/system-userspace-macros.at b/tests/ >> system-userspace-macros.at >> index 9d5f3bf419d3..8950a4de7287 100644 >> --- a/tests/system-userspace-macros.at >> +++ b/tests/system-userspace-macros.at >> @@ -98,6 +98,16 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP]) >> # >> m4_define([CHECK_CONNTRACK_NAT]) >> >> +# CHECK_CONNTRACK_TIMEOUT() >> +# >> +# Perform requirements checks for running conntrack customized timeout >> tests. >> +* The userspace datapath does not support this feature yet. >> +# >> +m4_define([CHECK_CONNTRACK_TIMEOUT], >> +[ >> + AT_SKIP_IF([:]) >> +]) >> + >> # CHECK_CT_DPIF_PER_ZONE_LIMIT() >> # >> # Perform requirements checks for running ovs-dpctl >> ct-[set|get|del]-limits per >> @@ -295,3 +305,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL], >> [ >> AT_SKIP_IF([:]) >> ]) >> + >> +# VSCTL_ADD_DATAPATH_TABLE() >> +# >> +# Create datapath table "netdev" for userspace tests in ovsdb >> +m4_define([VSCTL_ADD_DATAPATH_TABLE], >> +[ >> + AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- >> set Open_vSwitch . datapaths:"netdev"=@m], [0], [stdout]) >> +]) >> + >> +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters]) >> +# >> +# Add zone based timeout policy to userspace datapath >> +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY], >> +[ >> + AT_CHECK([ovs-vsctl add-zone-tp netdev $1], [0], [stdout]) >> +]) >> -- >> 2.7.4 >> >> _______________________________________________ >> dev mailing list >> dev@openvswitch.org >> https://mail.openvswitch.org/mailman/listinfo/ovs-dev >> >
On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote: > > > I did some more testing and found a similar problem as in V1. > > This test can be run successfully once and then fails after that. > Maybe you want to look into that. It is probably related to: > > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf > . > nfnetlink_cttimeout 16384 1 > . > > Darrell > Thanks for trying out the test. I can not reproduce the issue that you mentioned on my local VM. Can you provide your kernel version and system-kmod-testsuite.log? Thanks, -Yi-Hung
On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote: > > > > > > I did some more testing and found a similar problem as in V1. > > > > This test can be run successfully once and then fails after that. > > Maybe you want to look into that. It is probably related to: > > > > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf > > . > > nfnetlink_cttimeout 16384 1 > > . > > > > Darrell > > > > Thanks for trying out the test. I can not reproduce the issue that > you mentioned on my local VM. > > Can you provide your kernel version and system-kmod-testsuite.log? > > Thanks, > > -Yi-Hung > Here it is: dball@ubuntu:~/ovs$ uname -a Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux dball@ubuntu:~/ovs$ lsb_release -a No LSB modules are available. Distributor ID: Ubuntu Description: Ubuntu 16.04.2 LTS Release: 16.04 Codename: xenial # -*- compilation -*- 75. system-traffic.at:3182: testing conntrack - zone-based timeout policy ... net.netfilter.nf_conntrack_helper = 0 ../../tests/system-traffic.at:3185: modprobe openvswitch ../../tests/system-traffic.at:3185: ovsdb-tool create conf.db $abs_top_srcdir/vswitchd/vswitch.ovsschema ../../tests/system-traffic.at:3185: ovsdb-server --detach --no-chdir --pidfile --log-file --remote=punix:$OVS_RUNDIR/db.sock stderr: 2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log ../../tests/system-traffic.at:3185: sed < stderr ' /vlog|INFO|opened log file/d /ovsdb_server|INFO|ovsdb-server (Open vSwitch)/d' ../../tests/system-traffic.at:3185: ovs-vsctl --no-wait init ../../tests/system-traffic.at:3185: ovs-vswitchd --detach --no-chdir --pidfile --log-file -vvconn -vofproto_dpif -vunixctl stderr: 2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log 2019-08-06T19:11:47Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on NUMA node 0 2019-08-06T19:11:47Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and 1 CPU cores 2019-08-06T19:11:47Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: connecting... 2019-08-06T19:11:47Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: connected ../../tests/system-traffic.at:3185: sed < stderr ' /ovs_numa|INFO|Discovered /d /vlog|INFO|opened log file/d /vswitchd|INFO|ovs-vswitchd (Open vSwitch)/d /reconnect|INFO|/d /dpif_netlink|INFO|Generic Netlink family .ovs_datapath. does not exist/d /ofproto|INFO|using datapath ID/d /netdev_linux|INFO|.*device has unknown hardware address family/d /ofproto|INFO|datapath ID changed to fedcba9876543210/d /dpdk|INFO|DPDK Disabled - Use other_config:dpdk-init to enable/d /netlink_socket|INFO|netlink: could not enable listening to all nsid/d /probe tc:/d /tc: Using policy/d' ../../tests/system-traffic.at:3185: ovs-vsctl -- add-br br0 -- set Bridge br0 protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15 fail-mode=secure -- --- /dev/null 2019-02-26 18:50:08.043999906 -0800 +++ /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/stderr 2019-08-06 12:12:17.489401899 -0700 @@ -0,0 +1,2 @@ +2019-08-06T19:12:17Z|00002|fatal_signal|WARN|terminating with signal 14 (Alarm clock) +/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/test-source: line 874: 58958 Alarm clock ovs-vsctl -- add-br br0 -- set Bridge br0 protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15 fail-mode=secure -- ../../tests/system-traffic.at:3185: exit code was 142, expected 0 ovsdb-server.log: > 2019-08-06T19:11:47.418Z|00001|vlog|INFO|opened log file /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log > 2019-08-06T19:11:47.420Z|00002|ovsdb_server|INFO|ovsdb-server (Open vSwitch) 2.12.90 > 2019-08-06T19:11:57.433Z|00003|memory|INFO|4504 kB peak resident set size after 10.0 seconds > 2019-08-06T19:11:57.433Z|00004|memory|INFO|cells:122 monitors:3 sessions:2 ovs-vswitchd.log: > 2019-08-06T19:11:47.449Z|00001|vlog|INFO|opened log file /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log > 2019-08-06T19:11:47.449Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on NUMA node 0 > 2019-08-06T19:11:47.449Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and 1 CPU cores > 2019-08-06T19:11:47.450Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: connecting... > 2019-08-06T19:11:47.450Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: connected > 2019-08-06T19:11:47.473Z|00006|bridge|INFO|ovs-vswitchd (Open vSwitch) 2.12.90 > 2019-08-06T19:11:47.492Z|00007|ofproto_dpif|INFO|system@ovs-system: Datapath supports recirculation > 2019-08-06T19:11:47.492Z|00008|ofproto_dpif|INFO|system@ovs-system: VLAN header stack length probed as 2 > 2019-08-06T19:11:47.492Z|00009|ofproto_dpif|INFO|system@ovs-system: MPLS label stack length probed as 1 > 2019-08-06T19:11:47.492Z|00010|ofproto_dpif|INFO|system@ovs-system: Datapath supports truncate action > 2019-08-06T19:11:47.492Z|00011|ofproto_dpif|INFO|system@ovs-system: Datapath supports unique flow ids > 2019-08-06T19:11:47.492Z|00012|ofproto_dpif|INFO|system@ovs-system: Datapath supports clone action > 2019-08-06T19:11:47.492Z|00013|ofproto_dpif|INFO|system@ovs-system: Max sample nesting level probed as 10 > 2019-08-06T19:11:47.492Z|00014|ofproto_dpif|INFO|system@ovs-system: Datapath supports eventmask in conntrack action > 2019-08-06T19:11:47.492Z|00015|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_clear action > 2019-08-06T19:11:47.492Z|00016|ofproto_dpif|INFO|system@ovs-system: Max dp_hash algorithm probed to be 0 > 2019-08-06T19:11:47.492Z|00017|ofproto_dpif|INFO|system@ovs-system: Datapath supports check_pkt_len action > 2019-08-06T19:11:47.492Z|00018|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_state > 2019-08-06T19:11:47.492Z|00019|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_zone > 2019-08-06T19:11:47.492Z|00020|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_mark > 2019-08-06T19:11:47.492Z|00021|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_label > 2019-08-06T19:11:47.492Z|00022|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_state_nat > 2019-08-06T19:11:47.492Z|00023|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_orig_tuple > 2019-08-06T19:11:47.492Z|00024|ofproto_dpif|INFO|system@ovs-system: Datapath supports ct_orig_tuple6 ../tests/system-kmod-testsuite: line 1705: 59097 Alarm clock exit 1 2019-08-06T19:12:28Z|00001|daemon_unix|WARN|/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid: open: No such file or directory ovs-appctl: cannot read pidfile "/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid" (No such file or directory) 75. system-traffic.at:3182: 75. conntrack - zone-based timeout policy ( system-traffic.at:3182): FAILED (system-traffic.at:3185) dball@ubuntu:~/ovs$ lsmod | grep nf nfnetlink_cttimeout 16384 1
I did some further testing and ran into another issue; in this case, one, I did not expect. I added an additional sending of packets at the end of the test after this check: AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl ]) Below is new code dnl Do it again dnl Send ICMP and UDP traffic NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl 3 packets transmitted, 3 received, 0% packet loss, time 0ms ]) AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 ]) dnl Wait until the timeout expire. dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired. sleep 5 AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl ]) The test fails bcoz the second time with short timeouts, the conntrack entries are not cleanup up quickly @@ -0,0 +1,2 @@ +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 On Tue, Aug 6, 2019 at 12:16 PM Darrell Ball <dlu998@gmail.com> wrote: > > > On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > >> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote: >> > >> > >> > I did some more testing and found a similar problem as in V1. >> > >> > This test can be run successfully once and then fails after that. >> > Maybe you want to look into that. It is probably related to: >> > >> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf >> > . >> > nfnetlink_cttimeout 16384 1 >> > . >> > >> > Darrell >> > >> >> Thanks for trying out the test. I can not reproduce the issue that >> you mentioned on my local VM. >> >> Can you provide your kernel version and system-kmod-testsuite.log? >> >> Thanks, >> >> -Yi-Hung >> > > > Here it is: > > dball@ubuntu:~/ovs$ uname -a > Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC 2018 > x86_64 x86_64 x86_64 GNU/Linux > > dball@ubuntu:~/ovs$ lsb_release -a > No LSB modules are available. > Distributor ID: Ubuntu > Description: Ubuntu 16.04.2 LTS > Release: 16.04 > Codename: xenial > > # -*- compilation -*- > 75. system-traffic.at:3182: testing conntrack - zone-based timeout policy > ... > net.netfilter.nf_conntrack_helper = 0 > ../../tests/system-traffic.at:3185: modprobe openvswitch > ../../tests/system-traffic.at:3185: ovsdb-tool create conf.db > $abs_top_srcdir/vswitchd/vswitch.ovsschema > ../../tests/system-traffic.at:3185: ovsdb-server --detach --no-chdir > --pidfile --log-file --remote=punix:$OVS_RUNDIR/db.sock > stderr: > 2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file > /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log > ../../tests/system-traffic.at:3185: sed < stderr ' > /vlog|INFO|opened log file/d > /ovsdb_server|INFO|ovsdb-server (Open vSwitch)/d' > ../../tests/system-traffic.at:3185: ovs-vsctl --no-wait init > ../../tests/system-traffic.at:3185: ovs-vswitchd --detach --no-chdir > --pidfile --log-file -vvconn -vofproto_dpif -vunixctl > stderr: > 2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file > /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log > 2019-08-06T19:11:47Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on NUMA > node 0 > 2019-08-06T19:11:47Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and 1 CPU > cores > 2019-08-06T19:11:47Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: > connecting... > 2019-08-06T19:11:47Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: > connected > ../../tests/system-traffic.at:3185: sed < stderr ' > /ovs_numa|INFO|Discovered /d > /vlog|INFO|opened log file/d > /vswitchd|INFO|ovs-vswitchd (Open vSwitch)/d > /reconnect|INFO|/d > /dpif_netlink|INFO|Generic Netlink family .ovs_datapath. does not exist/d > /ofproto|INFO|using datapath ID/d > /netdev_linux|INFO|.*device has unknown hardware address family/d > /ofproto|INFO|datapath ID changed to fedcba9876543210/d > /dpdk|INFO|DPDK Disabled - Use other_config:dpdk-init to enable/d > /netlink_socket|INFO|netlink: could not enable listening to all nsid/d > /probe tc:/d > /tc: Using policy/d' > ../../tests/system-traffic.at:3185: ovs-vsctl -- add-br br0 -- set Bridge > br0 > protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15 > fail-mode=secure -- > --- /dev/null 2019-02-26 18:50:08.043999906 -0800 > +++ > /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/stderr > 2019-08-06 12:12:17.489401899 -0700 > @@ -0,0 +1,2 @@ > +2019-08-06T19:12:17Z|00002|fatal_signal|WARN|terminating with signal 14 > (Alarm clock) > +/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/test-source: > line 874: 58958 Alarm clock ovs-vsctl -- add-br br0 -- set > Bridge br0 > protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15 > fail-mode=secure -- > ../../tests/system-traffic.at:3185: exit code was 142, expected 0 > ovsdb-server.log: > > 2019-08-06T19:11:47.418Z|00001|vlog|INFO|opened log file > /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log > > 2019-08-06T19:11:47.420Z|00002|ovsdb_server|INFO|ovsdb-server (Open > vSwitch) 2.12.90 > > 2019-08-06T19:11:57.433Z|00003|memory|INFO|4504 kB peak resident set > size after 10.0 seconds > > 2019-08-06T19:11:57.433Z|00004|memory|INFO|cells:122 monitors:3 > sessions:2 > ovs-vswitchd.log: > > 2019-08-06T19:11:47.449Z|00001|vlog|INFO|opened log file > /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log > > 2019-08-06T19:11:47.449Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on > NUMA node 0 > > 2019-08-06T19:11:47.449Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and > 1 CPU cores > > > 2019-08-06T19:11:47.450Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: > connecting... > > > 2019-08-06T19:11:47.450Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock: > connected > > 2019-08-06T19:11:47.473Z|00006|bridge|INFO|ovs-vswitchd (Open vSwitch) > 2.12.90 > > 2019-08-06T19:11:47.492Z|00007|ofproto_dpif|INFO|system@ovs-system: > Datapath supports recirculation > > 2019-08-06T19:11:47.492Z|00008|ofproto_dpif|INFO|system@ovs-system: > VLAN header stack length probed as 2 > > 2019-08-06T19:11:47.492Z|00009|ofproto_dpif|INFO|system@ovs-system: > MPLS label stack length probed as 1 > > 2019-08-06T19:11:47.492Z|00010|ofproto_dpif|INFO|system@ovs-system: > Datapath supports truncate action > > 2019-08-06T19:11:47.492Z|00011|ofproto_dpif|INFO|system@ovs-system: > Datapath supports unique flow ids > > 2019-08-06T19:11:47.492Z|00012|ofproto_dpif|INFO|system@ovs-system: > Datapath supports clone action > > 2019-08-06T19:11:47.492Z|00013|ofproto_dpif|INFO|system@ovs-system: Max > sample nesting level probed as 10 > > 2019-08-06T19:11:47.492Z|00014|ofproto_dpif|INFO|system@ovs-system: > Datapath supports eventmask in conntrack action > > 2019-08-06T19:11:47.492Z|00015|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_clear action > > 2019-08-06T19:11:47.492Z|00016|ofproto_dpif|INFO|system@ovs-system: Max > dp_hash algorithm probed to be 0 > > 2019-08-06T19:11:47.492Z|00017|ofproto_dpif|INFO|system@ovs-system: > Datapath supports check_pkt_len action > > 2019-08-06T19:11:47.492Z|00018|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_state > > 2019-08-06T19:11:47.492Z|00019|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_zone > > 2019-08-06T19:11:47.492Z|00020|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_mark > > 2019-08-06T19:11:47.492Z|00021|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_label > > 2019-08-06T19:11:47.492Z|00022|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_state_nat > > 2019-08-06T19:11:47.492Z|00023|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_orig_tuple > > 2019-08-06T19:11:47.492Z|00024|ofproto_dpif|INFO|system@ovs-system: > Datapath supports ct_orig_tuple6 > ../tests/system-kmod-testsuite: line 1705: 59097 Alarm clock > exit 1 > 2019-08-06T19:12:28Z|00001|daemon_unix|WARN|/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid: > open: No such file or directory > ovs-appctl: cannot read pidfile > "/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid" > (No such file or directory) > 75. system-traffic.at:3182: 75. conntrack - zone-based timeout policy ( > system-traffic.at:3182): FAILED (system-traffic.at:3185) > > > dball@ubuntu:~/ovs$ lsmod | grep nf > nfnetlink_cttimeout 16384 1 > > >
On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote: > > I did some further testing and ran into another issue; in this case, one, I did not expect. > > I added an additional sending of packets at the end of the test after this check: > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl > ]) > > Below is new code > > dnl Do it again > dnl Send ICMP and UDP traffic > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl > 3 packets transmitted, 3 received, 0% packet loss, time 0ms > ]) > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl > icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > ]) > > dnl Wait until the timeout expire. > dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired. > sleep 5 > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl > ]) > > The test fails bcoz the second time with short timeouts, the conntrack entries are not cleanup up quickly > > @@ -0,0 +1,2 @@ > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > > > On Tue, Aug 6, 2019 at 12:16 PM Darrell Ball <dlu998@gmail.com> wrote: >> >> >> >> On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> wrote: >>> >>> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote: >>> > >>> > >>> > I did some more testing and found a similar problem as in V1. >>> > >>> > This test can be run successfully once and then fails after that. >>> > Maybe you want to look into that. It is probably related to: >>> > >>> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf >>> > . >>> > nfnetlink_cttimeout 16384 1 >>> > . >>> > >>> > Darrell >>> > >>> >>> Thanks for trying out the test. I can not reproduce the issue that >>> you mentioned on my local VM. >>> >>> Can you provide your kernel version and system-kmod-testsuite.log? >>> >>> Thanks, >>> >>> -Yi-Hung >> >> >> >> Here it is: >> >> dball@ubuntu:~/ovs$ uname -a >> Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux >> Thanks for reporting the issue. I am able to reproduce in similar set up. It should be resolved in v3. Thanks, -Yi-Hung
On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote: > > I did some further testing and ran into another issue; in this case, one, I did not expect. > > I added an additional sending of packets at the end of the test after this check: > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl > ]) > > Below is new code > > dnl Do it again > dnl Send ICMP and UDP traffic > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl > 3 packets transmitted, 3 received, 0% packet loss, time 0ms > ]) > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl > icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > ]) > > dnl Wait until the timeout expire. > dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired. > sleep 5 > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl > ]) > > The test fails bcoz the second time with short timeouts, the conntrack entries are not cleanup up quickly > > @@ -0,0 +1,2 @@ > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 Thanks for testing! This test actually catch a kernel bug when ovs kernel handles conntrack cache. It works for me on my ubuntu xenial VM with 4.4 kernel. Since this requires upstream kernel change, it will be backported to OVS once the fix gets upstream. Thanks, -Yi-Hung diff --git a/datapath/conntrack.c b/datapath/conntrack.c index f85d0a2572f6..ad48b559bcde 100644 --- a/datapath/conntrack.c +++ b/datapath/conntrack.c @@ -76,6 +76,7 @@ enum ovs_ct_nat { /* Conntrack action context for execution. */ struct ovs_conntrack_info { struct nf_conntrack_helper *helper; + struct nf_ct_timeout *nf_ct_timeout; struct nf_conntrack_zone zone; struct nf_conn *ct; u8 commit : 1; @@ -745,6 +746,13 @@ static bool skb_nfct_cached(struct net *net, if (help && rcu_access_pointer(help->helper) != info->helper) return false; } + if (info->nf_ct_timeout) { + struct nf_conn_timeout *timeout_ext; + + timeout_ext = nf_ct_timeout_find(ct); + if (!timeout_ext || info->nf_ct_timeout != timeout_ext->timeout) + return false; + } /* Force conntrack entry direction to the current packet? */ if (info->force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) { /* Delete the conntrack entry if confirmed, else just release @@ -1704,6 +1712,8 @@ int ovs_ct_copy_action(struct net *net, const struct nlattr *attr, ct_info.timeout)) pr_info_ratelimited("Failed to associated timeout " "policy `%s'\n", ct_info.timeout); + else + ct_info.nf_ct_timeout = nf_ct_timeout_find(ct_info.ct)->timeout; } if (helper) {
On Mon, Aug 12, 2019 at 5:22 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote: > > > > I did some further testing and ran into another issue; in this case, > one, I did not expect. > > > > I added an additional sending of packets at the end of the test after > this check: > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > > ]) > > > > Below is new code > > > > dnl Do it again > > dnl Send ICMP and UDP traffic > > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | > FORMAT_PING], [0], [dnl > > 3 packets transmitted, 3 received, 0% packet loss, time 0ms > > ]) > > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 > packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 > actions=resubmit(,0)"]) > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], > [0], [dnl > > > icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > > udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > ]) > > > > dnl Wait until the timeout expire. > > dnl We intend to wait a bit longer, because conntrack does not recycle > the entry right after it is expired. > > sleep 5 > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > > ]) > > > > The test fails bcoz the second time with short timeouts, the conntrack > entries are not cleanup up quickly > > > > @@ -0,0 +1,2 @@ > > > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > > Thanks for testing! This test actually catch a kernel bug when ovs > kernel handles conntrack cache. It works for me on my ubuntu xenial > VM with 4.4 kernel. > > Since this requires upstream kernel change, it will be backported to > OVS once the fix gets upstream. > > Thanks, > > -Yi-Hung > Does the below patch fix just the failed timeout policy for sending second and subsequent packet only or also the issue of failed test runs after the first run ? If the issue of subsequent failed test runs is different, what is the fix you want to use for it ? > > diff --git a/datapath/conntrack.c b/datapath/conntrack.c > index f85d0a2572f6..ad48b559bcde 100644 > --- a/datapath/conntrack.c > +++ b/datapath/conntrack.c > @@ -76,6 +76,7 @@ enum ovs_ct_nat { > /* Conntrack action context for execution. */ > struct ovs_conntrack_info { > struct nf_conntrack_helper *helper; > + struct nf_ct_timeout *nf_ct_timeout; > struct nf_conntrack_zone zone; > struct nf_conn *ct; > u8 commit : 1; > @@ -745,6 +746,13 @@ static bool skb_nfct_cached(struct net *net, > if (help && rcu_access_pointer(help->helper) != > info->helper) > return false; > } > + if (info->nf_ct_timeout) { > + struct nf_conn_timeout *timeout_ext; > + > + timeout_ext = nf_ct_timeout_find(ct); > + if (!timeout_ext || info->nf_ct_timeout != > timeout_ext->timeout) > + return false; > + } > /* Force conntrack entry direction to the current packet? */ > if (info->force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) { > /* Delete the conntrack entry if confirmed, else just > release > @@ -1704,6 +1712,8 @@ int ovs_ct_copy_action(struct net *net, const > struct nlattr *attr, > ct_info.timeout)) > pr_info_ratelimited("Failed to associated timeout " > "policy `%s'\n", > ct_info.timeout); > + else > + ct_info.nf_ct_timeout = > nf_ct_timeout_find(ct_info.ct)->timeout; > } > > if (helper) { >
On Mon, Aug 12, 2019 at 5:15 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote: > > > > I did some further testing and ran into another issue; in this case, > one, I did not expect. > > > > I added an additional sending of packets at the end of the test after > this check: > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > > ]) > > > > Below is new code > > > > dnl Do it again > > dnl Send ICMP and UDP traffic > > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | > FORMAT_PING], [0], [dnl > > 3 packets transmitted, 3 received, 0% packet loss, time 0ms > > ]) > > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 > packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 > actions=resubmit(,0)"]) > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], > [0], [dnl > > > icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > > udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > ]) > > > > dnl Wait until the timeout expire. > > dnl We intend to wait a bit longer, because conntrack does not recycle > the entry right after it is expired. > > sleep 5 > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > > ]) > > > > The test fails bcoz the second time with short timeouts, the conntrack > entries are not cleanup up quickly > > > > @@ -0,0 +1,2 @@ > > > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > > > > > > > On Tue, Aug 6, 2019 at 12:16 PM Darrell Ball <dlu998@gmail.com> wrote: > >> > >> > >> > >> On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> > wrote: > >>> > >>> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote: > >>> > > >>> > > >>> > I did some more testing and found a similar problem as in V1. > >>> > > >>> > This test can be run successfully once and then fails after that. > >>> > Maybe you want to look into that. It is probably related to: > >>> > > >>> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf > >>> > . > >>> > nfnetlink_cttimeout 16384 1 > >>> > . > >>> > > >>> > Darrell > >>> > > >>> > >>> Thanks for trying out the test. I can not reproduce the issue that > >>> you mentioned on my local VM. > >>> > >>> Can you provide your kernel version and system-kmod-testsuite.log? > >>> > >>> Thanks, > >>> > >>> -Yi-Hung > >> > >> > >> > >> Here it is: > >> > >> dball@ubuntu:~/ovs$ uname -a > >> Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC > 2018 x86_64 x86_64 x86_64 GNU/Linux > >> > Thanks for reporting the issue. I am able to reproduce in similar set > up. It should be resolved in v3. > What is the fix you want to use for this bug; it must be different from the second bug you have proposed a patch for in another response ? Thanks Darrell > > Thanks, > > -Yi-Hung >
On Mon, Aug 12, 2019 at 5:22 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote: > On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote: > > > > I did some further testing and ran into another issue; in this case, > one, I did not expect. > > > > I added an additional sending of packets at the end of the test after > this check: > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > > ]) > > > > Below is new code > > > > dnl Do it again > > dnl Send ICMP and UDP traffic > > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | > FORMAT_PING], [0], [dnl > > 3 packets transmitted, 3 received, 0% packet loss, time 0ms > > ]) > > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 > packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 > actions=resubmit(,0)"]) > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], > [0], [dnl > > > icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > > udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > ]) > > > > dnl Wait until the timeout expire. > > dnl We intend to wait a bit longer, because conntrack does not recycle > the entry right after it is expired. > > sleep 5 > > > > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], > [dnl > > ]) > > > > The test fails bcoz the second time with short timeouts, the conntrack > entries are not cleanup up quickly > > > > @@ -0,0 +1,2 @@ > > > +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 > > > +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 > > > Thanks for testing! This test actually catch a kernel bug when ovs > kernel handles conntrack cache. It works for me on my ubuntu xenial > VM with 4.4 kernel. > > Since this requires upstream kernel change, it will be backported to > OVS once the fix gets upstream. > > Thanks, > > -Yi-Hung > > diff --git a/datapath/conntrack.c b/datapath/conntrack.c > index f85d0a2572f6..ad48b559bcde 100644 > --- a/datapath/conntrack.c > +++ b/datapath/conntrack.c > @@ -76,6 +76,7 @@ enum ovs_ct_nat { > /* Conntrack action context for execution. */ > struct ovs_conntrack_info { > struct nf_conntrack_helper *helper; > + struct nf_ct_timeout *nf_ct_timeout; > struct nf_conntrack_zone zone; > struct nf_conn *ct; > u8 commit : 1; > @@ -745,6 +746,13 @@ static bool skb_nfct_cached(struct net *net, > if (help && rcu_access_pointer(help->helper) != > info->helper) > return false; > } > + if (info->nf_ct_timeout) { > + struct nf_conn_timeout *timeout_ext; > + > + timeout_ext = nf_ct_timeout_find(ct); > + if (!timeout_ext || info->nf_ct_timeout != > timeout_ext->timeout) > + return false; > + } > /* Force conntrack entry direction to the current packet? */ > if (info->force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) { > /* Delete the conntrack entry if confirmed, else just > release > @@ -1704,6 +1712,8 @@ int ovs_ct_copy_action(struct net *net, const > struct nlattr *attr, > ct_info.timeout)) > pr_info_ratelimited("Failed to associated timeout " > "policy `%s'\n", > ct_info.timeout); > + else > + ct_info.nf_ct_timeout = > nf_ct_timeout_find(ct_info.ct)->timeout; > } > Forgot to respond to this one earlier. I did review, unit test and system test these changes and they are fine. Thanks Darrell > > if (helper) { >
diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at index 554a61e9bd95..1bc6f246f426 100644 --- a/tests/system-kmod-macros.at +++ b/tests/system-kmod-macros.at @@ -100,6 +100,15 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP], # m4_define([CHECK_CONNTRACK_NAT]) +# CHECK_CONNTRACK_TIMEOUT() +# +# Perform requirements checks for running conntrack customized timeout tests. +# +m4_define([CHECK_CONNTRACK_TIMEOUT], +[ + AT_SKIP_IF([! cat /boot/config-$(uname -r) | grep NF_CONNTRACK_TIMEOUT | grep '=y' > /dev/null]) +]) + # CHECK_CT_DPIF_PER_ZONE_LIMIT() # # Perform requirements checks for running ovs-dpctl ct-[set|get|del]-limits per @@ -185,3 +194,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL], sublevel=$(uname -r | sed -e 's/\./ /g' | awk '{print $ 2}') AT_SKIP_IF([ ! ( test $version -lt $1 || ( test $version -eq $1 && test $sublevel -lt $2 ) || test $version -gt $3 || ( test $version -eq $3 && test $sublevel -gt $4 ) ) ]) ]) + +# VSCTL_ADD_DATAPATH_TABLE() +# +# Create system datapath table "system" for kernel tests in ovsdb +m4_define([VSCTL_ADD_DATAPATH_TABLE], +[ + AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- set Open_vSwitch . datapaths:"system"=@m], [0], [stdout]) +]) + +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters]) +# +# Add zone based timeout policy to kernel datapath +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY], +[ + AT_CHECK([ovs-vsctl add-zone-tp system $1], [0], [stdout]) +]) diff --git a/tests/system-traffic.at b/tests/system-traffic.at index 1a04199dcfe9..f4ac8a8f2c06 100644 --- a/tests/system-traffic.at +++ b/tests/system-traffic.at @@ -3179,6 +3179,72 @@ NXST_FLOW reply: OVS_TRAFFIC_VSWITCHD_STOP AT_CLEANUP +AT_SETUP([conntrack - zone-based timeout policy]) +CHECK_CONNTRACK() +CHECK_CONNTRACK_TIMEOUT() +OVS_TRAFFIC_VSWITCHD_START() + +ADD_NAMESPACES(at_ns0, at_ns1) + +ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24") +ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24") + +AT_DATA([flows.txt], [dnl +priority=1,action=drop +priority=10,arp,action=normal +priority=100,in_port=1,ip,action=ct(zone=5, table=1) +priority=100,in_port=2,ip,action=ct(zone=5, table=1) +table=1,in_port=2,ip,ct_state=+trk+est,action=1 +table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit,zone=5),2 +table=1,in_port=1,ip,ct_state=+trk+est,action=2 +]) + +AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt]) + +dnl Test with default timeout +dnl The default udp_single and icmp_first timeouts are 30 seconds in +dnl kernel DP, and 60 seconds in userspace DP. + +dnl Send ICMP and UDP traffic +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) + +sleep 4 + +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 +]) + +AT_CHECK([ovs-appctl dpctl/flush-conntrack]) + +dnl Shorten the udp_single and icmp_first timeout in zone 5 +VSCTL_ADD_DATAPATH_TABLE() +VSCTL_ADD_ZONE_TIMEOUT_POLICY([zone=5 udp_single=3 icmp_first=3]) + +dnl Send ICMP and UDP traffic +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"]) + +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5 +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5 +]) + +dnl Wait until the timeout expire. +dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired. +sleep 4 + +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl +]) + +OVS_TRAFFIC_VSWITCHD_STOP +AT_CLEANUP + AT_BANNER([conntrack - L7]) AT_SETUP([conntrack - IPv4 HTTP]) diff --git a/tests/system-userspace-macros.at b/tests/system-userspace-macros.at index 9d5f3bf419d3..8950a4de7287 100644 --- a/tests/system-userspace-macros.at +++ b/tests/system-userspace-macros.at @@ -98,6 +98,16 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP]) # m4_define([CHECK_CONNTRACK_NAT]) +# CHECK_CONNTRACK_TIMEOUT() +# +# Perform requirements checks for running conntrack customized timeout tests. +* The userspace datapath does not support this feature yet. +# +m4_define([CHECK_CONNTRACK_TIMEOUT], +[ + AT_SKIP_IF([:]) +]) + # CHECK_CT_DPIF_PER_ZONE_LIMIT() # # Perform requirements checks for running ovs-dpctl ct-[set|get|del]-limits per @@ -295,3 +305,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL], [ AT_SKIP_IF([:]) ]) + +# VSCTL_ADD_DATAPATH_TABLE() +# +# Create datapath table "netdev" for userspace tests in ovsdb +m4_define([VSCTL_ADD_DATAPATH_TABLE], +[ + AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- set Open_vSwitch . datapaths:"netdev"=@m], [0], [stdout]) +]) + +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters]) +# +# Add zone based timeout policy to userspace datapath +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY], +[ + AT_CHECK([ovs-vsctl add-zone-tp netdev $1], [0], [stdout]) +])
This patch adds a system traffic test to verify the zone-based conntrack timeout feature. The test uses ovs-vsctl commands to configure the customized ICMP and UDP timeout on zone 5 to a shorter period. It then injects ICMP and UDP traffic to conntrack, and checks if the corresponding conntrack entry expires after the predefined timeout. Signed-off-by: Yi-Hung Wei <yihung.wei@gmail.com> --- tests/system-kmod-macros.at | 25 +++++++++++++++ tests/system-traffic.at | 66 ++++++++++++++++++++++++++++++++++++++++ tests/system-userspace-macros.at | 26 ++++++++++++++++ 3 files changed, 117 insertions(+)