[ovs-dev,v2,9/9] system-traffic: Add zone-based conntrack timeout policy test
diff mbox series

Message ID 1564697253-37992-10-git-send-email-yihung.wei@gmail.com
State New
Headers show
Series
  • Support zone-based conntrack timeout policy
Related show

Commit Message

Yi-Hung Wei Aug. 1, 2019, 10:07 p.m. UTC
This patch adds a system traffic test to verify the zone-based conntrack
timeout feature.  The test uses ovs-vsctl commands to configure
the customized ICMP and UDP timeout on zone 5 to a shorter period.
It then injects ICMP and UDP traffic to conntrack, and checks if the
corresponding conntrack entry expires after the predefined timeout.

Signed-off-by: Yi-Hung Wei <yihung.wei@gmail.com>
---
 tests/system-kmod-macros.at      | 25 +++++++++++++++
 tests/system-traffic.at          | 66 ++++++++++++++++++++++++++++++++++++++++
 tests/system-userspace-macros.at | 26 ++++++++++++++++
 3 files changed, 117 insertions(+)

Comments

Darrell Ball Aug. 6, 2019, 4:03 a.m. UTC | #1
Thanks for the patch

I see the test is much improved now from V1 and passes - thanks

Ideally, tests should be associated with some code for context
It could be folded into patch 8


On Thu, Aug 1, 2019 at 3:12 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote:

> This patch adds a system traffic test to verify the zone-based conntrack
> timeout feature.  The test uses ovs-vsctl commands to configure
> the customized ICMP and UDP timeout on zone 5 to a shorter period.
> It then injects ICMP and UDP traffic to conntrack, and checks if the
> corresponding conntrack entry expires after the predefined timeout.
>
> Signed-off-by: Yi-Hung Wei <yihung.wei@gmail.com>
> ---
>  tests/system-kmod-macros.at      | 25 +++++++++++++++
>  tests/system-traffic.at          | 66
> ++++++++++++++++++++++++++++++++++++++++
>  tests/system-userspace-macros.at | 26 ++++++++++++++++
>  3 files changed, 117 insertions(+)
>
> diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at
> index 554a61e9bd95..1bc6f246f426 100644
> --- a/tests/system-kmod-macros.at
> +++ b/tests/system-kmod-macros.at
> @@ -100,6 +100,15 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP],
>  #
>  m4_define([CHECK_CONNTRACK_NAT])
>
> +# CHECK_CONNTRACK_TIMEOUT()
> +#
> +# Perform requirements checks for running conntrack customized timeout
> tests.
> +#
> +m4_define([CHECK_CONNTRACK_TIMEOUT],
> +[
> +    AT_SKIP_IF([! cat /boot/config-$(uname -r) | grep
> NF_CONNTRACK_TIMEOUT | grep '=y' > /dev/null])
> +])
> +
>  # CHECK_CT_DPIF_PER_ZONE_LIMIT()
>  #
>  # Perform requirements checks for running ovs-dpctl
> ct-[set|get|del]-limits per
> @@ -185,3 +194,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL],
>      sublevel=$(uname -r | sed -e 's/\./ /g' | awk '{print $ 2}')
>      AT_SKIP_IF([ ! ( test $version -lt $1 || ( test $version -eq $1 &&
> test $sublevel -lt $2 ) || test $version -gt $3 || ( test $version -eq $3
> && test $sublevel -gt $4 ) ) ])
>  ])
> +
> +# VSCTL_ADD_DATAPATH_TABLE()
> +#
> +# Create system datapath table "system" for kernel tests in ovsdb
> +m4_define([VSCTL_ADD_DATAPATH_TABLE],
> +[
> +    AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 --
> set Open_vSwitch . datapaths:"system"=@m], [0], [stdout])
> +])
> +
> +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters])
> +#
> +# Add zone based timeout policy to kernel datapath
> +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY],
> +[
> +    AT_CHECK([ovs-vsctl add-zone-tp system $1], [0], [stdout])
> +])
> diff --git a/tests/system-traffic.at b/tests/system-traffic.at
> index 1a04199dcfe9..f4ac8a8f2c06 100644
> --- a/tests/system-traffic.at
> +++ b/tests/system-traffic.at
> @@ -3179,6 +3179,72 @@ NXST_FLOW reply:
>  OVS_TRAFFIC_VSWITCHD_STOP
>  AT_CLEANUP
>
> +AT_SETUP([conntrack - zone-based timeout policy])
> +CHECK_CONNTRACK()
> +CHECK_CONNTRACK_TIMEOUT()
> +OVS_TRAFFIC_VSWITCHD_START()
> +
> +ADD_NAMESPACES(at_ns0, at_ns1)
> +
> +ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24")
> +ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24")
> +
> +AT_DATA([flows.txt], [dnl
> +priority=1,action=drop
> +priority=10,arp,action=normal
> +priority=100,in_port=1,ip,action=ct(zone=5, table=1)
> +priority=100,in_port=2,ip,action=ct(zone=5, table=1)
> +table=1,in_port=2,ip,ct_state=+trk+est,action=1
> +table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit,zone=5),2
> +table=1,in_port=1,ip,ct_state=+trk+est,action=2
> +])
> +
> +AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt])
> +
> +dnl Test with default timeout
> +dnl The default udp_single and icmp_first timeouts are 30 seconds in
> +dnl kernel DP, and 60 seconds in userspace DP.
> +
> +dnl Send ICMP and UDP traffic
> +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 |
> FORMAT_PING], [0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
> actions=resubmit(,0)"])
> +
> +sleep 4
> +
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
> [0], [dnl
>
> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
>
> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> +])
> +
> +AT_CHECK([ovs-appctl dpctl/flush-conntrack])
> +
> +dnl Shorten the udp_single and icmp_first timeout in zone 5
> +VSCTL_ADD_DATAPATH_TABLE()
> +VSCTL_ADD_ZONE_TIMEOUT_POLICY([zone=5 udp_single=3 icmp_first=3])
> +
> +dnl Send ICMP and UDP traffic
> +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 |
> FORMAT_PING], [0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
> actions=resubmit(,0)"])
> +
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
> [0], [dnl
>
> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
>
> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> +])
> +
> +dnl Wait until the timeout expire.
> +dnl We intend to wait a bit longer, because conntrack does not recycle
> the entry right after it is expired.
> +sleep 4
> +
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0],
> [dnl
> +])
> +
> +OVS_TRAFFIC_VSWITCHD_STOP
> +AT_CLEANUP
> +
>  AT_BANNER([conntrack - L7])
>
>  AT_SETUP([conntrack - IPv4 HTTP])
> diff --git a/tests/system-userspace-macros.at b/tests/
> system-userspace-macros.at
> index 9d5f3bf419d3..8950a4de7287 100644
> --- a/tests/system-userspace-macros.at
> +++ b/tests/system-userspace-macros.at
> @@ -98,6 +98,16 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP])
>  #
>  m4_define([CHECK_CONNTRACK_NAT])
>
> +# CHECK_CONNTRACK_TIMEOUT()
> +#
> +# Perform requirements checks for running conntrack customized timeout
> tests.
> +* The userspace datapath does not support this feature yet.
> +#
> +m4_define([CHECK_CONNTRACK_TIMEOUT],
> +[
> +    AT_SKIP_IF([:])
> +])
> +
>  # CHECK_CT_DPIF_PER_ZONE_LIMIT()
>  #
>  # Perform requirements checks for running ovs-dpctl
> ct-[set|get|del]-limits per
> @@ -295,3 +305,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL],
>  [
>      AT_SKIP_IF([:])
>  ])
> +
> +# VSCTL_ADD_DATAPATH_TABLE()
> +#
> +# Create datapath table "netdev" for userspace tests in ovsdb
> +m4_define([VSCTL_ADD_DATAPATH_TABLE],
> +[
> +    AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 --
> set Open_vSwitch . datapaths:"netdev"=@m], [0], [stdout])
> +])
> +
> +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters])
> +#
> +# Add zone based timeout policy to userspace datapath
> +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY],
> +[
> +    AT_CHECK([ovs-vsctl add-zone-tp netdev $1], [0], [stdout])
> +])
> --
> 2.7.4
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
Darrell Ball Aug. 6, 2019, 5:21 p.m. UTC | #2
On Mon, Aug 5, 2019 at 9:03 PM Darrell Ball <dlu998@gmail.com> wrote:

> Thanks for the patch
>
> I see the test is much improved now from V1 and passes - thanks
>
> Ideally, tests should be associated with some code for context
> It could be folded into patch 8
>

I did some more testing and found a similar problem as in V1.

This test can be run successfully once and then fails after that.
Maybe you want to look into that. It is probably related to:

dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf
.
nfnetlink_cttimeout    16384  1
.

Darrell


>
>
> On Thu, Aug 1, 2019 at 3:12 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote:
>
>> This patch adds a system traffic test to verify the zone-based conntrack
>> timeout feature.  The test uses ovs-vsctl commands to configure
>> the customized ICMP and UDP timeout on zone 5 to a shorter period.
>> It then injects ICMP and UDP traffic to conntrack, and checks if the
>> corresponding conntrack entry expires after the predefined timeout.
>>
>> Signed-off-by: Yi-Hung Wei <yihung.wei@gmail.com>
>> ---
>>  tests/system-kmod-macros.at      | 25 +++++++++++++++
>>  tests/system-traffic.at          | 66
>> ++++++++++++++++++++++++++++++++++++++++
>>  tests/system-userspace-macros.at | 26 ++++++++++++++++
>>  3 files changed, 117 insertions(+)
>>
>> diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at
>> index 554a61e9bd95..1bc6f246f426 100644
>> --- a/tests/system-kmod-macros.at
>> +++ b/tests/system-kmod-macros.at
>> @@ -100,6 +100,15 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP],
>>  #
>>  m4_define([CHECK_CONNTRACK_NAT])
>>
>> +# CHECK_CONNTRACK_TIMEOUT()
>> +#
>> +# Perform requirements checks for running conntrack customized timeout
>> tests.
>> +#
>> +m4_define([CHECK_CONNTRACK_TIMEOUT],
>> +[
>> +    AT_SKIP_IF([! cat /boot/config-$(uname -r) | grep
>> NF_CONNTRACK_TIMEOUT | grep '=y' > /dev/null])
>> +])
>> +
>>  # CHECK_CT_DPIF_PER_ZONE_LIMIT()
>>  #
>>  # Perform requirements checks for running ovs-dpctl
>> ct-[set|get|del]-limits per
>> @@ -185,3 +194,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL],
>>      sublevel=$(uname -r | sed -e 's/\./ /g' | awk '{print $ 2}')
>>      AT_SKIP_IF([ ! ( test $version -lt $1 || ( test $version -eq $1 &&
>> test $sublevel -lt $2 ) || test $version -gt $3 || ( test $version -eq $3
>> && test $sublevel -gt $4 ) ) ])
>>  ])
>> +
>> +# VSCTL_ADD_DATAPATH_TABLE()
>> +#
>> +# Create system datapath table "system" for kernel tests in ovsdb
>> +m4_define([VSCTL_ADD_DATAPATH_TABLE],
>> +[
>> +    AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 --
>> set Open_vSwitch . datapaths:"system"=@m], [0], [stdout])
>> +])
>> +
>> +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters])
>> +#
>> +# Add zone based timeout policy to kernel datapath
>> +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY],
>> +[
>> +    AT_CHECK([ovs-vsctl add-zone-tp system $1], [0], [stdout])
>> +])
>> diff --git a/tests/system-traffic.at b/tests/system-traffic.at
>> index 1a04199dcfe9..f4ac8a8f2c06 100644
>> --- a/tests/system-traffic.at
>> +++ b/tests/system-traffic.at
>> @@ -3179,6 +3179,72 @@ NXST_FLOW reply:
>>  OVS_TRAFFIC_VSWITCHD_STOP
>>  AT_CLEANUP
>>
>> +AT_SETUP([conntrack - zone-based timeout policy])
>> +CHECK_CONNTRACK()
>> +CHECK_CONNTRACK_TIMEOUT()
>> +OVS_TRAFFIC_VSWITCHD_START()
>> +
>> +ADD_NAMESPACES(at_ns0, at_ns1)
>> +
>> +ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24")
>> +ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24")
>> +
>> +AT_DATA([flows.txt], [dnl
>> +priority=1,action=drop
>> +priority=10,arp,action=normal
>> +priority=100,in_port=1,ip,action=ct(zone=5, table=1)
>> +priority=100,in_port=2,ip,action=ct(zone=5, table=1)
>> +table=1,in_port=2,ip,ct_state=+trk+est,action=1
>> +table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit,zone=5),2
>> +table=1,in_port=1,ip,ct_state=+trk+est,action=2
>> +])
>> +
>> +AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt])
>> +
>> +dnl Test with default timeout
>> +dnl The default udp_single and icmp_first timeouts are 30 seconds in
>> +dnl kernel DP, and 60 seconds in userspace DP.
>> +
>> +dnl Send ICMP and UDP traffic
>> +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 |
>> FORMAT_PING], [0], [dnl
>> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
>> +])
>> +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
>> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
>> actions=resubmit(,0)"])
>> +
>> +sleep 4
>> +
>> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
>> [0], [dnl
>>
>> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
>>
>> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
>> +])
>> +
>> +AT_CHECK([ovs-appctl dpctl/flush-conntrack])
>> +
>> +dnl Shorten the udp_single and icmp_first timeout in zone 5
>> +VSCTL_ADD_DATAPATH_TABLE()
>> +VSCTL_ADD_ZONE_TIMEOUT_POLICY([zone=5 udp_single=3 icmp_first=3])
>> +
>> +dnl Send ICMP and UDP traffic
>> +NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 |
>> FORMAT_PING], [0], [dnl
>> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
>> +])
>> +AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
>> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
>> actions=resubmit(,0)"])
>> +
>> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
>> [0], [dnl
>>
>> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
>>
>> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
>> +])
>> +
>> +dnl Wait until the timeout expire.
>> +dnl We intend to wait a bit longer, because conntrack does not recycle
>> the entry right after it is expired.
>> +sleep 4
>> +
>> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0],
>> [dnl
>> +])
>> +
>> +OVS_TRAFFIC_VSWITCHD_STOP
>> +AT_CLEANUP
>> +
>>  AT_BANNER([conntrack - L7])
>>
>>  AT_SETUP([conntrack - IPv4 HTTP])
>> diff --git a/tests/system-userspace-macros.at b/tests/
>> system-userspace-macros.at
>> index 9d5f3bf419d3..8950a4de7287 100644
>> --- a/tests/system-userspace-macros.at
>> +++ b/tests/system-userspace-macros.at
>> @@ -98,6 +98,16 @@ m4_define([CHECK_CONNTRACK_FRAG_OVERLAP])
>>  #
>>  m4_define([CHECK_CONNTRACK_NAT])
>>
>> +# CHECK_CONNTRACK_TIMEOUT()
>> +#
>> +# Perform requirements checks for running conntrack customized timeout
>> tests.
>> +* The userspace datapath does not support this feature yet.
>> +#
>> +m4_define([CHECK_CONNTRACK_TIMEOUT],
>> +[
>> +    AT_SKIP_IF([:])
>> +])
>> +
>>  # CHECK_CT_DPIF_PER_ZONE_LIMIT()
>>  #
>>  # Perform requirements checks for running ovs-dpctl
>> ct-[set|get|del]-limits per
>> @@ -295,3 +305,19 @@ m4_define([OVS_CHECK_KERNEL_EXCL],
>>  [
>>      AT_SKIP_IF([:])
>>  ])
>> +
>> +# VSCTL_ADD_DATAPATH_TABLE()
>> +#
>> +# Create datapath table "netdev" for userspace tests in ovsdb
>> +m4_define([VSCTL_ADD_DATAPATH_TABLE],
>> +[
>> +    AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 --
>> set Open_vSwitch . datapaths:"netdev"=@m], [0], [stdout])
>> +])
>> +
>> +# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters])
>> +#
>> +# Add zone based timeout policy to userspace datapath
>> +m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY],
>> +[
>> +    AT_CHECK([ovs-vsctl add-zone-tp netdev $1], [0], [stdout])
>> +])
>> --
>> 2.7.4
>>
>> _______________________________________________
>> dev mailing list
>> dev@openvswitch.org
>> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>>
>
Yi-Hung Wei Aug. 6, 2019, 6:07 p.m. UTC | #3
On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote:
>
>
> I did some more testing and found a similar problem as in V1.
>
> This test can be run successfully once and then fails after that.
> Maybe you want to look into that. It is probably related to:
>
> dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf
> .
> nfnetlink_cttimeout    16384  1
> .
>
> Darrell
>

Thanks for trying out the test.  I can not reproduce the issue that
you mentioned on my local VM.

Can you provide your kernel version and system-kmod-testsuite.log?

Thanks,

-Yi-Hung
Darrell Ball Aug. 6, 2019, 7:16 p.m. UTC | #4
On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> wrote:

> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote:
> >
> >
> > I did some more testing and found a similar problem as in V1.
> >
> > This test can be run successfully once and then fails after that.
> > Maybe you want to look into that. It is probably related to:
> >
> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf
> > .
> > nfnetlink_cttimeout    16384  1
> > .
> >
> > Darrell
> >
>
> Thanks for trying out the test.  I can not reproduce the issue that
> you mentioned on my local VM.
>
> Can you provide your kernel version and system-kmod-testsuite.log?
>
> Thanks,
>
> -Yi-Hung
>


Here it is:

dball@ubuntu:~/ovs$ uname -a
Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC 2018
x86_64 x86_64 x86_64 GNU/Linux

dball@ubuntu:~/ovs$ lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 16.04.2 LTS
Release: 16.04
Codename: xenial

#                             -*- compilation -*-
75. system-traffic.at:3182: testing conntrack - zone-based timeout policy
...
net.netfilter.nf_conntrack_helper = 0
../../tests/system-traffic.at:3185: modprobe openvswitch
../../tests/system-traffic.at:3185: ovsdb-tool create conf.db
$abs_top_srcdir/vswitchd/vswitch.ovsschema
../../tests/system-traffic.at:3185: ovsdb-server --detach --no-chdir
--pidfile --log-file --remote=punix:$OVS_RUNDIR/db.sock
stderr:
2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file
/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log
../../tests/system-traffic.at:3185: sed < stderr '
/vlog|INFO|opened log file/d
/ovsdb_server|INFO|ovsdb-server (Open vSwitch)/d'
../../tests/system-traffic.at:3185: ovs-vsctl --no-wait init
../../tests/system-traffic.at:3185: ovs-vswitchd  --detach --no-chdir
--pidfile --log-file -vvconn -vofproto_dpif -vunixctl
stderr:
2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file
/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log
2019-08-06T19:11:47Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on NUMA
node 0
2019-08-06T19:11:47Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and 1 CPU
cores
2019-08-06T19:11:47Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
connecting...
2019-08-06T19:11:47Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
connected
../../tests/system-traffic.at:3185: sed < stderr '
/ovs_numa|INFO|Discovered /d
/vlog|INFO|opened log file/d
/vswitchd|INFO|ovs-vswitchd (Open vSwitch)/d
/reconnect|INFO|/d
/dpif_netlink|INFO|Generic Netlink family .ovs_datapath. does not exist/d
/ofproto|INFO|using datapath ID/d
/netdev_linux|INFO|.*device has unknown hardware address family/d
/ofproto|INFO|datapath ID changed to fedcba9876543210/d
/dpdk|INFO|DPDK Disabled - Use other_config:dpdk-init to enable/d
/netlink_socket|INFO|netlink: could not enable listening to all nsid/d
/probe tc:/d
/tc: Using policy/d'
../../tests/system-traffic.at:3185: ovs-vsctl -- add-br br0 -- set Bridge
br0
protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15
fail-mode=secure  --
--- /dev/null 2019-02-26 18:50:08.043999906 -0800
+++
/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/stderr
2019-08-06 12:12:17.489401899 -0700
@@ -0,0 +1,2 @@
+2019-08-06T19:12:17Z|00002|fatal_signal|WARN|terminating with signal 14
(Alarm clock)
+/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/test-source:
line 874: 58958 Alarm clock             ovs-vsctl -- add-br br0 -- set
Bridge br0
protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15
fail-mode=secure --
../../tests/system-traffic.at:3185: exit code was 142, expected 0
ovsdb-server.log:
> 2019-08-06T19:11:47.418Z|00001|vlog|INFO|opened log file
/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log
> 2019-08-06T19:11:47.420Z|00002|ovsdb_server|INFO|ovsdb-server (Open
vSwitch) 2.12.90
> 2019-08-06T19:11:57.433Z|00003|memory|INFO|4504 kB peak resident set size
after 10.0 seconds
> 2019-08-06T19:11:57.433Z|00004|memory|INFO|cells:122 monitors:3 sessions:2
ovs-vswitchd.log:
> 2019-08-06T19:11:47.449Z|00001|vlog|INFO|opened log file
/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log
> 2019-08-06T19:11:47.449Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on
NUMA node 0
> 2019-08-06T19:11:47.449Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and
1 CPU cores
>
2019-08-06T19:11:47.450Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
connecting...
>
2019-08-06T19:11:47.450Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
connected
> 2019-08-06T19:11:47.473Z|00006|bridge|INFO|ovs-vswitchd (Open vSwitch)
2.12.90
> 2019-08-06T19:11:47.492Z|00007|ofproto_dpif|INFO|system@ovs-system:
Datapath supports recirculation
> 2019-08-06T19:11:47.492Z|00008|ofproto_dpif|INFO|system@ovs-system: VLAN
header stack length probed as 2
> 2019-08-06T19:11:47.492Z|00009|ofproto_dpif|INFO|system@ovs-system: MPLS
label stack length probed as 1
> 2019-08-06T19:11:47.492Z|00010|ofproto_dpif|INFO|system@ovs-system:
Datapath supports truncate action
> 2019-08-06T19:11:47.492Z|00011|ofproto_dpif|INFO|system@ovs-system:
Datapath supports unique flow ids
> 2019-08-06T19:11:47.492Z|00012|ofproto_dpif|INFO|system@ovs-system:
Datapath supports clone action
> 2019-08-06T19:11:47.492Z|00013|ofproto_dpif|INFO|system@ovs-system: Max
sample nesting level probed as 10
> 2019-08-06T19:11:47.492Z|00014|ofproto_dpif|INFO|system@ovs-system:
Datapath supports eventmask in conntrack action
> 2019-08-06T19:11:47.492Z|00015|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_clear action
> 2019-08-06T19:11:47.492Z|00016|ofproto_dpif|INFO|system@ovs-system: Max
dp_hash algorithm probed to be 0
> 2019-08-06T19:11:47.492Z|00017|ofproto_dpif|INFO|system@ovs-system:
Datapath supports check_pkt_len action
> 2019-08-06T19:11:47.492Z|00018|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_state
> 2019-08-06T19:11:47.492Z|00019|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_zone
> 2019-08-06T19:11:47.492Z|00020|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_mark
> 2019-08-06T19:11:47.492Z|00021|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_label
> 2019-08-06T19:11:47.492Z|00022|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_state_nat
> 2019-08-06T19:11:47.492Z|00023|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_orig_tuple
> 2019-08-06T19:11:47.492Z|00024|ofproto_dpif|INFO|system@ovs-system:
Datapath supports ct_orig_tuple6
../tests/system-kmod-testsuite: line 1705: 59097 Alarm clock
exit 1
2019-08-06T19:12:28Z|00001|daemon_unix|WARN|/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid:
open: No such file or directory
ovs-appctl: cannot read pidfile
"/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid"
(No such file or directory)
75. system-traffic.at:3182: 75. conntrack - zone-based timeout policy (
system-traffic.at:3182): FAILED (system-traffic.at:3185)


dball@ubuntu:~/ovs$ lsmod | grep nf
nfnetlink_cttimeout    16384  1
Darrell Ball Aug. 11, 2019, 7:30 p.m. UTC | #5
I did some further testing and ran into another issue; in this case, one, I
did not expect.

I added an additional sending of packets at the end of the test after this
check:

AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
])

Below is new code

dnl Do it again
dnl Send ICMP and UDP traffic
NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING],
[0], [dnl
3 packets transmitted, 3 received, 0% packet loss, time 0ms
])
AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
actions=resubmit(,0)"])

AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
[0], [dnl
icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
])

dnl Wait until the timeout expire.
dnl We intend to wait a bit longer, because conntrack does not recycle the
entry right after it is expired.
sleep 5

AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
])

The test fails bcoz the second time with short timeouts, the conntrack
entries are not cleanup up quickly

@@ -0,0 +1,2 @@
+icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
+udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5



On Tue, Aug 6, 2019 at 12:16 PM Darrell Ball <dlu998@gmail.com> wrote:

>
>
> On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> wrote:
>
>> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote:
>> >
>> >
>> > I did some more testing and found a similar problem as in V1.
>> >
>> > This test can be run successfully once and then fails after that.
>> > Maybe you want to look into that. It is probably related to:
>> >
>> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf
>> > .
>> > nfnetlink_cttimeout    16384  1
>> > .
>> >
>> > Darrell
>> >
>>
>> Thanks for trying out the test.  I can not reproduce the issue that
>> you mentioned on my local VM.
>>
>> Can you provide your kernel version and system-kmod-testsuite.log?
>>
>> Thanks,
>>
>> -Yi-Hung
>>
>
>
> Here it is:
>
> dball@ubuntu:~/ovs$ uname -a
> Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC 2018
> x86_64 x86_64 x86_64 GNU/Linux
>
> dball@ubuntu:~/ovs$ lsb_release -a
> No LSB modules are available.
> Distributor ID: Ubuntu
> Description: Ubuntu 16.04.2 LTS
> Release: 16.04
> Codename: xenial
>
> #                             -*- compilation -*-
> 75. system-traffic.at:3182: testing conntrack - zone-based timeout policy
> ...
> net.netfilter.nf_conntrack_helper = 0
> ../../tests/system-traffic.at:3185: modprobe openvswitch
> ../../tests/system-traffic.at:3185: ovsdb-tool create conf.db
> $abs_top_srcdir/vswitchd/vswitch.ovsschema
> ../../tests/system-traffic.at:3185: ovsdb-server --detach --no-chdir
> --pidfile --log-file --remote=punix:$OVS_RUNDIR/db.sock
> stderr:
> 2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file
> /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log
> ../../tests/system-traffic.at:3185: sed < stderr '
> /vlog|INFO|opened log file/d
> /ovsdb_server|INFO|ovsdb-server (Open vSwitch)/d'
> ../../tests/system-traffic.at:3185: ovs-vsctl --no-wait init
> ../../tests/system-traffic.at:3185: ovs-vswitchd  --detach --no-chdir
> --pidfile --log-file -vvconn -vofproto_dpif -vunixctl
> stderr:
> 2019-08-06T19:11:47Z|00001|vlog|INFO|opened log file
> /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log
> 2019-08-06T19:11:47Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on NUMA
> node 0
> 2019-08-06T19:11:47Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and 1 CPU
> cores
> 2019-08-06T19:11:47Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
> connecting...
> 2019-08-06T19:11:47Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
> connected
> ../../tests/system-traffic.at:3185: sed < stderr '
> /ovs_numa|INFO|Discovered /d
> /vlog|INFO|opened log file/d
> /vswitchd|INFO|ovs-vswitchd (Open vSwitch)/d
> /reconnect|INFO|/d
> /dpif_netlink|INFO|Generic Netlink family .ovs_datapath. does not exist/d
> /ofproto|INFO|using datapath ID/d
> /netdev_linux|INFO|.*device has unknown hardware address family/d
> /ofproto|INFO|datapath ID changed to fedcba9876543210/d
> /dpdk|INFO|DPDK Disabled - Use other_config:dpdk-init to enable/d
> /netlink_socket|INFO|netlink: could not enable listening to all nsid/d
> /probe tc:/d
> /tc: Using policy/d'
> ../../tests/system-traffic.at:3185: ovs-vsctl -- add-br br0 -- set Bridge
> br0
> protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15
> fail-mode=secure  --
> --- /dev/null 2019-02-26 18:50:08.043999906 -0800
> +++
> /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/stderr
> 2019-08-06 12:12:17.489401899 -0700
> @@ -0,0 +1,2 @@
> +2019-08-06T19:12:17Z|00002|fatal_signal|WARN|terminating with signal 14
> (Alarm clock)
> +/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/at-groups/75/test-source:
> line 874: 58958 Alarm clock             ovs-vsctl -- add-br br0 -- set
> Bridge br0
> protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15
> fail-mode=secure --
> ../../tests/system-traffic.at:3185: exit code was 142, expected 0
> ovsdb-server.log:
> > 2019-08-06T19:11:47.418Z|00001|vlog|INFO|opened log file
> /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovsdb-server.log
> > 2019-08-06T19:11:47.420Z|00002|ovsdb_server|INFO|ovsdb-server (Open
> vSwitch) 2.12.90
> > 2019-08-06T19:11:57.433Z|00003|memory|INFO|4504 kB peak resident set
> size after 10.0 seconds
> > 2019-08-06T19:11:57.433Z|00004|memory|INFO|cells:122 monitors:3
> sessions:2
> ovs-vswitchd.log:
> > 2019-08-06T19:11:47.449Z|00001|vlog|INFO|opened log file
> /home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.log
> > 2019-08-06T19:11:47.449Z|00002|ovs_numa|INFO|Discovered 1 CPU cores on
> NUMA node 0
> > 2019-08-06T19:11:47.449Z|00003|ovs_numa|INFO|Discovered 1 NUMA nodes and
> 1 CPU cores
> >
> 2019-08-06T19:11:47.450Z|00004|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
> connecting...
> >
> 2019-08-06T19:11:47.450Z|00005|reconnect|INFO|unix:/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/db.sock:
> connected
> > 2019-08-06T19:11:47.473Z|00006|bridge|INFO|ovs-vswitchd (Open vSwitch)
> 2.12.90
> > 2019-08-06T19:11:47.492Z|00007|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports recirculation
> > 2019-08-06T19:11:47.492Z|00008|ofproto_dpif|INFO|system@ovs-system:
> VLAN header stack length probed as 2
> > 2019-08-06T19:11:47.492Z|00009|ofproto_dpif|INFO|system@ovs-system:
> MPLS label stack length probed as 1
> > 2019-08-06T19:11:47.492Z|00010|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports truncate action
> > 2019-08-06T19:11:47.492Z|00011|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports unique flow ids
> > 2019-08-06T19:11:47.492Z|00012|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports clone action
> > 2019-08-06T19:11:47.492Z|00013|ofproto_dpif|INFO|system@ovs-system: Max
> sample nesting level probed as 10
> > 2019-08-06T19:11:47.492Z|00014|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports eventmask in conntrack action
> > 2019-08-06T19:11:47.492Z|00015|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_clear action
> > 2019-08-06T19:11:47.492Z|00016|ofproto_dpif|INFO|system@ovs-system: Max
> dp_hash algorithm probed to be 0
> > 2019-08-06T19:11:47.492Z|00017|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports check_pkt_len action
> > 2019-08-06T19:11:47.492Z|00018|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_state
> > 2019-08-06T19:11:47.492Z|00019|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_zone
> > 2019-08-06T19:11:47.492Z|00020|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_mark
> > 2019-08-06T19:11:47.492Z|00021|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_label
> > 2019-08-06T19:11:47.492Z|00022|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_state_nat
> > 2019-08-06T19:11:47.492Z|00023|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_orig_tuple
> > 2019-08-06T19:11:47.492Z|00024|ofproto_dpif|INFO|system@ovs-system:
> Datapath supports ct_orig_tuple6
> ../tests/system-kmod-testsuite: line 1705: 59097 Alarm clock
> exit 1
> 2019-08-06T19:12:28Z|00001|daemon_unix|WARN|/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid:
> open: No such file or directory
> ovs-appctl: cannot read pidfile
> "/home/dball/ovs/_gcc/tests/system-kmod-testsuite.dir/075/ovs-vswitchd.pid"
> (No such file or directory)
> 75. system-traffic.at:3182: 75. conntrack - zone-based timeout policy (
> system-traffic.at:3182): FAILED (system-traffic.at:3185)
>
>
> dball@ubuntu:~/ovs$ lsmod | grep nf
> nfnetlink_cttimeout    16384  1
>
>
>
Yi-Hung Wei Aug. 13, 2019, 12:15 a.m. UTC | #6
On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote:
>
> I did some further testing and ran into another issue; in this case, one, I did not expect.
>
> I added an additional sending of packets at the end of the test after this check:
>
> AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
> ])
>
> Below is new code
>
> dnl Do it again
> dnl Send ICMP and UDP traffic
> NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl
> 3 packets transmitted, 3 received, 0% packet loss, time 0ms
> ])
> AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"])
>
> AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl
> icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> ])
>
> dnl Wait until the timeout expire.
> dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired.
> sleep 5
>
> AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
> ])
>
> The test fails bcoz the second time with short timeouts, the conntrack entries are not cleanup up quickly
>
> @@ -0,0 +1,2 @@
> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
>
>
>
> On Tue, Aug 6, 2019 at 12:16 PM Darrell Ball <dlu998@gmail.com> wrote:
>>
>>
>>
>> On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com> wrote:
>>>
>>> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote:
>>> >
>>> >
>>> > I did some more testing and found a similar problem as in V1.
>>> >
>>> > This test can be run successfully once and then fails after that.
>>> > Maybe you want to look into that. It is probably related to:
>>> >
>>> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf
>>> > .
>>> > nfnetlink_cttimeout    16384  1
>>> > .
>>> >
>>> > Darrell
>>> >
>>>
>>> Thanks for trying out the test.  I can not reproduce the issue that
>>> you mentioned on my local VM.
>>>
>>> Can you provide your kernel version and system-kmod-testsuite.log?
>>>
>>> Thanks,
>>>
>>> -Yi-Hung
>>
>>
>>
>> Here it is:
>>
>> dball@ubuntu:~/ovs$ uname -a
>> Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
>>
Thanks for reporting the issue.  I am able to reproduce in similar set
up.  It should be resolved in v3.

Thanks,

-Yi-Hung
Yi-Hung Wei Aug. 13, 2019, 12:22 a.m. UTC | #7
On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote:
>
> I did some further testing and ran into another issue; in this case, one, I did not expect.
>
> I added an additional sending of packets at the end of the test after this check:
>
> AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
> ])
>
> Below is new code
>
> dnl Do it again
> dnl Send ICMP and UDP traffic
> NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl
> 3 packets transmitted, 3 received, 0% packet loss, time 0ms
> ])
> AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"])
>
> AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl
> icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> ])
>
> dnl Wait until the timeout expire.
> dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired.
> sleep 5
>
> AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
> ])
>
> The test fails bcoz the second time with short timeouts, the conntrack entries are not cleanup up quickly
>
> @@ -0,0 +1,2 @@
> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5


Thanks for testing!   This test actually catch a kernel bug when ovs
kernel handles conntrack cache.  It works for me on my ubuntu xenial
VM with 4.4 kernel.

Since this requires upstream kernel change, it will be backported to
OVS once the fix gets upstream.

Thanks,

-Yi-Hung

diff --git a/datapath/conntrack.c b/datapath/conntrack.c
index f85d0a2572f6..ad48b559bcde 100644
--- a/datapath/conntrack.c
+++ b/datapath/conntrack.c
@@ -76,6 +76,7 @@ enum ovs_ct_nat {
 /* Conntrack action context for execution. */
 struct ovs_conntrack_info {
        struct nf_conntrack_helper *helper;
+       struct nf_ct_timeout *nf_ct_timeout;
        struct nf_conntrack_zone zone;
        struct nf_conn *ct;
        u8 commit : 1;
@@ -745,6 +746,13 @@ static bool skb_nfct_cached(struct net *net,
                if (help && rcu_access_pointer(help->helper) != info->helper)
                        return false;
        }
+       if (info->nf_ct_timeout) {
+               struct nf_conn_timeout *timeout_ext;
+
+               timeout_ext = nf_ct_timeout_find(ct);
+               if (!timeout_ext || info->nf_ct_timeout != timeout_ext->timeout)
+                       return false;
+       }
        /* Force conntrack entry direction to the current packet? */
        if (info->force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) {
                /* Delete the conntrack entry if confirmed, else just release
@@ -1704,6 +1712,8 @@ int ovs_ct_copy_action(struct net *net, const
struct nlattr *attr,
                                      ct_info.timeout))
                        pr_info_ratelimited("Failed to associated timeout "
                                            "policy `%s'\n", ct_info.timeout);
+               else
+                       ct_info.nf_ct_timeout =
nf_ct_timeout_find(ct_info.ct)->timeout;
        }

        if (helper) {
Darrell Ball Aug. 13, 2019, 1:11 a.m. UTC | #8
On Mon, Aug 12, 2019 at 5:22 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote:

> On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote:
> >
> > I did some further testing and ran into another issue; in this case,
> one, I did not expect.
> >
> > I added an additional sending of packets at the end of the test after
> this check:
> >
> > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0],
> [dnl
> > ])
> >
> > Below is new code
> >
> > dnl Do it again
> > dnl Send ICMP and UDP traffic
> > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 |
> FORMAT_PING], [0], [dnl
> > 3 packets transmitted, 3 received, 0% packet loss, time 0ms
> > ])
> > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
> actions=resubmit(,0)"])
> >
> > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
> [0], [dnl
> >
> icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> >
> udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> > ])
> >
> > dnl Wait until the timeout expire.
> > dnl We intend to wait a bit longer, because conntrack does not recycle
> the entry right after it is expired.
> > sleep 5
> >
> > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0],
> [dnl
> > ])
> >
> > The test fails bcoz the second time with short timeouts, the conntrack
> entries are not cleanup up quickly
> >
> > @@ -0,0 +1,2 @@
> >
> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> >
> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
>
>
> Thanks for testing!   This test actually catch a kernel bug when ovs
> kernel handles conntrack cache.  It works for me on my ubuntu xenial
> VM with 4.4 kernel.
>
> Since this requires upstream kernel change, it will be backported to
> OVS once the fix gets upstream.
>
> Thanks,
>
> -Yi-Hung
>


Does the below patch fix just the failed timeout policy for sending second
and subsequent packet only
or also the issue of failed test runs after the first run ?

If the issue of subsequent failed test runs is different, what is the fix
you want to use for it ?



>
> diff --git a/datapath/conntrack.c b/datapath/conntrack.c
> index f85d0a2572f6..ad48b559bcde 100644
> --- a/datapath/conntrack.c
> +++ b/datapath/conntrack.c
> @@ -76,6 +76,7 @@ enum ovs_ct_nat {
>  /* Conntrack action context for execution. */
>  struct ovs_conntrack_info {
>         struct nf_conntrack_helper *helper;
> +       struct nf_ct_timeout *nf_ct_timeout;
>         struct nf_conntrack_zone zone;
>         struct nf_conn *ct;
>         u8 commit : 1;
> @@ -745,6 +746,13 @@ static bool skb_nfct_cached(struct net *net,
>                 if (help && rcu_access_pointer(help->helper) !=
> info->helper)
>                         return false;
>         }
> +       if (info->nf_ct_timeout) {
> +               struct nf_conn_timeout *timeout_ext;
> +
> +               timeout_ext = nf_ct_timeout_find(ct);
> +               if (!timeout_ext || info->nf_ct_timeout !=
> timeout_ext->timeout)
> +                       return false;
> +       }
>         /* Force conntrack entry direction to the current packet? */
>         if (info->force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) {
>                 /* Delete the conntrack entry if confirmed, else just
> release
> @@ -1704,6 +1712,8 @@ int ovs_ct_copy_action(struct net *net, const
> struct nlattr *attr,
>                                       ct_info.timeout))
>                         pr_info_ratelimited("Failed to associated timeout "
>                                             "policy `%s'\n",
> ct_info.timeout);
> +               else
> +                       ct_info.nf_ct_timeout =
> nf_ct_timeout_find(ct_info.ct)->timeout;
>         }
>
>         if (helper) {
>
Darrell Ball Aug. 13, 2019, 1:22 a.m. UTC | #9
On Mon, Aug 12, 2019 at 5:15 PM Yi-Hung Wei <yihung.wei@gmail.com> wrote:

> On Sun, Aug 11, 2019 at 12:30 PM Darrell Ball <dlu998@gmail.com> wrote:
> >
> > I did some further testing and ran into another issue; in this case,
> one, I did not expect.
> >
> > I added an additional sending of packets at the end of the test after
> this check:
> >
> > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0],
> [dnl
> > ])
> >
> > Below is new code
> >
> > dnl Do it again
> > dnl Send ICMP and UDP traffic
> > NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 |
> FORMAT_PING], [0], [dnl
> > 3 packets transmitted, 3 received, 0% packet loss, time 0ms
> > ])
> > AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1
> packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000
> actions=resubmit(,0)"])
> >
> > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort],
> [0], [dnl
> >
> icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> >
> udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> > ])
> >
> > dnl Wait until the timeout expire.
> > dnl We intend to wait a bit longer, because conntrack does not recycle
> the entry right after it is expired.
> > sleep 5
> >
> > AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0],
> [dnl
> > ])
> >
> > The test fails bcoz the second time with short timeouts, the conntrack
> entries are not cleanup up quickly
> >
> > @@ -0,0 +1,2 @@
> >
> +icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
> >
> +udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
> >
> >
> >
> > On Tue, Aug 6, 2019 at 12:16 PM Darrell Ball <dlu998@gmail.com> wrote:
> >>
> >>
> >>
> >> On Tue, Aug 6, 2019 at 11:07 AM Yi-Hung Wei <yihung.wei@gmail.com>
> wrote:
> >>>
> >>> On Tue, Aug 6, 2019 at 10:21 AM Darrell Ball <dlu998@gmail.com> wrote:
> >>> >
> >>> >
> >>> > I did some more testing and found a similar problem as in V1.
> >>> >
> >>> > This test can be run successfully once and then fails after that.
> >>> > Maybe you want to look into that. It is probably related to:
> >>> >
> >>> > dball@ubuntu:~/openvswitch/ovs$ lsmod | grep nf
> >>> > .
> >>> > nfnetlink_cttimeout    16384  1
> >>> > .
> >>> >
> >>> > Darrell
> >>> >
> >>>
> >>> Thanks for trying out the test.  I can not reproduce the issue that
> >>> you mentioned on my local VM.
> >>>
> >>> Can you provide your kernel version and system-kmod-testsuite.log?
> >>>
> >>> Thanks,
> >>>
> >>> -Yi-Hung
> >>
> >>
> >>
> >> Here it is:
> >>
> >> dball@ubuntu:~/ovs$ uname -a
> >> Linux ubuntu 4.4.0-119-generic #143-Ubuntu SMP Mon Apr 2 16:08:24 UTC
> 2018 x86_64 x86_64 x86_64 GNU/Linux
> >>
> Thanks for reporting the issue.  I am able to reproduce in similar set
> up.  It should be resolved in v3.
>

What is the fix you want to use for this bug; it must be different from the
second bug you have proposed a patch for
in another response ?

Thanks Darrell


>
> Thanks,
>
> -Yi-Hung
>

Patch
diff mbox series

diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at
index 554a61e9bd95..1bc6f246f426 100644
--- a/tests/system-kmod-macros.at
+++ b/tests/system-kmod-macros.at
@@ -100,6 +100,15 @@  m4_define([CHECK_CONNTRACK_FRAG_OVERLAP],
 #
 m4_define([CHECK_CONNTRACK_NAT])
 
+# CHECK_CONNTRACK_TIMEOUT()
+#
+# Perform requirements checks for running conntrack customized timeout tests.
+#
+m4_define([CHECK_CONNTRACK_TIMEOUT],
+[
+    AT_SKIP_IF([! cat /boot/config-$(uname -r) | grep NF_CONNTRACK_TIMEOUT | grep '=y' > /dev/null])
+])
+
 # CHECK_CT_DPIF_PER_ZONE_LIMIT()
 #
 # Perform requirements checks for running ovs-dpctl ct-[set|get|del]-limits per
@@ -185,3 +194,19 @@  m4_define([OVS_CHECK_KERNEL_EXCL],
     sublevel=$(uname -r | sed -e 's/\./ /g' | awk '{print $ 2}')
     AT_SKIP_IF([ ! ( test $version -lt $1 || ( test $version -eq $1 && test $sublevel -lt $2 ) || test $version -gt $3 || ( test $version -eq $3 && test $sublevel -gt $4 ) ) ])
 ])
+
+# VSCTL_ADD_DATAPATH_TABLE()
+#
+# Create system datapath table "system" for kernel tests in ovsdb
+m4_define([VSCTL_ADD_DATAPATH_TABLE],
+[
+    AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- set Open_vSwitch . datapaths:"system"=@m], [0], [stdout])
+])
+
+# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters])
+#
+# Add zone based timeout policy to kernel datapath
+m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY],
+[
+    AT_CHECK([ovs-vsctl add-zone-tp system $1], [0], [stdout])
+])
diff --git a/tests/system-traffic.at b/tests/system-traffic.at
index 1a04199dcfe9..f4ac8a8f2c06 100644
--- a/tests/system-traffic.at
+++ b/tests/system-traffic.at
@@ -3179,6 +3179,72 @@  NXST_FLOW reply:
 OVS_TRAFFIC_VSWITCHD_STOP
 AT_CLEANUP
 
+AT_SETUP([conntrack - zone-based timeout policy])
+CHECK_CONNTRACK()
+CHECK_CONNTRACK_TIMEOUT()
+OVS_TRAFFIC_VSWITCHD_START()
+
+ADD_NAMESPACES(at_ns0, at_ns1)
+
+ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24")
+ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24")
+
+AT_DATA([flows.txt], [dnl
+priority=1,action=drop
+priority=10,arp,action=normal
+priority=100,in_port=1,ip,action=ct(zone=5, table=1)
+priority=100,in_port=2,ip,action=ct(zone=5, table=1)
+table=1,in_port=2,ip,ct_state=+trk+est,action=1
+table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit,zone=5),2
+table=1,in_port=1,ip,ct_state=+trk+est,action=2
+])
+
+AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt])
+
+dnl Test with default timeout
+dnl The default udp_single and icmp_first timeouts are 30 seconds in
+dnl kernel DP, and 60 seconds in userspace DP.
+
+dnl Send ICMP and UDP traffic
+NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"])
+
+sleep 4
+
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl
+icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
+udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
+])
+
+AT_CHECK([ovs-appctl dpctl/flush-conntrack])
+
+dnl Shorten the udp_single and icmp_first timeout in zone 5
+VSCTL_ADD_DATAPATH_TABLE()
+VSCTL_ADD_ZONE_TIMEOUT_POLICY([zone=5 udp_single=3 icmp_first=3])
+
+dnl Send ICMP and UDP traffic
+NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+AT_CHECK([ovs-ofctl -O OpenFlow13 packet-out br0 "in_port=1 packet=50540000000a50540000000908004500001c000000000011a4cd0a0101010a0101020001000200080000 actions=resubmit(,0)"])
+
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2) | sort], [0], [dnl
+icmp,orig=(src=10.1.1.1,dst=10.1.1.2,id=<cleared>,type=8,code=0),reply=(src=10.1.1.2,dst=10.1.1.1,id=<cleared>,type=0,code=0),zone=5
+udp,orig=(src=10.1.1.1,dst=10.1.1.2,sport=<cleared>,dport=<cleared>),reply=(src=10.1.1.2,dst=10.1.1.1,sport=<cleared>,dport=<cleared>),zone=5
+])
+
+dnl Wait until the timeout expire.
+dnl We intend to wait a bit longer, because conntrack does not recycle the entry right after it is expired.
+sleep 4
+
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(10.1.1.2)], [0], [dnl
+])
+
+OVS_TRAFFIC_VSWITCHD_STOP
+AT_CLEANUP
+
 AT_BANNER([conntrack - L7])
 
 AT_SETUP([conntrack - IPv4 HTTP])
diff --git a/tests/system-userspace-macros.at b/tests/system-userspace-macros.at
index 9d5f3bf419d3..8950a4de7287 100644
--- a/tests/system-userspace-macros.at
+++ b/tests/system-userspace-macros.at
@@ -98,6 +98,16 @@  m4_define([CHECK_CONNTRACK_FRAG_OVERLAP])
 #
 m4_define([CHECK_CONNTRACK_NAT])
 
+# CHECK_CONNTRACK_TIMEOUT()
+#
+# Perform requirements checks for running conntrack customized timeout tests.
+* The userspace datapath does not support this feature yet.
+#
+m4_define([CHECK_CONNTRACK_TIMEOUT],
+[
+    AT_SKIP_IF([:])
+])
+
 # CHECK_CT_DPIF_PER_ZONE_LIMIT()
 #
 # Perform requirements checks for running ovs-dpctl ct-[set|get|del]-limits per
@@ -295,3 +305,19 @@  m4_define([OVS_CHECK_KERNEL_EXCL],
 [
     AT_SKIP_IF([:])
 ])
+
+# VSCTL_ADD_DATAPATH_TABLE()
+#
+# Create datapath table "netdev" for userspace tests in ovsdb
+m4_define([VSCTL_ADD_DATAPATH_TABLE],
+[
+    AT_CHECK([ovs-vsctl -- --id=@m create Datapath datapath_version=0 -- set Open_vSwitch . datapaths:"netdev"=@m], [0], [stdout])
+])
+
+# VSCTL_ADD_ZONE_TIMEOUT_POLICY([parameters])
+#
+# Add zone based timeout policy to userspace datapath
+m4_define([VSCTL_ADD_ZONE_TIMEOUT_POLICY],
+[
+    AT_CHECK([ovs-vsctl add-zone-tp netdev $1], [0], [stdout])
+])