| Message ID | 20210804090129.2432923-1-xsimonar@redhat.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | [ovs-dev,v3,1/2] system-ovn.at: Fix flaky "load-balancing" test. | expand |
| Context | Check | Description |
|---|---|---|
| ovsrobot/apply-robot | success | apply and check: success |
| ovsrobot/github-robot-_Build_and_Test | success | github build: passed |
| ovsrobot/github-robot-_ovn-kubernetes | fail | github build: failed |
On 04/08/2021 10:01, Xavier Simonart wrote: > This test sends requests to LB and checks that each server receives at > least one request. However, even if 20 requests are sent to 3 servers, > there was a possibility that one server did not get any request (0.3%). Could you add a link to a failing test from the CI? maybe with a "Reported-at" tag? > Is there no "fixes" tag? > Signed-off-by: Xavier Simonart <xsimonar@redhat.com> You might want to add yourself to the AUTHORS file. > --- > tests/system-ovn.at | 115 +++++++++++++++++++++++++++++--------------- > 1 file changed, 75 insertions(+), 40 deletions(-) > > diff --git a/tests/system-ovn.at b/tests/system-ovn.at > index 2d21fb262..456bb0599 100644 > --- a/tests/system-ovn.at > +++ b/tests/system-ovn.at > @@ -1449,45 +1449,72 @@ OVS_START_L7([bar2], [http]) > OVS_START_L7([bar3], [http]) > > dnl Should work with the virtual IP 30.0.0.1 address through NAT > -for i in `seq 1 20`; do > - echo Request $i > - NS_CHECK_EXEC([foo1], [wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) > -done > - > dnl Each server should have at least one connection. > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > -tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > +dnl With 20 requests, one server might not receive any connection > +dnl in 0.3% of cases, so run a few times. This is interesting. Does it mean we do not always evenly distribute the flows between backend? Have you checked if it could be a timing issue? For example, how long after the `wget` command does it take for the connections to show-up in conntrack? > + > +OVS_WAIT_UNTIL([ > + for i in `seq 1 20`; do > + echo Request $i; > + ip netns exec foo1 wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; > + done > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > ]) > > dnl Should work with the virtual IP 30.0.0.3 address through NAT > -for i in `seq 1 20`; do > - echo Request $i > - NS_CHECK_EXEC([foo1], [wget 30.0.0.3 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) > -done > - > dnl Each server should have at least one connection. > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > -tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > +OVS_WAIT_UNTIL([ > + for i in `seq 1 20`; do > + echo Request $i; > + ip netns exec foo1 wget 30.0.0.3 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; > + done > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > ]) > > dnl Test load-balancing that includes L4 ports in NAT. > -for i in `seq 1 20`; do > - echo Request $i > - NS_CHECK_EXEC([foo1], [wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) > -done > - > dnl Each server should have at least one connection. > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > +OVS_WAIT_UNTIL([ > + for i in `seq 1 20`; do > + echo Request $i; > + ip netns exec foo1 wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; > + done > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > ]) > > # Configure selection_fields. > @@ -1500,17 +1527,25 @@ OVS_WAIT_UNTIL([ > AT_CHECK([ovs-appctl dpctl/flush-conntrack]) > > dnl Test load-balancing that includes L4 ports in NAT. > -for i in `seq 1 20`; do > - echo Request $i > - NS_CHECK_EXEC([foo1], [wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) > -done > - > dnl Each server should have at least one connection. > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > +OVS_WAIT_UNTIL([ > + for i in `seq 1 20`; do > + echo Request $i; > + ip netns exec foo1 wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; > + done > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > ]) > Could you have wrapped the previous implementation in "OVS_WAIT_UNTIL"? Why did you split the check into ct, ct2, ct3? > AT_CHECK([ovs-appctl dpctl/flush-conntrack]) >
Hi Mark Thanks for your feedback, and sorry for delay as I moved to other tasks... On Thu, 2021-08-05 at 15:57 +0100, Mark Gray wrote: > On 04/08/2021 10:01, Xavier Simonart wrote: > > This test sends requests to LB and checks that each server receives > > at > > least one request. However, even if 20 requests are sent to 3 > > servers, > > there was a possibility that one server did not get any request > > (0.3%). > > Could you add a link to a failing test from the CI? maybe with a > "Reported-at" tag? I will add it, thanks > > > > Is there no "fixes" tag? Not sure. It was implemented that way since it was in OVS, before split between OVN/OVS. So not sure it would help to list any tag > > > Signed-off-by: Xavier Simonart <xsimonar@redhat.com> > > You might want to add yourself to the AUTHORS file. > > --- > > tests/system-ovn.at | 115 +++++++++++++++++++++++++++++----------- > > ---- > > 1 file changed, 75 insertions(+), 40 deletions(-) > > > > diff --git a/tests/system-ovn.at b/tests/system-ovn.at > > index 2d21fb262..456bb0599 100644 > > --- a/tests/system-ovn.at > > +++ b/tests/system-ovn.at > > @@ -1449,45 +1449,72 @@ OVS_START_L7([bar2], [http]) > > OVS_START_L7([bar3], [http]) > > > > dnl Should work with the virtual IP 30.0.0.1 address through NAT > > -for i in `seq 1 20`; do > > - echo Request $i > > - NS_CHECK_EXEC([foo1], [wget 30.0.0.1 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log]) > > -done > > - > > dnl Each server should have at least one connection. > > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | > > \ > > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > +dnl With 20 requests, one server might not receive any connection > > +dnl in 0.3% of cases, so run a few times. > > This is interesting. Does it mean we do not always evenly distribute > the > flows between backend? I think so. LB is based on tuple-hash. As src port changes, backend (might) change, but not in a round robin way. > > Have you checked if it could be a timing issue? For example, how long > after the `wget` command does it take for the connections to show-up > in > conntrack? Yes - quite quickly, after 20 wget I can see 20 connections shown in conntrack - but in some (rare) cases for instance 11 to one backend, 9 to another one, and 0 to the last one. > > > + > > +OVS_WAIT_UNTIL([ > > + for i in `seq 1 20`; do > > + echo Request $i; > > + ip netns exec foo1 wget 30.0.0.1 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log; > > + done > > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > > ]) > > > > dnl Should work with the virtual IP 30.0.0.3 address through NAT > > -for i in `seq 1 20`; do > > - echo Request $i > > - NS_CHECK_EXEC([foo1], [wget 30.0.0.3 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log]) > > -done > > - > > dnl Each server should have at least one connection. > > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | > > \ > > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > +OVS_WAIT_UNTIL([ > > + for i in `seq 1 20`; do > > + echo Request $i; > > + ip netns exec foo1 wget 30.0.0.3 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log; > > + done > > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > > ]) > > > > dnl Test load-balancing that includes L4 ports in NAT. > > -for i in `seq 1 20`; do > > - echo Request $i > > - NS_CHECK_EXEC([foo1], [wget 30.0.0.2:8000 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log]) > > -done > > - > > dnl Each server should have at least one connection. > > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > +OVS_WAIT_UNTIL([ > > + for i in `seq 1 20`; do > > + echo Request $i; > > + ip netns exec foo1 wget 30.0.0.2:8000 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log; > > + done > > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > > ]) > > > > # Configure selection_fields. > > @@ -1500,17 +1527,25 @@ OVS_WAIT_UNTIL([ > > AT_CHECK([ovs-appctl dpctl/flush-conntrack]) > > > > dnl Test load-balancing that includes L4 ports in NAT. > > -for i in `seq 1 20`; do > > - echo Request $i > > - NS_CHECK_EXEC([foo1], [wget 30.0.0.2:8000 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log]) > > -done > > - > > dnl Each server should have at least one connection. > > -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > - > > tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleare > > d>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cl > > eared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) > > +OVS_WAIT_UNTIL([ > > + for i in `seq 1 20`; do > > + echo Request $i; > > + ip netns exec foo1 wget 30.0.0.2:8000 -t 5 -T 1 --retry- > > connrefused -v -o wget$i.log; > > + done > > + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | > > \ > > + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ > > + grep > > "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<clea > > red>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=< > > cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" - > > c) > > + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" > > + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" > > + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" > > + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 > > ]) > > > > Could you have wrapped the previous implementation in > "OVS_WAIT_UNTIL"? > Why did you split the check into ct, ct2, ct3? > > > AT_CHECK([ovs-appctl dpctl/flush-conntrack]) > > > I split it as it was easierr for debugging. I let it in case we wanted to debug later, but I will move the implementation to something similar as before, using OVS_WAIT_FOR_OUTPUT. Thanks Xavier
diff --git a/tests/system-ovn.at b/tests/system-ovn.at index 2d21fb262..456bb0599 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -1449,45 +1449,72 @@ OVS_START_L7([bar2], [http]) OVS_START_L7([bar3], [http]) dnl Should work with the virtual IP 30.0.0.1 address through NAT -for i in `seq 1 20`; do - echo Request $i - NS_CHECK_EXEC([foo1], [wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) -done - dnl Each server should have at least one connection. -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl -tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) +dnl With 20 requests, one server might not receive any connection +dnl in 0.3% of cases, so run a few times. + +OVS_WAIT_UNTIL([ + for i in `seq 1 20`; do + echo Request $i; + ip netns exec foo1 wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; + done + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 ]) dnl Should work with the virtual IP 30.0.0.3 address through NAT -for i in `seq 1 20`; do - echo Request $i - NS_CHECK_EXEC([foo1], [wget 30.0.0.3 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) -done - dnl Each server should have at least one connection. -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl -tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) +OVS_WAIT_UNTIL([ + for i in `seq 1 20`; do + echo Request $i; + ip netns exec foo1 wget 30.0.0.3 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; + done + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.3) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.3,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 ]) dnl Test load-balancing that includes L4 ports in NAT. -for i in `seq 1 20`; do - echo Request $i - NS_CHECK_EXEC([foo1], [wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) -done - dnl Each server should have at least one connection. -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) +OVS_WAIT_UNTIL([ + for i in `seq 1 20`; do + echo Request $i; + ip netns exec foo1 wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; + done + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 ]) # Configure selection_fields. @@ -1500,17 +1527,25 @@ OVS_WAIT_UNTIL([ AT_CHECK([ovs-appctl dpctl/flush-conntrack]) dnl Test load-balancing that includes L4 ports in NAT. -for i in `seq 1 20`; do - echo Request $i - NS_CHECK_EXEC([foo1], [wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) -done - dnl Each server should have at least one connection. -AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ -sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) -tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>) +OVS_WAIT_UNTIL([ + for i in `seq 1 20`; do + echo Request $i; + ip netns exec foo1 wget 30.0.0.2:8000 -t 5 -T 1 --retry-connrefused -v -o wget$i.log; + done + ct1=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.2,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct2=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + ct3=$(ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.2) | \ + sed -e 's/zone=[[0-9]]*/zone=<cleared>/'| \ + grep "tcp,orig=(src=192.168.1.2,dst=30.0.0.2,sport=<cleared>,dport=<cleared>),reply=(src=172.16.1.4,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,labels=0x2,protoinfo=(state=<cleared>)" -c) + test $ct1 -gt 0 && echo "There are flows for 172.16.1.2" + test $ct2 -gt 0 && echo "There are flows for 172.16.1.3" + test $ct3 -gt 0 && echo "There are flows for 172.16.1.4" + test $ct1 -gt 0 && test $ct2 -gt 0 && test $ct3 -gt 0 ]) AT_CHECK([ovs-appctl dpctl/flush-conntrack])
This test sends requests to LB and checks that each server receives at least one request. However, even if 20 requests are sent to 3 servers, there was a possibility that one server did not get any request (0.3%). Signed-off-by: Xavier Simonart <xsimonar@redhat.com> --- tests/system-ovn.at | 115 +++++++++++++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 40 deletions(-)