[ovs-dev] tests: fixed multiple flaky tests

- Multiple "ovn-controller" tests
- I-P handle northd_internal_version change
- 2 HVs, 1 lport/HV, localport ports
- 1 LR with distributed router gateway port
- send gratuitous arp for NAT rules on distributed router
- send gratuitous arp on localnet
- 4 HV, 3 LS, 2 LR, packet test with HA distributed router gateway port
- external logical port
- 3 HVs, 1 VIFs/HV, 1 GW, 1 LS
- VLAN transparency, passthru=true, ARP responder disabled

Some other minor fixes:
- removed multiple definition of OVN_CHECK_PACKETS_UNIQ
- Added a check on ovs-pcap.in to prevent, in case of non existing input,
  filling the logs with "detected unhandled Python exception"

Signed-off-by: Xavier Simonart <xsimonar@redhat.com>
---
 tests/ovn-controller.at |  30 +++++-----
 tests/ovn.at            | 125 +++++++++++++++++++++++++---------------
 2 files changed, 96 insertions(+), 59 deletions(-)

Thanks for this, Xavier. I really like seeing new macros being added to 
the testsuite to increase reliability.

Acked-by: Mark Michelson <mmichels@redhat.com>

On 5/12/22 05:23, Xavier Simonart wrote:
> - Multiple "ovn-controller" tests
> - I-P handle northd_internal_version change
> - 2 HVs, 1 lport/HV, localport ports
> - 1 LR with distributed router gateway port
> - send gratuitous arp for NAT rules on distributed router
> - send gratuitous arp on localnet
> - 4 HV, 3 LS, 2 LR, packet test with HA distributed router gateway port
> - external logical port
> - 3 HVs, 1 VIFs/HV, 1 GW, 1 LS
> - VLAN transparency, passthru=true, ARP responder disabled
> 
> Some other minor fixes:
> - removed multiple definition of OVN_CHECK_PACKETS_UNIQ
> - Added a check on ovs-pcap.in to prevent, in case of non existing input,
>    filling the logs with "detected unhandled Python exception"
> 
> Signed-off-by: Xavier Simonart <xsimonar@redhat.com>
> ---
>   tests/ovn-controller.at |  30 +++++-----
>   tests/ovn.at            | 125 +++++++++++++++++++++++++---------------
>   2 files changed, 96 insertions(+), 59 deletions(-)
> 
> diff --git a/tests/ovn-controller.at b/tests/ovn-controller.at
> index 88ec2f269..3339e58ce 100644
> --- a/tests/ovn-controller.at
> +++ b/tests/ovn-controller.at
> @@ -901,7 +901,7 @@ read_counter() {
>   }
>   
>   ovn-nbctl create address_set name=as1
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1' drop
>   
>   # Add IPs to as1 for 10 times, 1 IP each time.
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -1074,7 +1074,7 @@ read_counter() {
>   }
>   
>   ovn-nbctl create address_set name=as1
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1 && tcp && tcp.dst == {111, 222, 333}' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1 && tcp && tcp.dst == {111, 222, 333}' drop
>   
>   # Add IPs to as1 for 10 times, 1 IP each time.
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -1269,7 +1269,7 @@ read_counter() {
>   
>   ovn-nbctl create address_set name=as1
>   ovn-nbctl create address_set name=as2
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1 && ip4.dst == $as2' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1 && ip4.dst == $as2' drop
>   
>   # Add IPs to as1 and as2, with some of the IPs overlapping
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -1426,7 +1426,7 @@ ovn-nbctl create address_set name=as1
>   ovn-nbctl create address_set name=as2
>   
>   # OR on different fields
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && (ip4.src == $as1 || ip4.dst == $as2)' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && (ip4.src == $as1 || ip4.dst == $as2)' drop
>   
>   # Add IPs to as1 and as2, with some of the IPs overlapping
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -1522,7 +1522,7 @@ ovn-nbctl create address_set name=as1
>   ovn-nbctl create address_set name=as2
>   
>   # OR on the same field
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == {$as1, $as2}' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == {$as1, $as2}' drop
>   
>   # Add IPs to as1 and as2, with some of the IPs overlapping
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -1622,7 +1622,7 @@ read_counter() {
>   }
>   
>   ovn-nbctl create address_set name=as1
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1 && ip4.dst == $as1' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip4.src == $as1 && ip4.dst == $as1' drop
>   
>   # Add IPs to as1 for 10 times, 1 IP each time.
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -1926,7 +1926,7 @@ read_counter() {
>   }
>   
>   ovn-nbctl create address_set name=as1
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && eth.src == $as1' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && eth.src == $as1' drop
>   
>   # Add MACs to as1 for 5 times.
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -2007,7 +2007,7 @@ read_counter() {
>   }
>   
>   ovn-nbctl create address_set name=as1
> -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip6.src == $as1' drop
> +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport == "ls1-lp1" && ip6.src == $as1' drop
>   
>   # Add IPs to as1 for 5 times, 1 IP each time.
>   reprocess_count_old=$(read_counter consider_logical_flow)
> @@ -2093,11 +2093,15 @@ AT_CHECK([echo $(($lflow_run_new - $lflow_run_old))], [0], [0
>   # The below steps should cause northd_internal_version change twice. One by
>   # ovn-sbctl, and the other by ovn-northd to change it back.
>   
> -lflow_run_old=$(read_counter lflow_run)
> -check ovn-sbctl set SB_Global . options:northd_internal_version=foo
> -check ovn-nbctl --wait=hv sync
> -lflow_run_new=$(read_counter lflow_run)
> -AT_CHECK([echo $(($lflow_run_new - $lflow_run_old))], [0], [2
> +# In some cases, both changes are catched by ovn-controller in the same run,
> +# ending up in no change.
> +
> +OVS_WAIT_UNTIL([
> +    lflow_run_old=$(read_counter lflow_run)
> +    check ovn-sbctl set SB_Global . options:northd_internal_version=foo
> +    check ovn-nbctl --wait=hv sync
> +    lflow_run_new=$(read_counter lflow_run)
> +    test x"$(($lflow_run_new - $lflow_run_old))" = x2
>   ])
>   
>   OVN_CLEANUP([hv1])
> diff --git a/tests/ovn.at b/tests/ovn.at
> index 403fbc85f..38ea31232 100644
> --- a/tests/ovn.at
> +++ b/tests/ovn.at
> @@ -63,16 +63,52 @@ m4_divert_text([PREPARE_TESTS],
>           test x"$(sort $rcv_text | comm -2 -3 expout -)" = "x"],
>          [dump_diff__ "$rcv_pcap" "$exp_text"])
>      }
> +   ovn_wait_packets_uniq__ () {
> +     echo "$3: waiting for packets from $2 at $1:"
> +     rcv_pcap=$1
> +     rcv_text=`echo "$rcv_pcap.packets" | sed 's/\.pcap//'`
> +     exp_text=$2
> +     OVS_WAIT_UNTIL(
> +       [$PYTHON "$ovs_srcdir/utilities/ovs-pcap.in" $rcv_pcap > $rcv_text
> +        sort $exp_text > expout
> +        test x"$(sort $rcv_text | uniq | comm -3 expout -)" = "x"],
> +       [dump_diff__ "$rcv_pcap" "$exp_text"])
> +   }
> +
> +   ovn_wait_patch_port_flows () {
> +     patch_port=$1
> +     hv=$2
> +     echo "$3: waiting for flows for $patch_port on $hv"
> +     # Patch port might be created after ports are reported up
> +     OVS_WAIT_UNTIL([
> +         test 1 = $(as $hv ovs-vsctl show | grep "Port $patch_port" | wc -l)
> +     ])
> +     # Wait for a flow outputing to patch port
> +     OVS_WAIT_UNTIL([
> +         hv_patch_ofport=$(as $hv ovs-vsctl --bare --columns ofport find Interface name=$patch_port)
> +         echo "$patch_port=$hv_patch_ofport"
> +         test 1 = $(as $hv ovs-ofctl dump-flows br-int | grep -c "output:$hv_patch_ofport")
> +     ])
> +   }
> +
> +   ovn_wait_remote_output_flows () {
> +     hv1=$1
> +     hv2=$2
> +     echo "$3: waiting for flows for remote output on $hv1"
> +     # Wait for a flow outputing  to remote output
> +     OVS_WAIT_UNTIL([
> +         ofport=$(as $hv1 ovs-vsctl --bare --columns ofport find Interface name=ovn-${hv2}-0)
> +         echo "tunnel port=$ofport"
> +         test 1 -le $(as $hv1 ovs-ofctl dump-flows br-int | grep -c "output:$ofport")
> +     ])
> +   }
>   ])
>   
>   m4_define([OVN_CHECK_PACKETS],
> -  [ovn_check_packets__ "$1" "$2" "__file__:__line__"
> +  [AT_CHECK([$PYTHON "$ovs_srcdir/utilities/ovs-pcap.in" $1 ], [0], [ignore])
> +   ovn_check_packets__ "$1" "$2" "__file__:__line__"
>      AT_CHECK([sort $rcv_text], [0], [expout], [ignore], [dump_diff__ "$1" "$2"])])
>   
> -m4_define([OVN_CHECK_PACKETS_UNIQ],
> -  [ovn_check_packets__ "$1" "$2" "__file__:__line__"
> -   AT_CHECK([sort $rcv_text | uniq], [0], [expout])])
> -
>   m4_define([OVN_CHECK_PACKETS_REMOVE_BROADCAST],
>     [ovn_check_packets_remove_broadcast__ "$1" "$2" "__file__:__line__"
>      AT_CHECK([sort $rcv_text], [0], [expout], [ignore], [dump_diff__ "$1" "$2"])])
> @@ -83,8 +119,14 @@ m4_define([OVN_CHECK_PACKETS_CONTAIN],
>   # OVN_CHECK_PACKETS_UNIQ succeeds if some expected packets are duplicated.
>   # It fails if unexpected packets are received.
>   m4_define([OVN_CHECK_PACKETS_UNIQ],
> -  [ovn_check_packets__ "$1" "$2" "__file__:__line__"
> -   AT_CHECK([sort $rcv_text | uniq], [0], [expout])])
> +  [ovn_wait_packets_uniq__ "$1" "$2" "__file__:__line__"])
> +
> +m4_define([OVN_WAIT_PATCH_PORT_FLOWS],
> +  [ovn_wait_patch_port_flows "$1" "$2" "__file__:__line__"])
> +
> +m4_define([OVN_WAIT_REMOTE_OUTPUT_FLOWS],
> +  [ovn_wait_remote_output_flows "$1" "$2" "__file__:__line__"])
> +
>   
>   AT_BANNER([OVN components])
>   
> @@ -3427,6 +3469,12 @@ done
>   
>   wait_for_ports_up
>   
> +# Remote output flows are setup whe pb of remote is received
> +# Hence they can be setup after both ports have been reported up.
> +OVN_WAIT_REMOTE_OUTPUT_FLOWS(["hv-1"],["hv-2"])
> +OVN_WAIT_REMOTE_OUTPUT_FLOWS(["hv-2"],["hv-1"])
> +
> +
>   ovn-sbctl dump-flows ls > lsflows
>   AT_CAPTURE_FILE([lsflows])
>   
> @@ -3560,16 +3608,8 @@ for i in 1 2; do
>       OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up lsp$i` = xup])
>   
>       # Patch port might be created after ports are reported up
> -    OVS_WAIT_UNTIL([
> -        test 1 = `ovs-vsctl show | \
> -        grep "Port patch-br-int-to-ln-100" | wc -l`
> -    ])
> -
>       # Wait for a flow outputing to patch port
> -    hv_patch_ofport=$(ovs-vsctl --bare --columns ofport find Interface name=patch-br-int-to-ln-100)
> -    OVS_WAIT_UNTIL([
> -        test 1 = $(ovs-ofctl dump-flows br-int | grep -c "output:$hv_patch_ofport")
> -    ])
> +    OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln-100"], ["hv-$i"])
>   done
>   
>   test_packet() {
> @@ -3637,16 +3677,8 @@ for i in 1 2; do
>       wait_for_ports_up lsp$i
>   
>       # Patch port might be created after ports are reported up
> -    OVS_WAIT_UNTIL([
> -        test 1 = `ovs-vsctl show | \
> -        grep "Port patch-br-int-to-ln-100" | wc -l`
> -    ])
> -
>       # Wait for a flow outputing to patch port
> -    hv_patch_ofport=$(ovs-vsctl --bare --columns ofport find Interface name=patch-br-int-to-ln-100)
> -    OVS_WAIT_UNTIL([
> -        test 1 = $(ovs-ofctl dump-flows br-int | grep -c "output:$hv_patch_ofport")
> -    ])
> +    OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln-100"], ["hv-$i"])
>   done
>   # create taps on fabric to check vlan encapsulation there
>   for i in 1 2; do
> @@ -3731,16 +3763,8 @@ for i in 1 2; do
>       OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up lsp$i` = xup])
>   
>       # Patch port might be created after ports are reported up
> -    OVS_WAIT_UNTIL([
> -        test 1 = `ovs-vsctl show | \
> -        grep "Port patch-br-int-to-ln" | wc -l`
> -    ])
> -
>       # Wait for a flow outputing to patch port
> -    hv_patch_ofport=$(ovs-vsctl --bare --columns ofport find Interface name=patch-br-int-to-ln)
> -    OVS_WAIT_UNTIL([
> -        test 1 = $(ovs-ofctl dump-flows br-int | grep -c "output:$hv_patch_ofport")
> -    ])
> +    OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln"], ["hv-$i"])
>   done
>   
>   for i in 1 2; do
> @@ -4123,6 +4147,9 @@ net_attach n2 br-phys
>   
>   ovs-vsctl add-port br-phys vif3 -- set Interface vif3 options:tx_pcap=hv3/vif3-tx.pcap options:rxq_pcap=hv3/vif3-rx.pcap ofport-request=1
>   
> +# vtep is quite slow setting up all flows
> +OVS_WAIT_UNTIL([test `as vtep ovs-ofctl dump-flows br-vtep_vtep_ls1 | grep "priority=0" | grep "output" | wc -l` -eq 1])
> +
>   # Pre-populate the hypervisors' ARP tables so that we don't lose any
>   # packets for ARP resolution (native tunneling doesn't queue packets
>   # for ARP resolution).
> @@ -5044,7 +5071,6 @@ for i in 1 2 3; do
>       as hv$i
>       ovs-vsctl add-br br-phys
>       ovn_attach n1 br-phys 192.168.0.$i
> -
>       for j in 1 2 3; do
>           ovs-vsctl add-port br-int vif$i$j -- set Interface vif$i$j external-ids:iface-id=lp$i$j options:tx_pcap=hv$i/vif$i$j-tx.pcap options:rxq_pcap=hv$i/vif$i$j-rx.pcap ofport-request=$i$j
>           ovn-nbctl lsp-add lsw0 lp$i$j
> @@ -11305,7 +11331,7 @@ grep actions=mod_dl_dst:f0:00:00:01:02:04 | wc -l` -eq 1
>       # Resend packet from foo1 to outside1
>       check as hv1 ovs-appctl netdev-dummy/receive hv1-vif1 $packet
>   
> -    OVN_CHECK_PACKETS([ext1/vif1-tx.pcap], [ext1-vif1.expected])
> +    OVN_CHECK_PACKETS_UNIQ([ext1/vif1-tx.pcap], [ext1-vif1.expected])
>       $PYTHON "$ovs_srcdir/utilities/ovs-pcap.in" $active_gw/br-phys_n1-tx.pcap  > packets
>       cat packets | grep $expected > exp
>       cat packets | grep $exp_gw_ip_garp | head -1 >> exp
> @@ -11565,8 +11591,7 @@ OVN_CHECK_PACKETS_UNIQ([hv3/vif1-tx.pcap], [hv3-vif1.expected])
>   as hv2 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
>   
>   # Wait until the patch ports are created in hv2 to connect br-int to br-phys
> -OVS_WAIT_UNTIL([test 1 = `as hv2 ovs-vsctl show | \
> -grep "Port patch-br-int-to-ln-alice" | wc -l`])
> +OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln-alice"], ["hv2"])
>   
>   dnl Allow some time for ovn-controller to catch up.
>   wait_for_ports_up
> @@ -11787,8 +11812,7 @@ ovs-vsctl -- add-port br-int hv3-vif2 -- \
>   AT_CHECK([as hv3 ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=physnet1:br-phys])
>   
>   # Wait until the patch ports are created in hv3 to connect br-int to br-phys
> -OVS_WAIT_UNTIL([test 1 = `as hv3 ovs-vsctl show | \
> -grep "Port patch-br-int-to-ln_port" | wc -l`])
> +OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln_port"], ["hv3"])
>   
>   # Re-add nat-addresses option
>   ovn-nbctl lsp-set-options lrp0-rp router-port=lrp0 nat-addresses="router"
> @@ -12536,8 +12560,8 @@ for i in 1 2; do
>       ovn_attach n1 br-phys 192.168.0.$i
>       ovs-vsctl add-port br-int vif01 -- \
>           set Interface vif01 external-ids:iface-id=lp01 \
> -                              options:tx_pcap=hv${i}/vif01-tx.pcap \
> -                              options:rxq_pcap=hv${i}/vif01-rx.pcap \
> +                              options:tx_pcap=hv${i}/vif${i}0-tx.pcap \
> +                              options:rxq_pcap=hv${i}/vif${i}0-rx.pcap \
>                                 ofport-request=${i}0
>   
>       ovs-vsctl add-port br-int vif${i}1 -- \
> @@ -12606,8 +12630,14 @@ test_packet() {
>       fi
>       vif=vif$inport
>       as $hv ovs-appctl netdev-dummy/receive $vif $packet
> -    if test $eout != drop; then
> -        echo $packet >> ${eout#lp}.expected
> +
> +    # For packets sent to localport, differentiate localport per hypervisor
> +    if test $eout = lp01; then
> +        echo $packet >> ${hv#hv}0.expected
> +    else
> +        if test $eout != drop; then
> +            echo $packet >> ${eout#lp}.expected
> +        fi
>       fi
>   }
>   
> @@ -16729,6 +16759,10 @@ grep controller | grep tp_src=546 | grep \
>   "ae.70.00.00.00.00.00.00.00.00.00.00.00.00.00.06" | wc -l], [0], [0
>   ])
>   
> +wait_row_count Chassis 1 name=hv1
> +wait_row_count Chassis 1 name=hv2
> +wait_row_count Chassis 1 name=hv3
> +
>   hv1_uuid=$(fetch_column Chassis _uuid name=hv1)
>   hv2_uuid=$(fetch_column Chassis _uuid name=hv2)
>   hv3_uuid=$(fetch_column Chassis _uuid name=hv3)
> @@ -17259,7 +17293,7 @@ reset_pcap_file hv2-ext2 hv2/ext2
>   reset_pcap_file br-phys_n1 hv2/br-phys_n1
>   reset_pcap_file br-phys hv2/br-phys
>   
> -as hv2
> +as hv3
>   ovs-vsctl show
>   reset_pcap_file hv3-ext3 hv3/ext3
>   reset_pcap_file br-phys_n1 hv3/br-phys_n1
> @@ -17335,7 +17369,6 @@ AT_CHECK([as hv2 ovs-ofctl dump-flows br-int \
>   table=31,dl_src=f0:00:00:00:00:03,dl_dst=a0:10:00:00:00:01 | \
>   grep -c "actions=drop"], [0], [1
>   ])
> -
>   # Stop ovn-controllers on hv1 and hv3.
>   as hv1 ovn-appctl -t ovn-controller exit
>   as hv3 ovn-appctl -t ovn-controller exit
>

On Tue, May 17, 2022 at 1:47 PM Mark Michelson <mmichels@redhat.com> wrote:

> Thanks for this, Xavier. I really like seeing new macros being added to
> the testsuite to increase reliability.
>
> Acked-by: Mark Michelson <mmichels@redhat.com>
>

Thanks for improving the test cases.

Applied to the main branch.

Numan

>
> On 5/12/22 05:23, Xavier Simonart wrote:
> > - Multiple "ovn-controller" tests
> > - I-P handle northd_internal_version change
> > - 2 HVs, 1 lport/HV, localport ports
> > - 1 LR with distributed router gateway port
> > - send gratuitous arp for NAT rules on distributed router
> > - send gratuitous arp on localnet
> > - 4 HV, 3 LS, 2 LR, packet test with HA distributed router gateway port
> > - external logical port
> > - 3 HVs, 1 VIFs/HV, 1 GW, 1 LS
> > - VLAN transparency, passthru=true, ARP responder disabled
> >
> > Some other minor fixes:
> > - removed multiple definition of OVN_CHECK_PACKETS_UNIQ
> > - Added a check on ovs-pcap.in to prevent, in case of non existing
> input,
> >    filling the logs with "detected unhandled Python exception"
> >
> > Signed-off-by: Xavier Simonart <xsimonar@redhat.com>
> > ---
> >   tests/ovn-controller.at |  30 +++++-----
> >   tests/ovn.at            | 125 +++++++++++++++++++++++++---------------
> >   2 files changed, 96 insertions(+), 59 deletions(-)
> >
> > diff --git a/tests/ovn-controller.at b/tests/ovn-controller.at
> > index 88ec2f269..3339e58ce 100644
> > --- a/tests/ovn-controller.at
> > +++ b/tests/ovn-controller.at
> > @@ -901,7 +901,7 @@ read_counter() {
> >   }
> >
> >   ovn-nbctl create address_set name=as1
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> ip4.src == $as1' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && ip4.src == $as1' drop
> >
> >   # Add IPs to as1 for 10 times, 1 IP each time.
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -1074,7 +1074,7 @@ read_counter() {
> >   }
> >
> >   ovn-nbctl create address_set name=as1
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> ip4.src == $as1 && tcp && tcp.dst == {111, 222, 333}' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && ip4.src == $as1 && tcp && tcp.dst == {111, 222, 333}' drop
> >
> >   # Add IPs to as1 for 10 times, 1 IP each time.
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -1269,7 +1269,7 @@ read_counter() {
> >
> >   ovn-nbctl create address_set name=as1
> >   ovn-nbctl create address_set name=as2
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> ip4.src == $as1 && ip4.dst == $as2' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && ip4.src == $as1 && ip4.dst == $as2' drop
> >
> >   # Add IPs to as1 and as2, with some of the IPs overlapping
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -1426,7 +1426,7 @@ ovn-nbctl create address_set name=as1
> >   ovn-nbctl create address_set name=as2
> >
> >   # OR on different fields
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> (ip4.src == $as1 || ip4.dst == $as2)' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && (ip4.src == $as1 || ip4.dst == $as2)' drop
> >
> >   # Add IPs to as1 and as2, with some of the IPs overlapping
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -1522,7 +1522,7 @@ ovn-nbctl create address_set name=as1
> >   ovn-nbctl create address_set name=as2
> >
> >   # OR on the same field
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> ip4.src == {$as1, $as2}' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && ip4.src == {$as1, $as2}' drop
> >
> >   # Add IPs to as1 and as2, with some of the IPs overlapping
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -1622,7 +1622,7 @@ read_counter() {
> >   }
> >
> >   ovn-nbctl create address_set name=as1
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> ip4.src == $as1 && ip4.dst == $as1' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && ip4.src == $as1 && ip4.dst == $as1' drop
> >
> >   # Add IPs to as1 for 10 times, 1 IP each time.
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -1926,7 +1926,7 @@ read_counter() {
> >   }
> >
> >   ovn-nbctl create address_set name=as1
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> eth.src == $as1' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && eth.src == $as1' drop
> >
> >   # Add MACs to as1 for 5 times.
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -2007,7 +2007,7 @@ read_counter() {
> >   }
> >
> >   ovn-nbctl create address_set name=as1
> > -check ovn-nbctl acl-add ls1 to-lport 100 'outport == "ls1-lp1" &&
> ip6.src == $as1' drop
> > +check ovn-nbctl --wait=hv acl-add ls1 to-lport 100 'outport ==
> "ls1-lp1" && ip6.src == $as1' drop
> >
> >   # Add IPs to as1 for 5 times, 1 IP each time.
> >   reprocess_count_old=$(read_counter consider_logical_flow)
> > @@ -2093,11 +2093,15 @@ AT_CHECK([echo $(($lflow_run_new -
> $lflow_run_old))], [0], [0
> >   # The below steps should cause northd_internal_version change twice.
> One by
> >   # ovn-sbctl, and the other by ovn-northd to change it back.
> >
> > -lflow_run_old=$(read_counter lflow_run)
> > -check ovn-sbctl set SB_Global . options:northd_internal_version=foo
> > -check ovn-nbctl --wait=hv sync
> > -lflow_run_new=$(read_counter lflow_run)
> > -AT_CHECK([echo $(($lflow_run_new - $lflow_run_old))], [0], [2
> > +# In some cases, both changes are catched by ovn-controller in the same
> run,
> > +# ending up in no change.
> > +
> > +OVS_WAIT_UNTIL([
> > +    lflow_run_old=$(read_counter lflow_run)
> > +    check ovn-sbctl set SB_Global . options:northd_internal_version=foo
> > +    check ovn-nbctl --wait=hv sync
> > +    lflow_run_new=$(read_counter lflow_run)
> > +    test x"$(($lflow_run_new - $lflow_run_old))" = x2
> >   ])
> >
> >   OVN_CLEANUP([hv1])
> > diff --git a/tests/ovn.at b/tests/ovn.at
> > index 403fbc85f..38ea31232 100644
> > --- a/tests/ovn.at
> > +++ b/tests/ovn.at
> > @@ -63,16 +63,52 @@ m4_divert_text([PREPARE_TESTS],
> >           test x"$(sort $rcv_text | comm -2 -3 expout -)" = "x"],
> >          [dump_diff__ "$rcv_pcap" "$exp_text"])
> >      }
> > +   ovn_wait_packets_uniq__ () {
> > +     echo "$3: waiting for packets from $2 at $1:"
> > +     rcv_pcap=$1
> > +     rcv_text=`echo "$rcv_pcap.packets" | sed 's/\.pcap//'`
> > +     exp_text=$2
> > +     OVS_WAIT_UNTIL(
> > +       [$PYTHON "$ovs_srcdir/utilities/ovs-pcap.in" $rcv_pcap >
> $rcv_text
> > +        sort $exp_text > expout
> > +        test x"$(sort $rcv_text | uniq | comm -3 expout -)" = "x"],
> > +       [dump_diff__ "$rcv_pcap" "$exp_text"])
> > +   }
> > +
> > +   ovn_wait_patch_port_flows () {
> > +     patch_port=$1
> > +     hv=$2
> > +     echo "$3: waiting for flows for $patch_port on $hv"
> > +     # Patch port might be created after ports are reported up
> > +     OVS_WAIT_UNTIL([
> > +         test 1 = $(as $hv ovs-vsctl show | grep "Port $patch_port" |
> wc -l)
> > +     ])
> > +     # Wait for a flow outputing to patch port
> > +     OVS_WAIT_UNTIL([
> > +         hv_patch_ofport=$(as $hv ovs-vsctl --bare --columns ofport
> find Interface name=$patch_port)
> > +         echo "$patch_port=$hv_patch_ofport"
> > +         test 1 = $(as $hv ovs-ofctl dump-flows br-int | grep -c
> "output:$hv_patch_ofport")
> > +     ])
> > +   }
> > +
> > +   ovn_wait_remote_output_flows () {
> > +     hv1=$1
> > +     hv2=$2
> > +     echo "$3: waiting for flows for remote output on $hv1"
> > +     # Wait for a flow outputing  to remote output
> > +     OVS_WAIT_UNTIL([
> > +         ofport=$(as $hv1 ovs-vsctl --bare --columns ofport find
> Interface name=ovn-${hv2}-0)
> > +         echo "tunnel port=$ofport"
> > +         test 1 -le $(as $hv1 ovs-ofctl dump-flows br-int | grep -c
> "output:$ofport")
> > +     ])
> > +   }
> >   ])
> >
> >   m4_define([OVN_CHECK_PACKETS],
> > -  [ovn_check_packets__ "$1" "$2" "__file__:__line__"
> > +  [AT_CHECK([$PYTHON "$ovs_srcdir/utilities/ovs-pcap.in" $1 ], [0],
> [ignore])
> > +   ovn_check_packets__ "$1" "$2" "__file__:__line__"
> >      AT_CHECK([sort $rcv_text], [0], [expout], [ignore], [dump_diff__
> "$1" "$2"])])
> >
> > -m4_define([OVN_CHECK_PACKETS_UNIQ],
> > -  [ovn_check_packets__ "$1" "$2" "__file__:__line__"
> > -   AT_CHECK([sort $rcv_text | uniq], [0], [expout])])
> > -
> >   m4_define([OVN_CHECK_PACKETS_REMOVE_BROADCAST],
> >     [ovn_check_packets_remove_broadcast__ "$1" "$2" "__file__:__line__"
> >      AT_CHECK([sort $rcv_text], [0], [expout], [ignore], [dump_diff__
> "$1" "$2"])])
> > @@ -83,8 +119,14 @@ m4_define([OVN_CHECK_PACKETS_CONTAIN],
> >   # OVN_CHECK_PACKETS_UNIQ succeeds if some expected packets are
> duplicated.
> >   # It fails if unexpected packets are received.
> >   m4_define([OVN_CHECK_PACKETS_UNIQ],
> > -  [ovn_check_packets__ "$1" "$2" "__file__:__line__"
> > -   AT_CHECK([sort $rcv_text | uniq], [0], [expout])])
> > +  [ovn_wait_packets_uniq__ "$1" "$2" "__file__:__line__"])
> > +
> > +m4_define([OVN_WAIT_PATCH_PORT_FLOWS],
> > +  [ovn_wait_patch_port_flows "$1" "$2" "__file__:__line__"])
> > +
> > +m4_define([OVN_WAIT_REMOTE_OUTPUT_FLOWS],
> > +  [ovn_wait_remote_output_flows "$1" "$2" "__file__:__line__"])
> > +
> >
> >   AT_BANNER([OVN components])
> >
> > @@ -3427,6 +3469,12 @@ done
> >
> >   wait_for_ports_up
> >
> > +# Remote output flows are setup whe pb of remote is received
> > +# Hence they can be setup after both ports have been reported up.
> > +OVN_WAIT_REMOTE_OUTPUT_FLOWS(["hv-1"],["hv-2"])
> > +OVN_WAIT_REMOTE_OUTPUT_FLOWS(["hv-2"],["hv-1"])
> > +
> > +
> >   ovn-sbctl dump-flows ls > lsflows
> >   AT_CAPTURE_FILE([lsflows])
> >
> > @@ -3560,16 +3608,8 @@ for i in 1 2; do
> >       OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up lsp$i` = xup])
> >
> >       # Patch port might be created after ports are reported up
> > -    OVS_WAIT_UNTIL([
> > -        test 1 = `ovs-vsctl show | \
> > -        grep "Port patch-br-int-to-ln-100" | wc -l`
> > -    ])
> > -
> >       # Wait for a flow outputing to patch port
> > -    hv_patch_ofport=$(ovs-vsctl --bare --columns ofport find Interface
> name=patch-br-int-to-ln-100)
> > -    OVS_WAIT_UNTIL([
> > -        test 1 = $(ovs-ofctl dump-flows br-int | grep -c
> "output:$hv_patch_ofport")
> > -    ])
> > +    OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln-100"], ["hv-$i"])
> >   done
> >
> >   test_packet() {
> > @@ -3637,16 +3677,8 @@ for i in 1 2; do
> >       wait_for_ports_up lsp$i
> >
> >       # Patch port might be created after ports are reported up
> > -    OVS_WAIT_UNTIL([
> > -        test 1 = `ovs-vsctl show | \
> > -        grep "Port patch-br-int-to-ln-100" | wc -l`
> > -    ])
> > -
> >       # Wait for a flow outputing to patch port
> > -    hv_patch_ofport=$(ovs-vsctl --bare --columns ofport find Interface
> name=patch-br-int-to-ln-100)
> > -    OVS_WAIT_UNTIL([
> > -        test 1 = $(ovs-ofctl dump-flows br-int | grep -c
> "output:$hv_patch_ofport")
> > -    ])
> > +    OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln-100"], ["hv-$i"])
> >   done
> >   # create taps on fabric to check vlan encapsulation there
> >   for i in 1 2; do
> > @@ -3731,16 +3763,8 @@ for i in 1 2; do
> >       OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up lsp$i` = xup])
> >
> >       # Patch port might be created after ports are reported up
> > -    OVS_WAIT_UNTIL([
> > -        test 1 = `ovs-vsctl show | \
> > -        grep "Port patch-br-int-to-ln" | wc -l`
> > -    ])
> > -
> >       # Wait for a flow outputing to patch port
> > -    hv_patch_ofport=$(ovs-vsctl --bare --columns ofport find Interface
> name=patch-br-int-to-ln)
> > -    OVS_WAIT_UNTIL([
> > -        test 1 = $(ovs-ofctl dump-flows br-int | grep -c
> "output:$hv_patch_ofport")
> > -    ])
> > +    OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln"], ["hv-$i"])
> >   done
> >
> >   for i in 1 2; do
> > @@ -4123,6 +4147,9 @@ net_attach n2 br-phys
> >
> >   ovs-vsctl add-port br-phys vif3 -- set Interface vif3
> options:tx_pcap=hv3/vif3-tx.pcap options:rxq_pcap=hv3/vif3-rx.pcap
> ofport-request=1
> >
> > +# vtep is quite slow setting up all flows
> > +OVS_WAIT_UNTIL([test `as vtep ovs-ofctl dump-flows br-vtep_vtep_ls1 |
> grep "priority=0" | grep "output" | wc -l` -eq 1])
> > +
> >   # Pre-populate the hypervisors' ARP tables so that we don't lose any
> >   # packets for ARP resolution (native tunneling doesn't queue packets
> >   # for ARP resolution).
> > @@ -5044,7 +5071,6 @@ for i in 1 2 3; do
> >       as hv$i
> >       ovs-vsctl add-br br-phys
> >       ovn_attach n1 br-phys 192.168.0.$i
> > -
> >       for j in 1 2 3; do
> >           ovs-vsctl add-port br-int vif$i$j -- set Interface vif$i$j
> external-ids:iface-id=lp$i$j options:tx_pcap=hv$i/vif$i$j-tx.pcap
> options:rxq_pcap=hv$i/vif$i$j-rx.pcap ofport-request=$i$j
> >           ovn-nbctl lsp-add lsw0 lp$i$j
> > @@ -11305,7 +11331,7 @@ grep actions=mod_dl_dst:f0:00:00:01:02:04 | wc
> -l` -eq 1
> >       # Resend packet from foo1 to outside1
> >       check as hv1 ovs-appctl netdev-dummy/receive hv1-vif1 $packet
> >
> > -    OVN_CHECK_PACKETS([ext1/vif1-tx.pcap], [ext1-vif1.expected])
> > +    OVN_CHECK_PACKETS_UNIQ([ext1/vif1-tx.pcap], [ext1-vif1.expected])
> >       $PYTHON "$ovs_srcdir/utilities/ovs-pcap.in"
> $active_gw/br-phys_n1-tx.pcap  > packets
> >       cat packets | grep $expected > exp
> >       cat packets | grep $exp_gw_ip_garp | head -1 >> exp
> > @@ -11565,8 +11591,7 @@ OVN_CHECK_PACKETS_UNIQ([hv3/vif1-tx.pcap],
> [hv3-vif1.expected])
> >   as hv2 ovs-vsctl set open .
> external-ids:ovn-bridge-mappings=phys:br-phys
> >
> >   # Wait until the patch ports are created in hv2 to connect br-int to
> br-phys
> > -OVS_WAIT_UNTIL([test 1 = `as hv2 ovs-vsctl show | \
> > -grep "Port patch-br-int-to-ln-alice" | wc -l`])
> > +OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln-alice"], ["hv2"])
> >
> >   dnl Allow some time for ovn-controller to catch up.
> >   wait_for_ports_up
> > @@ -11787,8 +11812,7 @@ ovs-vsctl -- add-port br-int hv3-vif2 -- \
> >   AT_CHECK([as hv3 ovs-vsctl set Open_vSwitch .
> external-ids:ovn-bridge-mappings=physnet1:br-phys])
> >
> >   # Wait until the patch ports are created in hv3 to connect br-int to
> br-phys
> > -OVS_WAIT_UNTIL([test 1 = `as hv3 ovs-vsctl show | \
> > -grep "Port patch-br-int-to-ln_port" | wc -l`])
> > +OVN_WAIT_PATCH_PORT_FLOWS(["patch-br-int-to-ln_port"], ["hv3"])
> >
> >   # Re-add nat-addresses option
> >   ovn-nbctl lsp-set-options lrp0-rp router-port=lrp0
> nat-addresses="router"
> > @@ -12536,8 +12560,8 @@ for i in 1 2; do
> >       ovn_attach n1 br-phys 192.168.0.$i
> >       ovs-vsctl add-port br-int vif01 -- \
> >           set Interface vif01 external-ids:iface-id=lp01 \
> > -                              options:tx_pcap=hv${i}/vif01-tx.pcap \
> > -                              options:rxq_pcap=hv${i}/vif01-rx.pcap \
> > +                              options:tx_pcap=hv${i}/vif${i}0-tx.pcap \
> > +                              options:rxq_pcap=hv${i}/vif${i}0-rx.pcap \
> >                                 ofport-request=${i}0
> >
> >       ovs-vsctl add-port br-int vif${i}1 -- \
> > @@ -12606,8 +12630,14 @@ test_packet() {
> >       fi
> >       vif=vif$inport
> >       as $hv ovs-appctl netdev-dummy/receive $vif $packet
> > -    if test $eout != drop; then
> > -        echo $packet >> ${eout#lp}.expected
> > +
> > +    # For packets sent to localport, differentiate localport per
> hypervisor
> > +    if test $eout = lp01; then
> > +        echo $packet >> ${hv#hv}0.expected
> > +    else
> > +        if test $eout != drop; then
> > +            echo $packet >> ${eout#lp}.expected
> > +        fi
> >       fi
> >   }
> >
> > @@ -16729,6 +16759,10 @@ grep controller | grep tp_src=546 | grep \
> >   "ae.70.00.00.00.00.00.00.00.00.00.00.00.00.00.06" | wc -l], [0], [0
> >   ])
> >
> > +wait_row_count Chassis 1 name=hv1
> > +wait_row_count Chassis 1 name=hv2
> > +wait_row_count Chassis 1 name=hv3
> > +
> >   hv1_uuid=$(fetch_column Chassis _uuid name=hv1)
> >   hv2_uuid=$(fetch_column Chassis _uuid name=hv2)
> >   hv3_uuid=$(fetch_column Chassis _uuid name=hv3)
> > @@ -17259,7 +17293,7 @@ reset_pcap_file hv2-ext2 hv2/ext2
> >   reset_pcap_file br-phys_n1 hv2/br-phys_n1
> >   reset_pcap_file br-phys hv2/br-phys
> >
> > -as hv2
> > +as hv3
> >   ovs-vsctl show
> >   reset_pcap_file hv3-ext3 hv3/ext3
> >   reset_pcap_file br-phys_n1 hv3/br-phys_n1
> > @@ -17335,7 +17369,6 @@ AT_CHECK([as hv2 ovs-ofctl dump-flows br-int \
> >   table=31,dl_src=f0:00:00:00:00:03,dl_dst=a0:10:00:00:00:01 | \
> >   grep -c "actions=drop"], [0], [1
> >   ])
> > -
> >   # Stop ovn-controllers on hv1 and hv3.
> >   as hv1 ovn-appctl -t ovn-controller exit
> >   as hv3 ovn-appctl -t ovn-controller exit
> >
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
>