diff mbox series

[ovs-dev,03/16] tests: Add some more "wait"s for ovn-northd synchronization.

Message ID 20201030002447.936548-3-blp@ovn.org
State Accepted
Headers show
Series [ovs-dev,01/16] tests: Drop support for glibc before version 2.11. | expand

Commit Message

Ben Pfaff Oct. 30, 2020, 12:24 a.m. UTC 
From: Leonid Ryzhyk <lryzhyk@vmware.com>

These also add "check" to the calls that are not already inside
AT_CHECK, to make sure that these calls succeed.

Signed-off-by: Ben Pfaff <blp@ovn.org>
---
 tests/ovn-controller-vtep.at |   8 +--
 tests/ovn-ic.at              |   3 +-
 tests/ovn-northd.at          |  36 ++++++------
 tests/ovn.at                 | 104 +++++++++++++++++++++--------------
 4 files changed, 87 insertions(+), 64 deletions(-)

Comments

0-day Robot Oct. 30, 2020, 1:06 a.m. UTC | #1 
Bleep bloop.  Greetings Ben Pfaff, I am a robot and I have tried out your patch.
Thanks for your contribution.

I encountered some error that I wasn't expecting.  See the details below.


checkpatch:
ERROR: Author Leonid Ryzhyk <lryzhyk@vmware.com> needs to sign off.
WARNING: Unexpected sign-offs from developers who are not authors or co-authors or committers: Ben Pfaff <blp@ovn.org>
Lines checked: 718, Warnings: 1, Errors: 1


Please check this out.  If you feel there has been an error, please email aconole@redhat.com

Thanks,
0-day Robot
diff mbox series

Patch

diff --git a/tests/ovn-controller-vtep.at b/tests/ovn-controller-vtep.at
index f0598fa4dc95..d3895a60f53c 100644
--- a/tests/ovn-controller-vtep.at
+++ b/tests/ovn-controller-vtep.at
@@ -375,7 +375,7 @@  AT_CHECK([ovs-vsctl --columns=options list Interface bfd1.2.3.5 | cut -d ':' -f2
 ])
 
 # adds another mac to logical switch port.
-AT_CHECK([ovn-nbctl lsp-set-addresses vif0 f0:ab:cd:ef:01:02 f0:ab:cd:ef:01:03])
+AT_CHECK([ovn-nbctl --wait=sb lsp-set-addresses vif0 f0:ab:cd:ef:01:02 f0:ab:cd:ef:01:03])
 OVS_WAIT_UNTIL([test -n "`vtep-ctl list Ucast_Macs_Remote | grep 03`"])
 AT_CHECK([vtep-ctl --columns=MAC list Ucast_Macs_Remote | cut -d ':' -f2- | tr -d ' ' | sort], [0], [dnl
 
@@ -400,7 +400,7 @@  AT_CHECK([vtep-ctl --columns=MAC list Ucast_Macs_Remote | cut -d ':' -f2- | tr -
 ])
 
 # removes one mac to logical switch port.
-AT_CHECK([ovn-nbctl lsp-set-addresses vif0 f0:ab:cd:ef:01:03])
+AT_CHECK([ovn-nbctl --wait=sb lsp-set-addresses vif0 f0:ab:cd:ef:01:03])
 OVS_WAIT_UNTIL([test -z "`vtep-ctl --columns=MAC list Ucast_Macs_Remote | grep 02`"])
 AT_CHECK([vtep-ctl --columns=MAC list Ucast_Macs_Remote | cut -d ':' -f2- | tr -d ' ' | sort], [0], [dnl
 "f0:ab:cd:ef:01:03"
@@ -408,7 +408,7 @@  AT_CHECK([vtep-ctl --columns=MAC list Ucast_Macs_Remote | cut -d ':' -f2- | tr -
 
 # migrates mac to logical switch port vif1 on 'br-void'.
 AT_CHECK([ovn-nbctl lsp-set-addresses vif0])
-AT_CHECK([ovn-nbctl lsp-set-addresses vif1 f0:ab:cd:ef:01:03])
+AT_CHECK([ovn-nbctl --wait=sb lsp-set-addresses vif1 f0:ab:cd:ef:01:03])
 OVS_WAIT_UNTIL([test -z "`vtep-ctl --columns=MAC list Ucast_Macs_Remote | grep 03`"])
 AT_CHECK([vtep-ctl --columns=MAC list Ucast_Macs_Remote | cut -d ':' -f2- | tr -d ' ' | sort], [0], [dnl
 ])
@@ -454,7 +454,7 @@  AT_CHECK([sed -n 's/^.*\(|WARN|.*\)$/\1/p' ovn-controller-vtep.log | sed 's/([[-
 ])
 
 # deletes vif1.
-AT_CHECK([ovn-nbctl lsp-del vif1])
+AT_CHECK([ovn-nbctl --wait=sb lsp-del vif1])
 
 # adds another lswitch 'br-void' in ovn-nb database.
 AT_CHECK([ovn-nbctl ls-add br-void])
diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
index 1d40ce958d6c..0638af401295 100644
--- a/tests/ovn-ic.at
+++ b/tests/ovn-ic.at
@@ -44,6 +44,7 @@  wait_row_count ic-sb:Datapath_Binding 1 transit_switch=ts2
 check_column "ts1 ts2" ic-sb:Datapath_Binding transit_switch
 check_column "ts1 ts2" nb:Logical_Switch name
 
+ovn-nbctl --wait=hv sync
 
 # Check SB DP key
 ts1_key=$(fetch_column ic-sb:Datapath_Binding tunnel_key transit_switch=ts1)
@@ -143,7 +144,7 @@  ovn-nbctl lrp-add lr1 lrp-lr1-ts1 aa:aa:aa:aa:aa:01 169.254.100.1/24
 ovn-nbctl lsp-add ts1 lsp-ts1-lr1
 ovn-nbctl lsp-set-addresses lsp-ts1-lr1 router
 ovn-nbctl lsp-set-type lsp-ts1-lr1 router
-ovn-nbctl lsp-set-options lsp-ts1-lr1 router-port=lrp-lr1-ts1
+ovn-nbctl --wait=hv lsp-set-options lsp-ts1-lr1 router-port=lrp-lr1-ts1
 
 AT_CHECK([ovn_as az2 ovn-nbctl show | uuidfilt], [0], [dnl
 switch <0> (ts1)
diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at
index 869bcd6f6cba..90f3c8f19241 100644
--- a/tests/ovn-northd.at
+++ b/tests/ovn-northd.at
@@ -304,10 +304,10 @@  check_row_count HA_Chassis 0
 
 # Create a logical router port and attach ha chassis group.
 ovn-nbctl lr-add lr0
-ovn-nbctl lrp-add lr0 lr0-public 00:00:20:20:12:13 172.168.0.100/24
+check ovn-nbctl --wait=sb lrp-add lr0 lr0-public 00:00:20:20:12:13 172.168.0.100/24
 
 hagrp1_uuid=`ovn-nbctl --bare --columns _uuid find ha_chassis_group name=hagrp1`
-ovn-nbctl set logical_router_port lr0-public ha_chassis_group=$hagrp1_uuid
+check ovn-nbctl --wait=sb set logical_router_port lr0-public ha_chassis_group=$hagrp1_uuid
 
 wait_row_count HA_Chassis_Group 1 name=hagrp1
 
@@ -371,7 +371,7 @@  wait_row_count HA_Chassis_Group 1 name=hagrp1
 wait_row_count HA_Chassis 3
 
 # Create a Gateway chassis. ovn-northd should ignore this.
-ovn-nbctl lrp-set-gateway-chassis lr0-public ch-1 20
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis lr0-public ch1 20
 
 # There should be only 1 HA chassis group in SB DB with the
 # name hagrp1.
@@ -409,7 +409,7 @@  ovn-nbctl lrp-add lr0 lr0-sw0 00:00:20:20:12:14 10.0.0.1/24
 ovn-nbctl lsp-add sw0 sw0-lr0
 ovn-nbctl lsp-set-type sw0-lr0 router
 ovn-nbctl lsp-set-addresses sw0-lr0 router
-ovn-nbctl lsp-set-options sw0-lr0 router-port=lr0-sw0
+check ovn-nbctl --wait=sb lsp-set-options sw0-lr0 router-port=lr0-sw0
 
 ovn-sbctl lsp-bind sw0-p1 comp1
 wait_row_count nb:Logical_Switch_Port 1 name=sw0-p1 up=true
@@ -437,7 +437,7 @@  ovn-nbctl lrp-add lr1 lr1-sw1 00:00:20:20:12:15 20.0.0.1/24
 ovn-nbctl lsp-add sw1 sw1-lr1
 ovn-nbctl lsp-set-type sw1-lr1 router
 ovn-nbctl lsp-set-addresses sw1-lr1 router
-ovn-nbctl lsp-set-options sw1-lr1 router-port=lr1-sw1
+check ovn-nbctl --wait=sb lsp-set-options sw1-lr1 router-port=lr1-sw1
 
 # Bind sw1-p1 in comp1.
 ovn-sbctl lsp-bind sw1-p1 comp1
@@ -452,7 +452,7 @@  ovn-nbctl lrp-add lr1 lr1-sw0 00:00:20:20:12:16 10.0.0.10/24
 ovn-nbctl lsp-add sw0 sw0-lr1
 ovn-nbctl lsp-set-type sw0-lr1 router
 ovn-nbctl lsp-set-addresses sw0-lr1 router
-ovn-nbctl lsp-set-options sw0-lr1 router-port=lr1-sw0
+check ovn-nbctl --wait=sb lsp-set-options sw0-lr1 router-port=lr1-sw0
 
 # Both comp1 and comp2 should be in 'ref_chassis' as sw1 is indirectly
 # connected to lr0
@@ -473,7 +473,7 @@  ovn-nbctl lrp-add lr2 lr2-sw2 00:00:20:20:12:17 30.0.0.1/24
 ovn-nbctl lsp-add sw2 sw2-lr2
 ovn-nbctl lsp-set-type sw2-lr2 router
 ovn-nbctl lsp-set-addresses sw2-lr2 router
-ovn-nbctl lsp-set-options sw2-lr2 router-port=lr2-sw2
+check ovn-nbctl --wait=sb lsp-set-options sw2-lr2 router-port=lr2-sw2
 
 # Bind sw2-p1 to comp1
 ovn-sbctl lsp-bind sw2-p1 comp1
@@ -488,14 +488,14 @@  ovn-nbctl lrp-add lr2 lr2-sw1 00:00:20:20:12:18 20.0.0.10/24
 ovn-nbctl lsp-add sw1 sw1-lr2
 ovn-nbctl lsp-set-type sw1-lr2 router
 ovn-nbctl lsp-set-addresses sw1-lr2 router
-ovn-nbctl lsp-set-options sw1-lr2 router-port=lr2-sw1
+check ovn-nbctl --wait=sb lsp-set-options sw1-lr2 router-port=lr2-sw1
 
 # sw2-p1 is indirectly connected to lr0. So comp1 (and comp2) should be in
 # 'ref_chassis'
 wait_column "$exp_ref_ch_list" HA_Chassis_Group ref_chassis
 
 # Create sw0-p2 and bind it to comp1
-ovn-nbctl lsp-add sw0 sw0-p2
+check ovn-nbctl --wait=sb lsp-add sw0 sw0-p2
 ovn-sbctl lsp-bind sw0-p2 comp1
 wait_row_count nb:Logical_Switch_Port 1 name=sw0-p2 up=true
 wait_column "$exp_ref_ch_list" HA_Chassis_Group ref_chassis
@@ -539,7 +539,7 @@  ovn-nbctl lsp-set-addresses sw0-pext1 "00:00:00:00:00:03 10.0.0.3"
 ovn-nbctl lsp-set-addresses sw0-pext2 "00:00:00:00:00:03 10.0.0.4"
 ovn-nbctl lsp-set-addresses sw0-p1 "00:00:00:00:00:03 10.0.0.5"
 
-ovn-nbctl --wait=sb ha-chassis-group-add hagrp1
+check ovn-nbctl --wait=sb ha-chassis-group-add hagrp1
 
 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch1 30
 ovn-nbctl --wait=sb ha-chassis-group-add-chassis hagrp1 ch2 20
@@ -589,7 +589,7 @@  ha_chassis_group=$hagrp1_uuid
 wait_row_count Port_Binding 0 logical_port=sw0-p1 'chassis!=[[]]'
 
 # Clear ha_chassis_group for sw0-pext1
-ovn-nbctl --wait=sb clear logical_switch_port sw0-pext1 ha_chassis_group
+check ovn-nbctl --wait=sb clear logical_switch_port sw0-pext1 ha_chassis_group
 
 wait_row_count Port_Binding 0 logical_port=sw0-pext1 'chassis!=[[]]'
 
@@ -685,9 +685,9 @@  ovn-nbctl ls-add S1
 ovn-nbctl lsp-add S1 S1-R1
 ovn-nbctl lsp-set-type S1-R1 router
 ovn-nbctl lsp-set-addresses S1-R1 router
-ovn-nbctl --wait=sb lsp-set-options S1-R1 router-port=R1-S1
+ovn-nbctl lsp-set-options S1-R1 router-port=R1-S1
 
-ovn-nbctl lrp-set-gateway-chassis R1-S1 gw1
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis R1-S1 gw1
 
 ovn-nbctl lrp-set-redirect-type R1-S1 bridged
 wait_row_count Port_Binding 1 logical_port=cr-R1-S1 options:redirect-type=bridged
@@ -708,9 +708,9 @@  ovn-nbctl ls-add S1
 ovn-nbctl lsp-add S1 S1-R1
 ovn-nbctl lsp-set-type S1-R1 router
 ovn-nbctl lsp-set-addresses S1-R1 router
-ovn-nbctl --wait=sb lsp-set-options S1-R1 router-port=R1-S1
+ovn-nbctl lsp-set-options S1-R1 router-port=R1-S1
 
-ovn-nbctl lrp-set-gateway-chassis R1-S1 gw1
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis R1-S1 gw1
 
 # IPV4
 ovn-nbctl lr-nat-add R1 dnat_and_snat  172.16.1.1 50.0.0.11
@@ -799,9 +799,9 @@  ovn-nbctl ls-add S1
 ovn-nbctl lsp-add S1 S1-R1
 ovn-nbctl lsp-set-type S1-R1 router
 ovn-nbctl lsp-set-addresses S1-R1 router
-ovn-nbctl --wait=sb lsp-set-options S1-R1 router-port=R1-S1
+ovn-nbctl lsp-set-options S1-R1 router-port=R1-S1
 
-ovn-nbctl lrp-set-gateway-chassis R1-S1 gw1
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis R1-S1 gw1
 
 uuid=`ovn-sbctl --columns=_uuid --bare find Port_Binding logical_port=cr-R1-S1`
 echo "CR-LRP UUID is: " $uuid
@@ -1272,7 +1272,7 @@  ovn-nbctl lr-lb-add lr0 lb4
 
 ovn-nbctl lr-nat-add lr0 snat 192.168.2.1 10.0.0.0/24
 ovn-nbctl lr-nat-add lr0 dnat_and_snat 192.168.2.4 10.0.0.4
-ovn-nbctl lr-nat-add lr0 dnat 192.168.2.5 10.0.0.5
+check ovn-nbctl --wait=sb lr-nat-add lr0 dnat 192.168.2.5 10.0.0.5
 
 OVS_WAIT_UNTIL([test 1 = $(ovn-sbctl dump-flows lr0 | grep lr_in_unsnat | \
 grep "ip4 && ip4.dst == 192.168.2.1 && tcp && tcp.dst == 8080" -c) ])
diff --git a/tests/ovn.at b/tests/ovn.at
index 46f13f46617b..912f84c25dbb 100644
--- a/tests/ovn.at
+++ b/tests/ovn.at
@@ -1782,6 +1782,7 @@  get_lsp_uuid () {
 
 ovn-nbctl create Port_Group name=pg1 ports=`get_lsp_uuid lp22`,`get_lsp_uuid lp33`
 ovn-nbctl acl-add lsw0 to-lport 1000 'eth.type == 0x1238 && outport == @pg1' drop
+check ovn-nbctl --wait=hv sync
 
 # Pre-populate the hypervisors' ARP tables so that we don't lose any
 # packets for ARP resolution (native tunneling doesn't queue packets
@@ -3559,6 +3560,8 @@  for i in 1 2 3; do
     done
 done
 
+check ovn-nbctl --wait=hv sync
+
 # Pre-populate the hypervisors' ARP tables so that we don't lose any
 # packets for ARP resolution (native tunneling doesn't queue packets
 # for ARP resolution).
@@ -3566,7 +3569,6 @@  OVN_POPULATE_ARP
 
 # Allow some time for ovn-northd and ovn-controller to catch up.
 # XXX This should be more systematic.
-sleep 1
 
 # test_ip INPORT SRC_MAC DST_MAC SRC_IP DST_IP OUTPORT...
 #
@@ -5775,7 +5777,7 @@  cat 2.expected | cut -c 53- > expout
 AT_CHECK([cat 2.packets | cut -c 53-], [0], [expout])
 
 # Now add the dhcp option T1 to the dhcp options.
-ovn-nbctl set dhcp_options ${d1} options:T1=4000
+check ovn-nbctl --wait=hv set dhcp_options ${d1} options:T1=4000
 
 reset_pcap_file hv1-vif1 hv1/vif1
 reset_pcap_file hv1-vif2 hv1/vif2
@@ -6649,7 +6651,7 @@  ls2_p1_ip=172.16.1.2
 ls3_p1_mac=00:00:00:01:02:05
 
 # Create a drop policy
-ovn-nbctl lr-policy-add R1 10 "ip4.src==192.168.1.0/24 && ip4.dst==172.16.1.0/24" drop
+check ovn-nbctl --wait=hv lr-policy-add R1 10 "ip4.src==192.168.1.0/24 && ip4.dst==172.16.1.0/24" drop
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | grep "192.168.1.0" | wc -l], [0], [dnl
@@ -6677,7 +6679,7 @@  rcvd_packet=`cat vif2.packets`
 AT_FAIL_IF([rcvd_packet = ""])
 
 # Override drop policy with allow
-ovn-nbctl lr-policy-add R1 20 "ip4.src==192.168.1.0/24 && ip4.dst==172.16.1.0/24" allow
+check ovn-nbctl --wait=hv lr-policy-add R1 20 "ip4.src==192.168.1.0/24 && ip4.dst==172.16.1.0/24" allow
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | grep "192.168.1.0" | wc -l], [0], [dnl
@@ -6691,6 +6693,7 @@  packet="inport==\"ls1-lp1\" && eth.src==$ls1_p1_mac && eth.dst==$ls1_ro_mac &&
 as pbr-hv ovs-appctl -t ovn-controller inject-pkt "$packet"
 
 # Check if packet hit the allow policy
+sleep 1
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | \
     grep "192.168.1.0" | \
     grep "priority=20" | wc -l], [0], [dnl
@@ -6706,7 +6709,7 @@  echo $expected | ovstest test-ovn expr-to-packets > expected
 OVN_CHECK_PACKETS([pbr-hv/vif2-tx.pcap], [expected])
 
 # Override allow policy with reroute
-ovn-nbctl lr-policy-add R1 30 "ip4.src==192.168.1.0/24 && ip4.dst==172.16.1.0/24" reroute 20.20.1.2
+check ovn-nbctl --wait=hv lr-policy-add R1 30 "ip4.src==192.168.1.0/24 && ip4.dst==172.16.1.0/24" reroute 20.20.1.2
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | \
@@ -6720,6 +6723,7 @@  packet="inport==\"ls1-lp1\" && eth.src==$ls1_p1_mac && eth.dst==$ls1_ro_mac &&
        ip4 && ip.ttl==64 && ip4.src==$ls1_p1_ip && ip4.dst==$ls2_p1_ip &&
        udp && udp.src==53 && udp.dst==4369"
 as pbr-hv ovs-appctl -t ovn-controller inject-pkt "$packet"
+sleep 1
 
 echo "southbound flows"
 
@@ -6834,7 +6838,7 @@  ls2_p1_ip=2002::2
 ls3_p1_mac=00:00:00:01:02:05
 
 # Create a drop policy
-ovn-nbctl lr-policy-add R1 10 "ip6.src==2001::/64 && ip6.dst==2002::/64" drop
+check ovn-nbctl --wait=sb lr-policy-add R1 10 "ip6.src==2001::/64 && ip6.dst==2002::/64" drop
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | grep "2001" | wc -l], [0], [dnl
@@ -6862,7 +6866,7 @@  rcvd_packet=`cat vif2.packets`
 AT_FAIL_IF([rcvd_packet = ""])
 
 # Override drop policy with allow
-ovn-nbctl lr-policy-add R1 20 "ip6.src==2001::/64 && ip6.dst==2002::/64" allow
+check ovn-nbctl --wait=sb lr-policy-add R1 20 "ip6.src==2001::/64 && ip6.dst==2002::/64" allow
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | grep "2001" | wc -l], [0], [dnl
@@ -6891,7 +6895,7 @@  echo $expected | ovstest test-ovn expr-to-packets > expected
 OVN_CHECK_PACKETS([pbr-hv/vif2-tx.pcap], [expected])
 
 # Override allow policy with reroute
-ovn-nbctl lr-policy-add R1 30 "ip6.src==2001::/64 && ip6.dst==2002::/64" reroute 2003::2
+check ovn-nbctl --wait=sb lr-policy-add R1 30 "ip6.src==2001::/64 && ip6.dst==2002::/64" reroute 2003::2
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep lr_in_policy | \
@@ -6905,6 +6909,7 @@  packet="inport==\"ls1-lp1\" && eth.src==$ls1_p1_mac && eth.dst==$ls1_ro_mac &&
        ip6 && ip.ttl==64 && ip6.src==$ls1_p1_ip && ip6.dst==$ls2_p1_ip &&
        udp && udp.src==53 && udp.dst==4369"
 as pbr-hv ovs-appctl -t ovn-controller inject-pkt "$packet"
+sleep 1
 
 echo "southbound flows"
 
@@ -7921,7 +7926,7 @@  AT_CHECK([ovs-vsctl add-port br-eth0 snoopvif -- set Interface snoopvif options:
 AT_CHECK([ovn-nbctl lsp-add ls0 ln_port])
 AT_CHECK([ovn-nbctl lsp-set-addresses ln_port unknown])
 AT_CHECK([ovn-nbctl lsp-set-type ln_port localnet])
-AT_CHECK([ovn-nbctl lsp-set-options ln_port network_name=physnet1])
+AT_CHECK([ovn-nbctl --wait=hv lsp-set-options ln_port network_name=physnet1])
 
 # Wait until the patch ports are created in hv1 to connect br-int to br-eth0
 OVS_WAIT_UNTIL([test 1 = `as hv1 ovs-vsctl show | \
@@ -8419,10 +8424,12 @@  ovn-nbctl --wait=sb sync
 # Add an ACL that rate-limits logs at 10 per second.
 ovn-nbctl meter-add http-rl1 drop 10 pktps
 ovn-nbctl --log --severity=alert --meter=http-rl1 --name=http-acl1 acl-add lsw0 to-lport 1000 'tcp.dst==80' drop
+check ovn-nbctl --wait=hv sync
 
 # Add an ACL that rate-limits logs at 5 per second.
 ovn-nbctl meter-add http-rl2 drop 5 pktps
 ovn-nbctl --log --severity=alert --meter=http-rl2 --name=http-acl2 acl-add lsw0 to-lport 1000 'tcp.dst==81' allow
+check ovn-nbctl --wait=hv sync
 
 # Add an ACL that doesn't rate-limit logs.
 ovn-nbctl --log --severity=alert --name=http-acl3 acl-add lsw0 to-lport 1000 'tcp.dst==82' drop
@@ -8637,7 +8644,7 @@  AT_CHECK([as hv ovs-ofctl dump-flows br-int -O OpenFlow13 | grep meter | wc -l],
 ])
 
 # check meter with chassis not resident
-ovn-nbctl qos-add lsw0 to-lport 1001 'inport=="lp3" && is_chassis_resident("lp3")' rate=11123 burst=111230
+ovn-nbctl --wait=hv qos-add lsw0 to-lport 1001 'inport=="lp3" && is_chassis_resident("lp3")' rate=11123 burst=111230
 as hv check_row_count nb:QoS 1
 
 # check no meter table
@@ -8651,7 +8658,7 @@  ovn-nbctl qos-del lsw0
 ovn-nbctl qos-add lsw0 to-lport 1001 'inport=="lp1" && is_chassis_resident("lp1")' rate=100000 burst=100000
 ovn-nbctl qos-add lsw0 to-lport 1001 'inport=="lp2" && is_chassis_resident("lp2")' rate=100000 burst=100000
 ovn-nbctl qos-add lsw0 to-lport 1002 'inport=="lp1" && is_chassis_resident("lp1")' rate=100001 burst=100001
-ovn-nbctl qos-add lsw0 to-lport 1002 'inport=="lp2" && is_chassis_resident("lp2")' rate=100001 burst=100001
+check ovn-nbctl --wait=hv qos-add lsw0 to-lport 1002 'inport=="lp2" && is_chassis_resident("lp2")' rate=100001 burst=100001
 
 AT_CHECK([as hv ovs-ofctl dump-meters br-int -O OpenFlow13 | grep meter | wc -l], [0], [4
 ])
@@ -10820,7 +10827,7 @@  OVS_WAIT_UNTIL([test 1 = `ovn-sbctl dump-flows foo | grep ls_in_l2_lkup | \
 grep rp-foo | grep -v is_chassis_resident | grep priority=50 -c`])
 
 # Set the option 'reside-on-redirect-chassis' for foo
-ovn-nbctl set logical_router_port foo options:reside-on-redirect-chassis=true
+check ovn-nbctl --wait=hv set logical_router_port foo options:reside-on-redirect-chassis=true
 # Check that there is a logical flow in logical switch foo's pipeline
 # to set the outport to rp-foo with the condition is_chassis_redirect.
 ovn-sbctl dump-flows foo
@@ -13508,7 +13515,7 @@  ovn-nbctl create Address_Set name=set2 \
 addresses=\"10.0.0.7\",\"10.0.0.8\",\"10.0.0.9\"
 ovn-nbctl acl-add ls1 to-lport 1001 \
 'ip4 && ip4.src == $set1 && ip4.dst == $set1' allow
-ovn-nbctl acl-add ls1 to-lport 1001 \
+check ovn-nbctl --wait=hv acl-add ls1 to-lport 1001 \
 'ip4 && ip4.src == $set1 && ip4.dst == $set2' drop
 
 # test_ip INPORT SRC_MAC DST_MAC SRC_IP DST_IP OUTPORT...
@@ -13982,12 +13989,12 @@  test_packet() {
 # Test drop rule
 # --------------
 ovn-nbctl acl-del lsw0
-ovn-nbctl --log --severity=info --name=drop-acl acl-add lsw0 to-lport 5000 'outport == @pg1 && eth.src == $set1 && eth.type == 0x1234' drop
+check ovn-nbctl --wait=hv --log --severity=info --name=drop-acl acl-add lsw0 to-lport 5000 'outport == @pg1 && eth.src == $set1 && eth.type == 0x1234' drop
 for sf in 0 1; do
     if test ${sf} = 1; then
         # Add a stateful rule and re-run the check to make sure the
         # drop rule is still effective..
-        ovn-nbctl acl-add lsw0 from-lport 2000  "inport == lp31 && ip" allow-related
+        ovn-nbctl --wait=hv acl-add lsw0 from-lport 2000  "inport == lp31 && ip" allow-related
     fi
     for is in 1 2 3; do
         s=${is}1
@@ -14021,12 +14028,12 @@  ovn-nbctl acl-del lsw0
 # drop all packets to 11 and 21.
 ovn-nbctl acl-add lsw0 to-lport 5000 'outport == @pg1 && eth.src == $set1' drop
 # allow 0x1234 between 11 and 21
-ovn-nbctl --log --severity=info --name=allow-acl acl-add lsw0 to-lport 6000 'outport == @pg1 && eth.src == $set1 && eth.type == 0x1234' allow
+check ovn-nbctl --wait=hv --log --severity=info --name=allow-acl acl-add lsw0 to-lport 6000 'outport == @pg1 && eth.src == $set1 && eth.type == 0x1234' allow
 for sf in 0 1; do
     if test ${sf} = 1; then
         # Add a stateful rule and re-run the check to make sure the
         # allow rule is still effective..
-        ovn-nbctl acl-add lsw0 from-lport 2000  "inport == lp31 && ip" allow-related
+        check ovn-nbctl --wait=hv acl-add lsw0 from-lport 2000  "inport == lp31 && ip" allow-related
     fi
     for is in 1 2 3; do
         s=${is}1
@@ -15375,17 +15382,17 @@  ovn-nbctl lsp-add sw1 sw1-p1
 ovn-nbctl lsp-add sw1 sw1-p2
 ovn-nbctl lsp-add sw1 sw1-p3
 ovn-nbctl lsp-add sw1 sw1-p4
+ovn-nbctl --wait=sb lsp-add sw1 sw1-p5
 
-ovn-nbctl lsp-set-addresses sw1-p1 "00:00:00:00:00:01 10.0.0.1 aef0::1" "00:00:00:00:00:02 10.0.0.2 aef0::2"
-ovn-nbctl lsp-set-addresses sw1-p2 "00:00:00:00:00:03 dynamic"
-ovn-nbctl lsp-set-addresses sw1-p3 "dynamic"
-ovn-nbctl lsp-set-addresses sw1-p4 "router"
-ovn-nbctl lsp-set-addresses sw1-p5 "unknown"
+check ovn-nbctl --wait=sb lsp-set-addresses sw1-p1 "00:00:00:00:00:01 10.0.0.1 aef0::1" "00:00:00:00:00:02 10.0.0.2 aef0::2"
+check ovn-nbctl --wait=sb lsp-set-addresses sw1-p2 "00:00:00:00:00:03 dynamic"
+check ovn-nbctl --wait=sb lsp-set-addresses sw1-p3 "dynamic"
+check ovn-nbctl --wait=sb lsp-set-addresses sw1-p4 "router"
+check ovn-nbctl --wait=sb lsp-set-addresses sw1-p5 "unknown"
 
 ovn-nbctl list logical_switch_port
 
 # Now try to add duplicate addresses on a new port. These should all fail
-ovn-nbctl --wait=sb lsp-add sw1 sw1-p5
 AT_CHECK([ovn-nbctl lsp-set-addresses sw1-p5 "00:00:00:00:00:04 10.0.0.1"], [1], [],
 [ovn-nbctl: Error on switch sw1: duplicate IPv4 address '10.0.0.1' found on logical switch port 'sw1-p1'
 ])
@@ -16576,6 +16583,7 @@  send_arp_reply 1 1 $eth_src $eth_dst $spa $tpa
 
 OVS_WAIT_UNTIL([test x$(ovn-sbctl --bare --columns chassis find port_binding \
 logical_port=sw0-vir) = x$hv1_ch_uuid], [0], [])
+sleep 1
 
 AT_CHECK([test x$(ovn-sbctl --bare --columns virtual_parent find port_binding \
 logical_port=sw0-vir) = xsw0-p1])
@@ -16592,6 +16600,7 @@  as hv1 ovs-vsctl del-port hv1-vif1
 
 OVS_WAIT_UNTIL([test x$(ovn-sbctl --bare --columns chassis find port_binding \
 logical_port=sw0-vir) = x], [0], [])
+sleep 1
 
 AT_CHECK([test x$(ovn-sbctl --bare --columns virtual_parent find port_binding \
 logical_port=sw0-vir) = x])
@@ -17044,7 +17053,7 @@  ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
 OVN_POPULATE_ARP
 
 # Enable IGMP snooping on sw1.
-ovn-nbctl set Logical_Switch sw1       \
+check ovn-nbctl --wait=hv set Logical_Switch sw1       \
     other_config:mcast_querier="false" \
     other_config:mcast_snoop="true"
 
@@ -17055,7 +17064,7 @@  OVN_CHECK_PACKETS([hv1/vif2-tx.pcap], [expected])
 OVN_CHECK_PACKETS([hv2/vif1-tx.pcap], [expected])
 OVN_CHECK_PACKETS([hv2/vif2-tx.pcap], [expected])
 
-ovn-nbctl --wait=hv sync
+check ovn-nbctl --wait=hv sync
 
 # Inject IGMP Join for 239.0.1.68 on sw1-p11.
 send_igmp_v3_report hv1-vif1 hv1 \
@@ -17072,6 +17081,7 @@  OVS_WAIT_UNTIL([
     total_entries=`ovn-sbctl find IGMP_Group | grep "239.0.1.68" -c`
     test "${total_entries}" = "2"
 ])
+check ovn-nbctl --wait=hv sync
 
 # Send traffic and make sure it gets forwarded only on the two ports that
 # joined.
@@ -17101,6 +17111,7 @@  send_igmp_v3_report hv1-vif1 hv1 \
 
 # Check IGMP_Group table on both HV.
 wait_row_count IGMP_Group 1 address=239.0.1.68
+check ovn-nbctl --wait=hv sync
 
 # Send traffic and make sure it gets forwarded only on the port that joined.
 as hv1 reset_pcap_file hv1-vif1 hv1/vif1
@@ -17126,6 +17137,7 @@  OVN_CHECK_PACKETS([hv2/vif3-tx.pcap], [expected_empty])
 # Flush IGMP groups.
 ovn-sbctl ip-multicast-flush sw1
 wait_row_count IGMP_Group 0 address=239.0.1.68
+check ovn-nbctl --wait=hv sync
 
 # Check that traffic for 224.0.0.X is flooded even if some hosts register for
 # it.
@@ -17183,6 +17195,8 @@  ovn-nbctl set Logical_Switch sw3       \
     other_config:mcast_querier="false" \
     other_config:mcast_snoop="true"
 
+check ovn-nbctl --wait=hv sync
+
 # Send traffic from sw3 and make sure rtr doesn't relay it.
 > expected_empty
 
@@ -17228,6 +17242,7 @@  send_igmp_v3_report hv2-vif3 hv2 \
 
 # Check that the IGMP Group is learned by all switches.
 wait_row_count IGMP_Group 2 address=239.0.1.68
+check ovn-nbctl --wait=hv sync
 
 # Send traffic from sw3 and make sure it is relayed by rtr.
 # to ports that joined.
@@ -17277,6 +17292,7 @@  OVS_WAIT_UNTIL([
     total_entries=`ovn-sbctl find IGMP_Group | grep "239.0.1.68" -c`
     test "${total_entries}" = "3"
 ])
+check ovn-nbctl --wait=hv sync
 
 # Send traffic from sw3 and make sure it is relayed by rtr
 # to ports that joined.
@@ -17340,6 +17356,7 @@  OVS_WAIT_UNTIL([
     total_entries=`ovn-sbctl find IGMP_Group | grep "239.0.1.68" -c`
     test "${total_entries}" = "0"
 ])
+check ovn-nbctl --wait=hv sync
 
 as hv1 reset_pcap_file hv1-vif1 hv1/vif1
 as hv1 reset_pcap_file hv1-vif2 hv1/vif2
@@ -17378,6 +17395,7 @@  OVS_WAIT_UNTIL([
     total_entries=`ovn-sbctl find IGMP_Group | grep "239.0.1.68" -c`
     test "${total_entries}" = "1"
 ])
+check ovn-nbctl --wait=hv sync
 
 # Send traffic from sw1-p21
 send_ip_multicast_pkt hv2-vif1 hv2 \
@@ -17393,8 +17411,6 @@  store_ip_multicast_pkt \
     $(ip_to_hex 10 0 0 42) $(ip_to_hex 239 0 1 68) 1e 1f cb70 11 \
     e518e518000a3b3a0000 expected_routed
 
-# Sleep a bit to make sure no duplicate traffic is received
-sleep 1
 
 # Check that traffic is switched to sw1-p11 and sw1-p12
 # Check that IGMP join is flooded on sw1-p21
@@ -17700,7 +17716,7 @@  OVN_CHECK_PACKETS([hv1/vif2-tx.pcap], [expected])
 OVN_CHECK_PACKETS([hv2/vif1-tx.pcap], [expected])
 OVN_CHECK_PACKETS([hv2/vif2-tx.pcap], [expected])
 
-ovn-nbctl --wait=hv sync
+check ovn-nbctl --wait=hv sync
 
 # Inject MLD Join for ff0a:dead:beef::1 on sw1-p11.
 send_mld_v2_report hv1-vif1 hv1 \
@@ -17716,6 +17732,9 @@  send_mld_v2_report hv2-vif1 hv2 \
 # Check that the IP multicast group is learned on both hv.
 wait_row_count IGMP_Group 2 address='"ff0a:dead:beef::1"'
 
+# This gives the ovn-controller nodes a chance to see the new IGMP_Group.
+check ovn-nbctl --wait=hv sync
+
 # Send traffic and make sure it gets forwarded only on the two ports that
 # joined.
 > expected
@@ -17813,7 +17832,7 @@  OVN_CHECK_PACKETS([hv2/vif2-tx.pcap], [expected])
 
 # Enable multicast snooping and querier on sw2 and set query interval to
 # minimum.
-ovn-nbctl set Logical_Switch sw2 \
+check ovn-nbctl --wait=hv set Logical_Switch sw2 \
     other_config:mcast_snoop="true" \
     other_config:mcast_querier="true" \
     other_config:mcast_query_interval=1 \
@@ -17824,7 +17843,6 @@  ovn-nbctl set Logical_Switch sw2 \
 > expected
 store_mld_query 0000000002fe fe8000000000000000000000000000fe expected
 store_mld_query 0000000002fe fe8000000000000000000000000000fe expected
-sleep 1
 
 OVN_CHECK_PACKETS([hv1/vif3-tx.pcap], [expected])
 OVN_CHECK_PACKETS([hv2/vif3-tx.pcap], [expected])
@@ -17834,7 +17852,7 @@  ovn-nbctl set Logical_Switch sw2 \
     other_config:mcast_querier="false"
 
 # Enable multicast snooping on sw3.
-ovn-nbctl set Logical_Switch sw3       \
+check ovn-nbctl --wait=sb set Logical_Switch sw3       \
     other_config:mcast_querier="false" \
     other_config:mcast_snoop="true"
 
@@ -17868,8 +17886,7 @@  OVN_CHECK_PACKETS([hv2/vif3-tx.pcap], [expected_empty])
 OVN_CHECK_PACKETS([hv2/vif4-tx.pcap], [expected_empty])
 
 # Enable multicast relay on rtr
-ovn-nbctl set logical_router rtr \
-    options:mcast_relay="true"
+check ovn-nbctl --wait=sb set logical_router rtr options:mcast_relay="true"
 
 # Inject MLD Join for ff0a:dead:beef::1 on sw1-p11.
 send_mld_v2_report hv1-vif1 hv1 \
@@ -18045,6 +18062,7 @@  send_mld_v2_report hv1-vif2 hv1 \
 
 # Check that the IP multicast group is learned.
 wait_row_count IGMP_Group 1 address='"ff0a:dead:beef::1"'
+check ovn-nbctl --wait=hv sync
 
 # Send traffic from sw1-p21
 send_ip_multicast_pkt hv2-vif1 hv2 \
@@ -18295,6 +18313,8 @@  m4_define([DVR_N_S_ARP_HANDLING],
    ovn-nbctl lrp-set-gateway-chassis router-to-underlay hv3
    ovn-nbctl --wait=sb sync
 
+   wait_row_count Port_Binding 1 logical_port=cr-router-to-underlay
+
    # Dump a bunch of info helpful for debugging if there's a failure.
 
    echo "------ OVN dump ------"
@@ -19833,6 +19853,7 @@  ovn-nbctl lr-nat-add rtr1 dnat_and_snat 10.0.0.111 42.42.42.1
 ovn-nbctl lr-nat-add rtr1 dnat_and_snat 10::111 42::1
 ovn-nbctl lr-nat-add rtr2 dnat_and_snat 10.0.0.222 42.42.42.2
 ovn-nbctl lr-nat-add rtr2 dnat_and_snat 10::222 42::2
+check ovn-nbctl --wait=hv sync
 
 # Configure FIP1 and FIP2 on rtr1 for sw1-p1 and sw1-p2.
 ovn-nbctl lr-nat-add rtr1 dnat_and_snat 10.0.0.121 20.0.0.11 sw1-p1 00:00:00:01:00:00
@@ -20147,6 +20168,7 @@  for az in `seq 1 $n_az`; do
         ovn-nbctl lsp-set-options lsp-ts$i-lr$az-$i router-port=lrp-lr$az-$i-ts$i
         ovn-nbctl lrp-set-gateway-chassis lrp-lr$az-$i-ts$i gw$az
     done
+    check ovn-nbctl --wait=hv sync
 done
 
 # Pre-populate the hypervisors' ARP tables so that we don't lose any
@@ -20379,10 +20401,7 @@  ovs-vsctl add-port br-int vif3 -- set Interface vif3 external-ids:iface-id=lsp22
 
 # Add a forwarding group on ls2 with lsp21 and lsp22 as child ports
 # virtual IP - 172.16.1.11, virtual MAC - 00:11:de:ad:be:ef
-ovn-nbctl fwd-group-add fwd_grp1 ls2 172.16.1.11 00:11:de:ad:be:ef lsp21 lsp22
-
-# Allow some time for ovn-northd and ovn-controller to catch up.
-sleep 1
+check ovn-nbctl --wait=hv fwd-group-add fwd_grp1 ls2 172.16.1.11 00:11:de:ad:be:ef lsp21 lsp22
 
 # Check logical flow
 AT_CHECK([ovn-sbctl dump-flows | grep ls_in_l2_lkup | grep fwd_group | wc -l], [0], [dnl
@@ -20426,10 +20445,7 @@  ovn-nbctl fwd-group-del fwd_grp1
 
 # Add a forwarding group with liveness on ls2 with lsp21 and lsp22 as child
 # ports virtual IP - 172.16.1.11, virtual MAC - 00:11:de:ad:be:ef
-ovn-nbctl --liveness fwd-group-add fwd_grp1 ls2 172.16.1.11 00:11:de:ad:be:ef lsp21 lsp22
-
-# Allow some time for ovn-northd and ovn-controller to catch up.
-sleep 1
+check ovn-nbctl --wait=hv --liveness fwd-group-add fwd_grp1 ls2 172.16.1.11 00:11:de:ad:be:ef lsp21 lsp22
 
 # Verify openflow group members
 ofport_lsp21=$(as hv1 ovs-vsctl --bare --columns ofport find Interface name=ovn-hv2-0)
@@ -20744,6 +20760,7 @@  ovn-nbctl lsp-set-options ln-public network_name=public
 
 ovn-nbctl lrp-set-gateway-chassis lr0-public hv1 20
 ovn-nbctl lr-nat-add lr0 snat 172.168.0.100 10.0.0.0/24
+check ovn-nbctl --wait=hv sync
 lr0_dp_uuid=$(ovn-sbctl --bare --columns _uuid list datapath_binding lr0)
 ovn-sbctl create mac_binding datapath=$lr0_dp_uuid ip=172.168.0.120 \
 logical_port=lr0-public mac="10\:54\:00\:00\:00\:03"
@@ -20793,6 +20810,7 @@  OVS_WAIT_UNTIL([
 ])
 
 as hv1 ovs-ofctl dump-flows br-int table=20 | grep NXM_NX_PKT_MARK
+check ovn-nbctl --wait=hv sync
 
 OVS_WAIT_UNTIL([
     test 1 -eq $(as hv1 ovs-ofctl dump-flows br-int table=20 | \
@@ -20815,6 +20833,7 @@  AT_CHECK([as hv1 ovs-ofctl --protocols=OpenFlow13 add-flows br-phys flows.txt])
 ip_to_hex() {
      printf "%02x%02x%02x%02x" "$@"
 }
+sleep 5
 
 send_ipv4_pkt() {
     local hv=$1 inport=$2 eth_src=$3 eth_dst=$4
@@ -22363,6 +22382,7 @@  OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [1.expected])
 # Delete one of the ACLs.
 ovn-nbctl acl-del ls1 to-lport 1001 \
 'outport == "lsp1" && ip4 && ip4.src == {10.0.0.2, 10.0.0.3}'
+check ovn-nbctl --wait=hv sync
 
 test_ip 2 f00000000002 f00000000001 $sip $dip 1
 OVN_CHECK_PACKETS([hv1/vif1-tx.pcap], [1.expected])
@@ -22375,6 +22395,7 @@  ovn-nbctl acl-add ls1 to-lport 1001 \
 # Delete the ACL that has "allow" action
 ovn-nbctl acl-del ls1 to-lport 1001 \
 'outport == "lsp1" && ip4 && ip4.src == 10.0.0.2'
+check ovn-nbctl --wait=hv sync
 
 # Packet should be dropped
 test_ip 2 f00000000002 f00000000001 $sip $dip
@@ -22385,6 +22406,7 @@  ovn-nbctl acl-add ls1 to-lport 1001 \
 'outport == "lsp1" && ip4 && ip4.src == 10.0.0.2' allow
 ovn-nbctl acl-del ls1 to-lport 1001 \
 'outport == "lsp1" && ip4 && ip4.src == {10.0.0.2, 10.0.0.3}'
+check ovn-nbctl --wait=hv sync
 
 # Packet should be received
 test_ip 2 f00000000002 f00000000001 $sip $dip 1