diff mbox series

[ovs-dev,ovn] Allow force_snat options to work for dual-stack routers.

Message ID 20200716180606.2820933-1-mmichels@redhat.com
State Superseded
Headers show
Series [ovs-dev,ovn] Allow force_snat options to work for dual-stack routers. | expand

Commit Message

Mark Michelson July 16, 2020, 6:06 p.m. UTC
The lb_force_snat and dnat_force_snat options could accept only a single
IP address. For routers that only route traffic of a single IP address
family, this is fine. However, if a router routes both IPv4 and IPv6
traffic, then this limitation is a problem.

This patch addresses this problem by allowing for these options to
specify both an IPv4 and IPv6 address.

Signed-off-by: Mark Michelson <mmichels@redhat.com>
---
 northd/ovn-northd.c | 179 ++++++++-------
 ovn-nb.xml          |  24 +-
 tests/system-ovn.at | 541 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 649 insertions(+), 95 deletions(-)

Comments

Numan Siddique Aug. 6, 2020, 6:42 p.m. UTC | #1
On Thu, Jul 16, 2020 at 11:37 PM Mark Michelson <mmichels@redhat.com> wrote:
>
> The lb_force_snat and dnat_force_snat options could accept only a single
> IP address. For routers that only route traffic of a single IP address
> family, this is fine. However, if a router routes both IPv4 and IPv6
> traffic, then this limitation is a problem.
>
> This patch addresses this problem by allowing for these options to
> specify both an IPv4 and IPv6 address.
>
> Signed-off-by: Mark Michelson <mmichels@redhat.com>

Hi Mark,

A couple of  comments below.

Thanks
Numan



> ---
>  northd/ovn-northd.c | 179 ++++++++-------
>  ovn-nb.xml          |  24 +-
>  tests/system-ovn.at | 541 ++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 649 insertions(+), 95 deletions(-)
>
> diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c
> index 192198272..2c05d1c2a 100644
> --- a/northd/ovn-northd.c
> +++ b/northd/ovn-northd.c
> @@ -7801,44 +7801,37 @@ op_put_v6_networks(struct ds *ds, const struct ovn_port *op)
>      ds_put_cstr(ds, "}");
>  }
>
> -static const char *
> +static bool
>  get_force_snat_ip(struct ovn_datapath *od, const char *key_type,
> -                  struct v46_ip *ip)
> +                  struct lport_addresses *laddrs)
>  {
>      char *key = xasprintf("%s_force_snat_ip", key_type);
> -    const char *ip_address = smap_get(&od->nbr->options, key);
> +    const char *addresses = smap_get(&od->nbr->options, key);
>      free(key);
>
> -    if (ip_address) {
> -        ovs_be32 mask;
> -        ip->family = AF_INET;
> -        char *error = ip_parse_masked(ip_address, &ip->ipv4, &mask);
> -        if (error || mask != OVS_BE32_MAX) {
> -            free(error);
> -            struct in6_addr mask_v6, v6_exact = IN6ADDR_EXACT_INIT;
> -            ip->family = AF_INET6;
> -            error = ipv6_parse_masked(ip_address, &ip->ipv6, &mask_v6);
> -            if (error || memcmp(&mask_v6, &v6_exact, sizeof(mask_v6))) {
> -                static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
> -                VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"",
> -                             ip_address, UUID_ARGS(&od->key));
> -                memset(ip, 0, sizeof *ip);
> -                ip->family = AF_UNSPEC;
> -                return NULL;
> -            }
> -        }
> -        return ip_address;
> +    if (!addresses) {
> +        return false;
>      }
>
> -    memset(ip, 0, sizeof *ip);
> -    ip->family = AF_UNSPEC;
> -    return NULL;
> +    if (!extract_ip_addresses(addresses, laddrs) ||
> +        laddrs->n_ipv4_addrs > 1 ||
> +        laddrs->n_ipv6_addrs > 1 ||
> +        (laddrs->n_ipv4_addrs && laddrs->ipv4_addrs[0].plen != 32) ||
> +        (laddrs->n_ipv6_addrs && laddrs->ipv6_addrs[0].plen != 128)) {
> +        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
> +        VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"",
> +                     addresses, UUID_ARGS(&od->key));
> +        destroy_lport_addresses(laddrs);
> +        return false;
> +    }
> +
> +    return true;
>  }
>
>  static void
>  add_router_lb_flow(struct hmap *lflows, struct ovn_datapath *od,
>                     struct ds *match, struct ds *actions, int priority,
> -                   const char *lb_force_snat_ip, struct lb_vip *lb_vip,
> +                   bool lb_force_snat_ip, struct lb_vip *lb_vip,
>                     const char *proto, struct nbrec_load_balancer *lb,
>                     struct shash *meter_groups, struct sset *nat_entries)
>  {
> @@ -8159,6 +8152,29 @@ build_lrouter_nd_flow(struct ovn_datapath *od, struct ovn_port *op,
>      ds_destroy(&actions);
>  }
>
> +static void
> +build_lrouter_force_snat_flows(struct hmap *lflows, struct ovn_datapath *od,
> +                               const char *ip_version, const char *ip_addr,
> +                               const char *context)
> +{
> +    struct ds match = DS_EMPTY_INITIALIZER;
> +    struct ds actions = DS_EMPTY_INITIALIZER;
> +    ds_put_format(&match, "ip%s && ip%s.dst == %s",
> +                  ip_version, ip_version, ip_addr);
> +    ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110,
> +                  ds_cstr(&match), "ct_snat;");
> +
> +    /* Higher priority rules to force SNAT with the IP addresses
> +     * configured in the Gateway router.  This only takes effect
> +     * when the packet has already been DNATed or load balanced once. */
> +    ds_clear(&match);
> +    ds_put_format(&match, "flags.force_snat_for_%s == 1 && ip%s",
> +                  context, ip_version);
> +    ds_put_format(&actions, "ct_snat(%s);", ip_addr);
> +    ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100,
> +                  ds_cstr(&match), ds_cstr(&actions));

There is a memory leak here. I think you need to call ds_destroy for
both 'match' and 'actions'.


> +}
> +
>  static void
>  build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
>                      struct hmap *lflows, struct shash *meter_groups,
> @@ -8609,24 +8625,37 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
>              }
>          }
>
> -        /* A gateway router can have 2 SNAT IP addresses to force DNATed and
> +        /* A gateway router can have 4 SNAT IP addresses to force DNATed and
>           * LBed traffic respectively to be SNATed.  In addition, there can be
>           * a number of SNAT rules in the NAT table. */
>          struct v46_ip *snat_ips = xmalloc(sizeof *snat_ips
> -                                          * (op->od->nbr->n_nat + 2));
> +                                          * (op->od->nbr->n_nat + 4));
>          size_t n_snat_ips = 0;
> +        struct lport_addresses snat_addrs;
>
> -        struct v46_ip snat_ip;
> -        const char *dnat_force_snat_ip = get_force_snat_ip(op->od, "dnat",
> -                                                           &snat_ip);
> -        if (dnat_force_snat_ip) {
> -            snat_ips[n_snat_ips++] = snat_ip;
> +        if (get_force_snat_ip(op->od, "dnat", &snat_addrs)) {
> +            if (snat_addrs.n_ipv4_addrs) {
> +                snat_ips[n_snat_ips].family = AF_INET;
> +                snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr;
> +            }
> +            if (snat_addrs.n_ipv6_addrs) {
> +                snat_ips[n_snat_ips].family = AF_INET6;
> +                snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr;
> +            }
> +            destroy_lport_addresses(&snat_addrs);
>          }
>
> -        const char *lb_force_snat_ip = get_force_snat_ip(op->od, "lb",
> -                                                         &snat_ip);
> -        if (lb_force_snat_ip) {
> -            snat_ips[n_snat_ips++] = snat_ip;
> +        memset(&snat_addrs, 0, sizeof(snat_addrs));
> +        if (get_force_snat_ip(op->od, "lb", &snat_addrs)) {
> +            if (snat_addrs.n_ipv4_addrs) {
> +                snat_ips[n_snat_ips].family = AF_INET;
> +                snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr;
> +            }
> +            if (snat_addrs.n_ipv6_addrs) {
> +                snat_ips[n_snat_ips].family = AF_INET6;
> +                snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr;
> +            }
> +            destroy_lport_addresses(&snat_addrs);
>          }
>
>          for (size_t i = 0; i < op->od->nbr->n_nat; i++) {
> @@ -8985,11 +9014,12 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
>
>          struct sset nat_entries = SSET_INITIALIZER(&nat_entries);
>
> -        struct v46_ip snat_ip, lb_snat_ip;
> -        const char *dnat_force_snat_ip = get_force_snat_ip(od, "dnat",
> -                                                           &snat_ip);
> -        const char *lb_force_snat_ip = get_force_snat_ip(od, "lb",
> -                                                         &lb_snat_ip);
> +        struct lport_addresses dnat_force_snat_addrs;
> +        struct lport_addresses lb_force_snat_addrs;
> +        bool dnat_force_snat_ip = get_force_snat_ip(od, "dnat",
> +                                                    &dnat_force_snat_addrs);
> +        bool lb_force_snat_ip = get_force_snat_ip(od, "lb",
> +                                                  &lb_force_snat_addrs);

'dnat_force_snat_addrs and 'lb_force_snat_addrs' are destroyed below
if the logical router doesn't
have a gateway router port.

What if there is a logical router with these options set and also has
a gw router port. Even though
its misconfiguration from the user, but ovn-northd will leak the memory.

I think this should be handled properly.


>
>          for (int i = 0; i < od->nbr->n_nat; i++) {
>              const struct nbrec_nat *nat;
> @@ -9455,49 +9485,30 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
>          }
>
>          /* Handle force SNAT options set in the gateway router. */
> -        if (dnat_force_snat_ip && !od->l3dgw_port) {
> -            /* If a packet with destination IP address as that of the
> -             * gateway router (as set in options:dnat_force_snat_ip) is seen,
> -             * UNSNAT it. */
> -            ds_clear(&match);
> -            ds_put_format(&match, "ip && ip%s.dst == %s",
> -                          snat_ip.family == AF_INET ? "4" : "6",
> -                          dnat_force_snat_ip);
> -            ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110,
> -                          ds_cstr(&match), "ct_snat;");
> -
> -            /* Higher priority rules to force SNAT with the IP addresses
> -             * configured in the Gateway router.  This only takes effect
> -             * when the packet has already been DNATed once. */
> -            ds_clear(&match);
> -            ds_put_format(&match, "flags.force_snat_for_dnat == 1 && ip");
> -            ds_clear(&actions);
> -            ds_put_format(&actions, "ct_snat(%s);", dnat_force_snat_ip);
> -            ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100,
> -                          ds_cstr(&match), ds_cstr(&actions));
> -        }
> -        if (lb_force_snat_ip && !od->l3dgw_port) {
> -            /* If a packet with destination IP address as that of the
> -             * gateway router (as set in options:lb_force_snat_ip) is seen,
> -             * UNSNAT it. */
> -            ds_clear(&match);
> -            ds_put_format(&match, "ip && ip%s.dst == %s",
> -                          lb_snat_ip.family == AF_INET ? "4" : "6",
> -                          lb_force_snat_ip);
> -            ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 100,
> -                          ds_cstr(&match), "ct_snat;");
> -
> -            /* Load balanced traffic will have flags.force_snat_for_lb set.
> -             * Force SNAT it. */
> -            ds_clear(&match);
> -            ds_put_format(&match, "flags.force_snat_for_lb == 1 && ip");
> -            ds_clear(&actions);
> -            ds_put_format(&actions, "ct_snat(%s);", lb_force_snat_ip);
> -            ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100,
> -                          ds_cstr(&match), ds_cstr(&actions));
> -        }
> -
>          if (!od->l3dgw_port) {
> +            if (dnat_force_snat_ip) {
> +                if (dnat_force_snat_addrs.n_ipv4_addrs) {
> +                    build_lrouter_force_snat_flows(lflows, od, "4",
> +                        dnat_force_snat_addrs.ipv4_addrs[0].addr_s, "dnat");
> +                }
> +                if (dnat_force_snat_addrs.n_ipv6_addrs) {
> +                    build_lrouter_force_snat_flows(lflows, od, "6",
> +                        dnat_force_snat_addrs.ipv6_addrs[0].addr_s, "dnat");
> +                }
> +                destroy_lport_addresses(&dnat_force_snat_addrs);
> +            }
> +            if (lb_force_snat_ip) {
> +                if (lb_force_snat_addrs.n_ipv4_addrs) {
> +                    build_lrouter_force_snat_flows(lflows, od, "4",
> +                        lb_force_snat_addrs.ipv4_addrs[0].addr_s, "lb");
> +                }
> +                if (lb_force_snat_addrs.n_ipv6_addrs) {
> +                    build_lrouter_force_snat_flows(lflows, od, "6",
> +                        lb_force_snat_addrs.ipv6_addrs[0].addr_s, "lb");
> +                }
> +                destroy_lport_addresses(&lb_force_snat_addrs);
> +            }
> +
>              /* For gateway router, re-circulate every packet through
>              * the DNAT zone.  This helps with the following.
>              *
> diff --git a/ovn-nb.xml b/ovn-nb.xml
> index db5908cd5..9f3da3563 100644
> --- a/ovn-nb.xml
> +++ b/ovn-nb.xml
> @@ -1817,27 +1817,29 @@
>        </column>
>        <column name="options" key="dnat_force_snat_ip">
>          <p>
> -          If set, indicates the IP address to use to force SNAT a packet
> -          that has already been DNATed in the gateway router.  When multiple
> -          gateway routers are configured, a packet can potentially enter any
> -          of the gateway router, get DNATted and eventually reach the logical
> -          switch port.  For the return traffic to go back to the same gateway
> -          router (for unDNATing), the packet needs a SNAT in the first place.
> -          This can be achieved by setting the above option with a gateway
> -          specific IP address.
> +          If set, indicates a set of IP addresses to use to force SNAT a
> +          packet that has already been DNATed in the gateway router.  When
> +          multiple gateway routers are configured, a packet can potentially
> +          enter any of the gateway router, get DNATted and eventually reach the
> +          logical switch port.  For the return traffic to go back to the same
> +          gateway router (for unDNATing), the packet needs a SNAT in the first
> +          place. This can be achieved by setting the above option with a
> +          gateway specific set of IP addresses. This option may have exactly
> +          one IPv4 and/or one IPv6 address on it, separated by a a space.
>          </p>
>        </column>
>        <column name="options" key="lb_force_snat_ip">
>          <p>
> -          If set, indicates the IP address to use to force SNAT a packet
> +          If set, indicates a set of IP addresses to use to force SNAT a packet
>            that has already been load-balanced in the gateway router.  When
>            multiple gateway routers are configured, a packet can potentially
>            enter any of the gateway routers, get DNATted as part of the load-
>            balancing and eventually reach the logical switch port.
>            For the return traffic to go back to the same gateway router (for
>            unDNATing), the packet needs a SNAT in the first place.  This can be
> -          achieved by setting the above option with a gateway specific IP
> -          address.
> +          achieved by setting the above option with a gateway specific set of
> +          IP addresses. This option may have exactly one IPv4 and/or one IPv6
> +          address on it, separated by a space character.
>          </p>
>        </column>
>        <column name="options" key="mcast_relay" type='{"type": "boolean"}'>
> diff --git a/tests/system-ovn.at b/tests/system-ovn.at
> index 2999e52fd..079d61975 100644
> --- a/tests/system-ovn.at
> +++ b/tests/system-ovn.at
> @@ -1026,6 +1026,323 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
>  /connection dropped.*/d"])
>  AT_CLEANUP
>
> +AT_SETUP([ovn -- multiple gateway routers, SNAT and DNAT - Dual Stack])
> +AT_KEYWORDS([ovnnat])
> +
> +CHECK_CONNTRACK()
> +CHECK_CONNTRACK_NAT()
> +ovn_start
> +OVS_TRAFFIC_VSWITCHD_START()
> +ADD_BR([br-int])
> +
> +# Set external-ids in br-int needed for ovn-controller
> +ovs-vsctl \
> +        -- set Open_vSwitch . external-ids:system-id=hv1 \
> +        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
> +        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
> +        -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \
> +        -- set bridge br-int fail-mode=secure other-config:disable-in-band=true
> +
> +# Start ovn-controller
> +start_daemon ovn-controller
> +
> +# Logical network:
> +# Three LRs - R1, R2 and R3 that are connected to each other via LS "join"
> +# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24
> +# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2
> +# has alice (172.16.1.0/24 and fd30::/64) connected to it.  R3 has bob
> +# (172.16.1.0/24 andfd30::/64) connected to it. Note how both alice and bob
> +# have the same subnets behind them.  We are trying to simulate external network
> +# via those 2 switches. In real world the switch ports of these switches will
> +# have addresses set as "unknown" to make them learning switches. Or those
> +# switches will be "localnet" ones.
> +#
> +#    foo -- R1 -- join - R2 -- alice
> +#           |          |
> +#    bar ----          - R3 --- bob
> +
> +ovn-nbctl create Logical_Router name=R1
> +ovn-nbctl create Logical_Router name=R2 options:chassis=hv1
> +ovn-nbctl create Logical_Router name=R3 options:chassis=hv1
> +
> +ovn-nbctl ls-add foo
> +ovn-nbctl ls-add bar
> +ovn-nbctl ls-add alice
> +ovn-nbctl ls-add bob
> +ovn-nbctl ls-add join
> +
> +# Connect foo to R1
> +ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64
> +ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \
> +    type=router options:router-port=foo addresses=\"00:00:01:01:02:03\"
> +
> +# Connect bar to R1
> +ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64
> +ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \
> +    type=router options:router-port=bar addresses=\"00:00:01:01:02:04\"
> +
> +# Connect alice to R2
> +ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd30::1/64
> +ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \
> +    type=router options:router-port=alice addresses=\"00:00:02:01:02:03\"
> +
> +# Connect bob to R3
> +ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd30::2/64
> +ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \
> +    type=router options:router-port=bob addresses=\"00:00:03:01:02:03\"
> +
> +# Connect R1 to join
> +ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64
> +ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \
> +    type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"'
> +
> +# Connect R2 to join
> +ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64
> +ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \
> +    type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"'
> +
> +# Connect R3 to join
> +ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64
> +ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \
> +    type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"'
> +
> +# Install static routes with source ip address as the policy for routing.
> +# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3.
> +ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2
> +ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3
> +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2
> +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3
> +
> +# Static routes.
> +ovn-nbctl lr-route-add R2 fd11::/64 fd20::1
> +ovn-nbctl lr-route-add R2 fd12::/64 fd20::1
> +ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1
> +ovn-nbctl lr-route-add R3 fd11::/64 fd20::1
> +ovn-nbctl lr-route-add R3 fd12::/64 fd20::1
> +ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1
> +
> +# For gateway routers R2 and R3, set a force SNAT rule.
> +ovn-nbctl set logical_router R2 options:dnat_force_snat_ip="20.0.0.2 fd20::2"
> +ovn-nbctl set logical_router R3 options:dnat_force_snat_ip="20.0.0.3 fd20::3"
> +
> +# Logical port 'foo1' in switch 'foo'.
> +ADD_NAMESPACES(foo1)
> +ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \
> +         "192.168.1.1")
> +ovn-nbctl lsp-add foo foo1 \
> +-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2"
> +
> +ADD_NAMESPACES(foo16)
> +ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:00:02:02:03", \
> +         "fd11::1")
> +OVS_WAIT_UNTIL([test "$(ip netns exec foo16 ip a | grep fd11::2 | grep tentative)" = ""])
> +ovn-nbctl lsp-add foo foo16 \
> +-- lsp-set-addresses foo16 "f0:00:00:02:02:03 fd11::2"
> +
> +# Logical port 'alice1' in switch 'alice'.
> +ADD_NAMESPACES(alice1)
> +ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \
> +         "172.16.1.1")
> +ovn-nbctl lsp-add alice alice1 \
> +-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3"
> +
> +ADD_NAMESPACES(alice16)
> +ADD_VETH(alice16, alice16, br-int, "fd30::3/64", "f0:00:00:02:02:04", \
> +         "fd30::1")
> +OVS_WAIT_UNTIL([test "$(ip netns exec alice16 ip a | grep fd30::3 | grep tentative)" = ""])
> +ovn-nbctl lsp-add alice alice16 \
> +-- lsp-set-addresses alice16 "f0:00:00:02:02:04 fd30::3"
> +
> +# Logical port 'bar1' in switch 'bar'.
> +ADD_NAMESPACES(bar1)
> +ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \
> +"192.168.2.1")
> +ovn-nbctl lsp-add bar bar1 \
> +-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2"
> +
> +ADD_NAMESPACES(bar16)
> +ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:00:02:02:05", \
> +         "fd12::1")
> +OVS_WAIT_UNTIL([test "$(ip netns exec bar16 ip a | grep fd12::2 | grep tentative)" = ""])
> +ovn-nbctl lsp-add bar bar16 \
> +-- lsp-set-addresses bar16 "f0:00:00:02:02:05 fd12::2"
> +
> +# Logical port 'bob1' in switch 'bob'.
> +ADD_NAMESPACES(bob1)
> +ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \
> +         "172.16.1.2")
> +ovn-nbctl lsp-add bob bob1 \
> +-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4"
> +
> +ADD_NAMESPACES(bob16)
> +ADD_VETH(bob16, bob16, br-int, "fd30::4/64", "f0:00:00:02:02:06", \
> +         "fd30::2")
> +OVS_WAIT_UNTIL([test "$(ip netns exec bob16 ip a | grep fd30::4 | grep tentative)" = ""])
> +ovn-nbctl lsp-add bob bob16 \
> +-- lsp-set-addresses bob16 "f0:00:00:02:02:06 fd30::4"
> +
> +# Router R2
> +# Add a DNAT rule.
> +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \
> +    external_ip=30.0.0.2 -- add logical_router R2 nat @nat
> +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \
> +    external_ip='"fd40::2"' -- add logical_router R2 nat @nat
> +
> +# Add a SNAT rule
> +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.1.2 \
> +    external_ip=30.0.0.1 -- add logical_router R2 nat @nat
> +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd11::2"' \
> +    external_ip='"fd40::1"' -- add logical_router R2 nat @nat
> +
> +# Router R3
> +# Add a DNAT rule.
> +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \
> +    external_ip=30.0.0.3 -- add logical_router R3 nat @nat
> +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \
> +    external_ip='"fd40::3"' -- add logical_router R3 nat @nat
> +
> +# Add a SNAT rule
> +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.2.2 \
> +    external_ip=30.0.0.4 -- add logical_router R3 nat @nat
> +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd12::2"' \
> +    external_ip='"fd40::4"' -- add logical_router R3 nat @nat
> +
> +# wait for ovn-controller to catch up.
> +ovn-nbctl --wait=hv sync
> +OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=fd40::4)'])
> +OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=30.0.0.4)'])
> +
> +# North-South DNAT: 'alice1' should be able to ping 'foo1' via 30.0.0.2
> +NS_CHECK_EXEC([alice1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.2 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# North-South DNAT: 'alice16' should be able to ping 'foo16' via fd30::2
> +NS_CHECK_EXEC([alice16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::2 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# Check conntrack entries.
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.3) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmp,orig=(src=172.16.1.3,dst=30.0.0.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=0,code=0),zone=<cleared>
> +])
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::3) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmpv6,orig=(src=fd30::3,dst=fd40::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::3,id=<cleared>,type=129,code=0),zone=<cleared>
> +])
> +
> +# But foo1 should receive traffic from 20.0.0.2
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.2) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmp,orig=(src=172.16.1.3,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.2,id=<cleared>,type=0,code=0),zone=<cleared>
> +])
> +# But foo16 should receive traffic from fd20::2
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmpv6,orig=(src=fd30::3,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::2,id=<cleared>,type=129,code=0),zone=<cleared>
> +])
> +
> +# North-South DNAT: 'bob1' should be able to ping 'foo1' via 30.0.0.3
> +NS_CHECK_EXEC([bob1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.3 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# North-South DNAT: 'bob16' should be able to ping 'foo16' via fd40::3
> +NS_CHECK_EXEC([bob16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::3 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# Check conntrack entries.
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.4) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmp,orig=(src=172.16.1.4,dst=30.0.0.3,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.4,id=<cleared>,type=0,code=0),zone=<cleared>
> +])
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::4) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmpv6,orig=(src=fd30::4,dst=fd40::3,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::4,id=<cleared>,type=129,code=0),zone=<cleared>
> +])
> +
> +# But foo1 should receive traffic from 20.0.0.3
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.3) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmp,orig=(src=172.16.1.4,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.3,id=<cleared>,type=0,code=0),zone=<cleared>
> +])
> +
> +# But foo16 should receive traffic from fd20::3
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::3) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmpv6,orig=(src=fd30::4,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::3,id=<cleared>,type=129,code=0),zone=<cleared>
> +])
> +
> +# South-North SNAT: 'bar1' pings 'bob1'. But 'bob1' receives traffic
> +# from 30.0.0.4
> +NS_CHECK_EXEC([bar1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.4 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +# South-North SNAT: 'bar16' pings 'bob16'. But 'bob16' receives traffic
> +# from fd40::4
> +NS_CHECK_EXEC([bar16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::4 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# We verify that SNAT indeed happened via 'dump-conntrack' command.
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.4) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmp,orig=(src=192.168.2.2,dst=172.16.1.4,id=<cleared>,type=8,code=0),reply=(src=172.16.1.4,dst=30.0.0.4,id=<cleared>,type=0,code=0),zone=<cleared>
> +])
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::4) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmpv6,orig=(src=fd12::2,dst=fd30::4,id=<cleared>,type=128,code=0),reply=(src=fd30::4,dst=fd40::4,id=<cleared>,type=129,code=0),zone=<cleared>
> +])
> +
> +# South-North SNAT: 'foo1' pings 'alice1'. But 'alice1' receives traffic
> +# from 30.0.0.1
> +NS_CHECK_EXEC([foo1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.3 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# South-North SNAT: 'foo16' pings 'alice16'. But 'alice16' receives traffic
> +# from fd40::1
> +NS_CHECK_EXEC([foo16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::3 | FORMAT_PING], \
> +[0], [dnl
> +3 packets transmitted, 3 received, 0% packet loss, time 0ms
> +])
> +
> +# We verify that SNAT indeed happened via 'dump-conntrack' command.
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmp,orig=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=8,code=0),reply=(src=172.16.1.3,dst=30.0.0.1,id=<cleared>,type=0,code=0),zone=<cleared>
> +])
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::1) | \
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +icmpv6,orig=(src=fd11::2,dst=fd30::3,id=<cleared>,type=128,code=0),reply=(src=fd30::3,dst=fd40::1,id=<cleared>,type=129,code=0),zone=<cleared>
> +])
> +
> +OVS_APP_EXIT_AND_WAIT([ovn-controller])
> +
> +as ovn-sb
> +OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +as ovn-nb
> +OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +as northd
> +OVS_APP_EXIT_AND_WAIT([ovn-northd])
> +
> +as
> +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
> +/connection dropped.*/d"])
> +AT_CLEANUP
> +
> +
>  AT_SETUP([ovn -- load-balancing])
>  AT_KEYWORDS([ovnlb])
>
> @@ -2405,6 +2722,230 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
>  /connection dropped.*/d"])
>  AT_CLEANUP
>
> +AT_SETUP([ovn -- multiple gateway routers, load-balancing - Dual Stack])
> +AT_KEYWORDS([ovnlb])
> +
> +CHECK_CONNTRACK()
> +CHECK_CONNTRACK_NAT()
> +ovn_start
> +OVS_TRAFFIC_VSWITCHD_START()
> +ADD_BR([br-int])
> +
> +# Set external-ids in br-int needed for ovn-controller
> +ovs-vsctl \
> +        -- set Open_vSwitch . external-ids:system-id=hv1 \
> +        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
> +        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
> +        -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \
> +        -- set bridge br-int fail-mode=secure other-config:disable-in-band=true
> +
> +# Start ovn-controller
> +start_daemon ovn-controller
> +
> +# Logical network:
> +# Three LRs - R1, R2 and R3 that are connected to each other via LS "join"
> +# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24
> +# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2
> +# has alice (172.16.1.0/24 and fd72::/64) connected to it.  R3 has bob
> +# (172.16.1.0/24 and fd72::/64) connected to it. Note how both alice and
> +# bob have the same subnets behind them.  We are trying to simulate external
> +# network via those 2 switches. In real world the switch ports of these
> +# switches will have addresses set as "unknown" to make them learning switches.
> +# Or those switches will be "localnet" ones.
> +#
> +#    foo -- R1 -- join - R2 -- alice
> +#           |          |
> +#    bar ----          - R3 --- bob
> +
> +ovn-nbctl create Logical_Router name=R1
> +ovn-nbctl create Logical_Router name=R2 options:chassis=hv1
> +ovn-nbctl create Logical_Router name=R3 options:chassis=hv1
> +
> +ovn-nbctl ls-add foo
> +ovn-nbctl ls-add bar
> +ovn-nbctl ls-add alice
> +ovn-nbctl ls-add bob
> +ovn-nbctl ls-add join
> +
> +# Connect foo to R1
> +ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64
> +ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \
> +    type=router options:router-port=foo addresses=\"00:00:01:01:02:03\"
> +
> +# Connect bar to R1
> +ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64
> +ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \
> +    type=router options:router-port=bar addresses=\"00:00:01:01:02:04\"
> +
> +# Connect alice to R2
> +ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd72::1/64
> +ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \
> +    type=router options:router-port=alice addresses=\"00:00:02:01:02:03\"
> +
> +# Connect bob to R3
> +ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd72::2/64
> +ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \
> +    type=router options:router-port=bob addresses=\"00:00:03:01:02:03\"
> +
> +# Connect R1 to join
> +ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64
> +ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \
> +    type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"'
> +
> +# Connect R2 to join
> +ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64
> +ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \
> +    type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"'
> +
> +# Connect R3 to join
> +ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64
> +ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \
> +    type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"'
> +
> +# Install static routes with source ip address as the policy for routing.
> +# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3.
> +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2
> +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3
> +ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2
> +ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3
> +
> +# Static routes.
> +ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1
> +ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1
> +ovn-nbctl lr-route-add R2 fd11::/64 fd20::1
> +ovn-nbctl lr-route-add R2 fd12::/64 fd20::1
> +ovn-nbctl lr-route-add R3 fd11::/64 fd20::1
> +ovn-nbctl lr-route-add R3 fd12::/64 fd20::1
> +
> +# For gateway routers R2 and R3, set a force SNAT rule.
> +ovn-nbctl set logical_router R2 options:lb_force_snat_ip="20.0.0.2 fd20::2"
> +ovn-nbctl set logical_router R3 options:lb_force_snat_ip="20.0.0.3 fd20::3"
> +
> +# Logical port 'foo1' in switch 'foo'.
> +ADD_NAMESPACES(foo1)
> +ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \
> +         "192.168.1.1")
> +ovn-nbctl lsp-add foo foo1 \
> +-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2"
> +
> +# Logical port 'foo16' in switch 'foo'.
> +ADD_NAMESPACES(foo16)
> +ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:06:01:02:03", \
> +         "fd11::1")
> +ovn-nbctl lsp-add foo foo16 \
> +-- lsp-set-addresses foo16 "f0:00:06:01:02:03 fd11::2"
> +
> +# Logical port 'alice1' in switch 'alice'.
> +ADD_NAMESPACES(alice1)
> +ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \
> +         "172.16.1.1")
> +ovn-nbctl lsp-add alice alice1 \
> +-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3"
> +
> +# Logical port 'alice16' in switch 'alice'.
> +ADD_NAMESPACES(alice16)
> +ADD_VETH(alice16, alice16, br-int, "fd72::3/64", "f0:00:06:01:02:04", \
> +         "fd72::1")
> +ovn-nbctl lsp-add alice alice16 \
> +-- lsp-set-addresses alice16 "f0:00:06:01:02:04 fd72::3"
> +
> +# Logical port 'bar1' in switch 'bar'.
> +ADD_NAMESPACES(bar1)
> +ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \
> +"192.168.2.1")
> +ovn-nbctl lsp-add bar bar1 \
> +-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2"
> +
> +# Logical port 'bar16' in switch 'bar'.
> +ADD_NAMESPACES(bar16)
> +ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:06:01:02:05", \
> +"fd12::1")
> +ovn-nbctl lsp-add bar bar16 \
> +-- lsp-set-addresses bar16 "f0:00:06:01:02:05 fd12::2"
> +
> +# Logical port 'bob1' in switch 'bob'.
> +ADD_NAMESPACES(bob1)
> +ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \
> +         "172.16.1.2")
> +ovn-nbctl lsp-add bob bob1 \
> +-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4"
> +
> +# Logical port 'bob16' in switch 'bob'.
> +ADD_NAMESPACES(bob16)
> +ADD_VETH(bob16, bob16, br-int, "fd72::4/64", "f0:00:06:01:02:06", \
> +         "fd72::2")
> +ovn-nbctl lsp-add bob bob16 \
> +-- lsp-set-addresses bob16 "f0:00:06:01:02:06 fd72::4"
> +
> +# Config OVN load-balancer with a VIP.
> +uuid=`ovn-nbctl  create load_balancer vips:30.0.0.1="192.168.1.2,192.168.2.2" \
> +vips:\"fd30::1\"=\"fd11::2,fd12::2\"`
> +ovn-nbctl set logical_router R2 load_balancer=$uuid
> +ovn-nbctl set logical_router R3 load_balancer=$uuid
> +
> +# Wait for ovn-controller to catch up.
> +ovn-nbctl --wait=hv sync
> +OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \
> +grep 'nat(dst=192.168.2.2)'])
> +OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \
> +grep 'nat(dst=fd12::2)'])
> +
> +# Start webservers in 'foo1', 'foo16, 'bar1', and 'bar16'.
> +OVS_START_L7([foo1], [http])
> +OVS_START_L7([bar1], [http])
> +OVS_START_L7([foo16], [http6])
> +OVS_START_L7([bar16], [http6])
> +
> +dnl Should work with the virtual IP address through NAT
> +for i in `seq 1 20`; do
> +    echo Request $i
> +    NS_CHECK_EXEC([alice1], [wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log])
> +done
> +
> +for i in `seq 1 20`; do
> +    echo Request ${i}_6
> +    NS_CHECK_EXEC([alice16], [wget http://[[fd30::1]] -t 5 -T 1 --retry-connrefused -v -o wget${i}_6.log])
> +done
> +
> +dnl Each server should have at least one connection.
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) |
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +])
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::1) | grep -v fe80 |
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +])
> +
> +dnl Force SNAT should have worked.
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0) |
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +tcp,orig=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +tcp,orig=(src=172.16.1.3,dst=192.168.2.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +])
> +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | grep -v fe80 |
> +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
> +tcp,orig=(src=fd72::3,dst=fd11::2,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +tcp,orig=(src=fd72::3,dst=fd12::2,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
> +])
> +OVS_APP_EXIT_AND_WAIT([ovn-controller])
> +
> +as ovn-sb
> +OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +as ovn-nb
> +OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +as northd
> +OVS_APP_EXIT_AND_WAIT([ovn-northd])
> +
> +as
> +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
> +/connection dropped.*/d"])
> +AT_CLEANUP
> +
>  AT_SETUP([ovn -- load balancing in router with gateway router port])
>  AT_KEYWORDS([ovnlb])
>
> --
> 2.25.4
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
diff mbox series

Patch

diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c
index 192198272..2c05d1c2a 100644
--- a/northd/ovn-northd.c
+++ b/northd/ovn-northd.c
@@ -7801,44 +7801,37 @@  op_put_v6_networks(struct ds *ds, const struct ovn_port *op)
     ds_put_cstr(ds, "}");
 }
 
-static const char *
+static bool
 get_force_snat_ip(struct ovn_datapath *od, const char *key_type,
-                  struct v46_ip *ip)
+                  struct lport_addresses *laddrs)
 {
     char *key = xasprintf("%s_force_snat_ip", key_type);
-    const char *ip_address = smap_get(&od->nbr->options, key);
+    const char *addresses = smap_get(&od->nbr->options, key);
     free(key);
 
-    if (ip_address) {
-        ovs_be32 mask;
-        ip->family = AF_INET;
-        char *error = ip_parse_masked(ip_address, &ip->ipv4, &mask);
-        if (error || mask != OVS_BE32_MAX) {
-            free(error);
-            struct in6_addr mask_v6, v6_exact = IN6ADDR_EXACT_INIT;
-            ip->family = AF_INET6;
-            error = ipv6_parse_masked(ip_address, &ip->ipv6, &mask_v6);
-            if (error || memcmp(&mask_v6, &v6_exact, sizeof(mask_v6))) {
-                static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
-                VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"",
-                             ip_address, UUID_ARGS(&od->key));
-                memset(ip, 0, sizeof *ip);
-                ip->family = AF_UNSPEC;
-                return NULL;
-            }
-        }
-        return ip_address;
+    if (!addresses) {
+        return false;
     }
 
-    memset(ip, 0, sizeof *ip);
-    ip->family = AF_UNSPEC;
-    return NULL;
+    if (!extract_ip_addresses(addresses, laddrs) ||
+        laddrs->n_ipv4_addrs > 1 ||
+        laddrs->n_ipv6_addrs > 1 ||
+        (laddrs->n_ipv4_addrs && laddrs->ipv4_addrs[0].plen != 32) ||
+        (laddrs->n_ipv6_addrs && laddrs->ipv6_addrs[0].plen != 128)) {
+        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+        VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"",
+                     addresses, UUID_ARGS(&od->key));
+        destroy_lport_addresses(laddrs);
+        return false;
+    }
+
+    return true;
 }
 
 static void
 add_router_lb_flow(struct hmap *lflows, struct ovn_datapath *od,
                    struct ds *match, struct ds *actions, int priority,
-                   const char *lb_force_snat_ip, struct lb_vip *lb_vip,
+                   bool lb_force_snat_ip, struct lb_vip *lb_vip,
                    const char *proto, struct nbrec_load_balancer *lb,
                    struct shash *meter_groups, struct sset *nat_entries)
 {
@@ -8159,6 +8152,29 @@  build_lrouter_nd_flow(struct ovn_datapath *od, struct ovn_port *op,
     ds_destroy(&actions);
 }
 
+static void
+build_lrouter_force_snat_flows(struct hmap *lflows, struct ovn_datapath *od,
+                               const char *ip_version, const char *ip_addr,
+                               const char *context)
+{
+    struct ds match = DS_EMPTY_INITIALIZER;
+    struct ds actions = DS_EMPTY_INITIALIZER;
+    ds_put_format(&match, "ip%s && ip%s.dst == %s",
+                  ip_version, ip_version, ip_addr);
+    ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110,
+                  ds_cstr(&match), "ct_snat;");
+
+    /* Higher priority rules to force SNAT with the IP addresses
+     * configured in the Gateway router.  This only takes effect
+     * when the packet has already been DNATed or load balanced once. */
+    ds_clear(&match);
+    ds_put_format(&match, "flags.force_snat_for_%s == 1 && ip%s",
+                  context, ip_version);
+    ds_put_format(&actions, "ct_snat(%s);", ip_addr);
+    ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100,
+                  ds_cstr(&match), ds_cstr(&actions));
+}
+
 static void
 build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
                     struct hmap *lflows, struct shash *meter_groups,
@@ -8609,24 +8625,37 @@  build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
             }
         }
 
-        /* A gateway router can have 2 SNAT IP addresses to force DNATed and
+        /* A gateway router can have 4 SNAT IP addresses to force DNATed and
          * LBed traffic respectively to be SNATed.  In addition, there can be
          * a number of SNAT rules in the NAT table. */
         struct v46_ip *snat_ips = xmalloc(sizeof *snat_ips
-                                          * (op->od->nbr->n_nat + 2));
+                                          * (op->od->nbr->n_nat + 4));
         size_t n_snat_ips = 0;
+        struct lport_addresses snat_addrs;
 
-        struct v46_ip snat_ip;
-        const char *dnat_force_snat_ip = get_force_snat_ip(op->od, "dnat",
-                                                           &snat_ip);
-        if (dnat_force_snat_ip) {
-            snat_ips[n_snat_ips++] = snat_ip;
+        if (get_force_snat_ip(op->od, "dnat", &snat_addrs)) {
+            if (snat_addrs.n_ipv4_addrs) {
+                snat_ips[n_snat_ips].family = AF_INET;
+                snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr;
+            }
+            if (snat_addrs.n_ipv6_addrs) {
+                snat_ips[n_snat_ips].family = AF_INET6;
+                snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr;
+            }
+            destroy_lport_addresses(&snat_addrs);
         }
 
-        const char *lb_force_snat_ip = get_force_snat_ip(op->od, "lb",
-                                                         &snat_ip);
-        if (lb_force_snat_ip) {
-            snat_ips[n_snat_ips++] = snat_ip;
+        memset(&snat_addrs, 0, sizeof(snat_addrs));
+        if (get_force_snat_ip(op->od, "lb", &snat_addrs)) {
+            if (snat_addrs.n_ipv4_addrs) {
+                snat_ips[n_snat_ips].family = AF_INET;
+                snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr;
+            }
+            if (snat_addrs.n_ipv6_addrs) {
+                snat_ips[n_snat_ips].family = AF_INET6;
+                snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr;
+            }
+            destroy_lport_addresses(&snat_addrs);
         }
 
         for (size_t i = 0; i < op->od->nbr->n_nat; i++) {
@@ -8985,11 +9014,12 @@  build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
 
         struct sset nat_entries = SSET_INITIALIZER(&nat_entries);
 
-        struct v46_ip snat_ip, lb_snat_ip;
-        const char *dnat_force_snat_ip = get_force_snat_ip(od, "dnat",
-                                                           &snat_ip);
-        const char *lb_force_snat_ip = get_force_snat_ip(od, "lb",
-                                                         &lb_snat_ip);
+        struct lport_addresses dnat_force_snat_addrs;
+        struct lport_addresses lb_force_snat_addrs;
+        bool dnat_force_snat_ip = get_force_snat_ip(od, "dnat",
+                                                    &dnat_force_snat_addrs);
+        bool lb_force_snat_ip = get_force_snat_ip(od, "lb",
+                                                  &lb_force_snat_addrs);
 
         for (int i = 0; i < od->nbr->n_nat; i++) {
             const struct nbrec_nat *nat;
@@ -9455,49 +9485,30 @@  build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
         }
 
         /* Handle force SNAT options set in the gateway router. */
-        if (dnat_force_snat_ip && !od->l3dgw_port) {
-            /* If a packet with destination IP address as that of the
-             * gateway router (as set in options:dnat_force_snat_ip) is seen,
-             * UNSNAT it. */
-            ds_clear(&match);
-            ds_put_format(&match, "ip && ip%s.dst == %s",
-                          snat_ip.family == AF_INET ? "4" : "6",
-                          dnat_force_snat_ip);
-            ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110,
-                          ds_cstr(&match), "ct_snat;");
-
-            /* Higher priority rules to force SNAT with the IP addresses
-             * configured in the Gateway router.  This only takes effect
-             * when the packet has already been DNATed once. */
-            ds_clear(&match);
-            ds_put_format(&match, "flags.force_snat_for_dnat == 1 && ip");
-            ds_clear(&actions);
-            ds_put_format(&actions, "ct_snat(%s);", dnat_force_snat_ip);
-            ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100,
-                          ds_cstr(&match), ds_cstr(&actions));
-        }
-        if (lb_force_snat_ip && !od->l3dgw_port) {
-            /* If a packet with destination IP address as that of the
-             * gateway router (as set in options:lb_force_snat_ip) is seen,
-             * UNSNAT it. */
-            ds_clear(&match);
-            ds_put_format(&match, "ip && ip%s.dst == %s",
-                          lb_snat_ip.family == AF_INET ? "4" : "6",
-                          lb_force_snat_ip);
-            ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 100,
-                          ds_cstr(&match), "ct_snat;");
-
-            /* Load balanced traffic will have flags.force_snat_for_lb set.
-             * Force SNAT it. */
-            ds_clear(&match);
-            ds_put_format(&match, "flags.force_snat_for_lb == 1 && ip");
-            ds_clear(&actions);
-            ds_put_format(&actions, "ct_snat(%s);", lb_force_snat_ip);
-            ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100,
-                          ds_cstr(&match), ds_cstr(&actions));
-        }
-
         if (!od->l3dgw_port) {
+            if (dnat_force_snat_ip) {
+                if (dnat_force_snat_addrs.n_ipv4_addrs) {
+                    build_lrouter_force_snat_flows(lflows, od, "4",
+                        dnat_force_snat_addrs.ipv4_addrs[0].addr_s, "dnat");
+                }
+                if (dnat_force_snat_addrs.n_ipv6_addrs) {
+                    build_lrouter_force_snat_flows(lflows, od, "6",
+                        dnat_force_snat_addrs.ipv6_addrs[0].addr_s, "dnat");
+                }
+                destroy_lport_addresses(&dnat_force_snat_addrs);
+            }
+            if (lb_force_snat_ip) {
+                if (lb_force_snat_addrs.n_ipv4_addrs) {
+                    build_lrouter_force_snat_flows(lflows, od, "4",
+                        lb_force_snat_addrs.ipv4_addrs[0].addr_s, "lb");
+                }
+                if (lb_force_snat_addrs.n_ipv6_addrs) {
+                    build_lrouter_force_snat_flows(lflows, od, "6",
+                        lb_force_snat_addrs.ipv6_addrs[0].addr_s, "lb");
+                }
+                destroy_lport_addresses(&lb_force_snat_addrs);
+            }
+
             /* For gateway router, re-circulate every packet through
             * the DNAT zone.  This helps with the following.
             *
diff --git a/ovn-nb.xml b/ovn-nb.xml
index db5908cd5..9f3da3563 100644
--- a/ovn-nb.xml
+++ b/ovn-nb.xml
@@ -1817,27 +1817,29 @@ 
       </column>
       <column name="options" key="dnat_force_snat_ip">
         <p>
-          If set, indicates the IP address to use to force SNAT a packet
-          that has already been DNATed in the gateway router.  When multiple
-          gateway routers are configured, a packet can potentially enter any
-          of the gateway router, get DNATted and eventually reach the logical
-          switch port.  For the return traffic to go back to the same gateway
-          router (for unDNATing), the packet needs a SNAT in the first place.
-          This can be achieved by setting the above option with a gateway
-          specific IP address.
+          If set, indicates a set of IP addresses to use to force SNAT a
+          packet that has already been DNATed in the gateway router.  When
+          multiple gateway routers are configured, a packet can potentially
+          enter any of the gateway router, get DNATted and eventually reach the
+          logical switch port.  For the return traffic to go back to the same
+          gateway router (for unDNATing), the packet needs a SNAT in the first
+          place. This can be achieved by setting the above option with a
+          gateway specific set of IP addresses. This option may have exactly
+          one IPv4 and/or one IPv6 address on it, separated by a a space.
         </p>
       </column>
       <column name="options" key="lb_force_snat_ip">
         <p>
-          If set, indicates the IP address to use to force SNAT a packet
+          If set, indicates a set of IP addresses to use to force SNAT a packet
           that has already been load-balanced in the gateway router.  When
           multiple gateway routers are configured, a packet can potentially
           enter any of the gateway routers, get DNATted as part of the load-
           balancing and eventually reach the logical switch port.
           For the return traffic to go back to the same gateway router (for
           unDNATing), the packet needs a SNAT in the first place.  This can be
-          achieved by setting the above option with a gateway specific IP
-          address.
+          achieved by setting the above option with a gateway specific set of
+          IP addresses. This option may have exactly one IPv4 and/or one IPv6
+          address on it, separated by a space character.
         </p>
       </column>
       <column name="options" key="mcast_relay" type='{"type": "boolean"}'>
diff --git a/tests/system-ovn.at b/tests/system-ovn.at
index 2999e52fd..079d61975 100644
--- a/tests/system-ovn.at
+++ b/tests/system-ovn.at
@@ -1026,6 +1026,323 @@  OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
 /connection dropped.*/d"])
 AT_CLEANUP
 
+AT_SETUP([ovn -- multiple gateway routers, SNAT and DNAT - Dual Stack])
+AT_KEYWORDS([ovnnat])
+
+CHECK_CONNTRACK()
+CHECK_CONNTRACK_NAT()
+ovn_start
+OVS_TRAFFIC_VSWITCHD_START()
+ADD_BR([br-int])
+
+# Set external-ids in br-int needed for ovn-controller
+ovs-vsctl \
+        -- set Open_vSwitch . external-ids:system-id=hv1 \
+        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
+        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
+        -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \
+        -- set bridge br-int fail-mode=secure other-config:disable-in-band=true
+
+# Start ovn-controller
+start_daemon ovn-controller
+
+# Logical network:
+# Three LRs - R1, R2 and R3 that are connected to each other via LS "join"
+# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24
+# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2
+# has alice (172.16.1.0/24 and fd30::/64) connected to it.  R3 has bob
+# (172.16.1.0/24 andfd30::/64) connected to it. Note how both alice and bob
+# have the same subnets behind them.  We are trying to simulate external network
+# via those 2 switches. In real world the switch ports of these switches will
+# have addresses set as "unknown" to make them learning switches. Or those
+# switches will be "localnet" ones.
+#
+#    foo -- R1 -- join - R2 -- alice
+#           |          |
+#    bar ----          - R3 --- bob
+
+ovn-nbctl create Logical_Router name=R1
+ovn-nbctl create Logical_Router name=R2 options:chassis=hv1
+ovn-nbctl create Logical_Router name=R3 options:chassis=hv1
+
+ovn-nbctl ls-add foo
+ovn-nbctl ls-add bar
+ovn-nbctl ls-add alice
+ovn-nbctl ls-add bob
+ovn-nbctl ls-add join
+
+# Connect foo to R1
+ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64
+ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \
+    type=router options:router-port=foo addresses=\"00:00:01:01:02:03\"
+
+# Connect bar to R1
+ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64
+ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \
+    type=router options:router-port=bar addresses=\"00:00:01:01:02:04\"
+
+# Connect alice to R2
+ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd30::1/64
+ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \
+    type=router options:router-port=alice addresses=\"00:00:02:01:02:03\"
+
+# Connect bob to R3
+ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd30::2/64
+ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \
+    type=router options:router-port=bob addresses=\"00:00:03:01:02:03\"
+
+# Connect R1 to join
+ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64
+ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \
+    type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"'
+
+# Connect R2 to join
+ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64
+ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \
+    type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"'
+
+# Connect R3 to join
+ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64
+ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \
+    type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"'
+
+# Install static routes with source ip address as the policy for routing.
+# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3.
+ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2
+ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3
+ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2
+ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3
+
+# Static routes.
+ovn-nbctl lr-route-add R2 fd11::/64 fd20::1
+ovn-nbctl lr-route-add R2 fd12::/64 fd20::1
+ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1
+ovn-nbctl lr-route-add R3 fd11::/64 fd20::1
+ovn-nbctl lr-route-add R3 fd12::/64 fd20::1
+ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1
+
+# For gateway routers R2 and R3, set a force SNAT rule.
+ovn-nbctl set logical_router R2 options:dnat_force_snat_ip="20.0.0.2 fd20::2"
+ovn-nbctl set logical_router R3 options:dnat_force_snat_ip="20.0.0.3 fd20::3"
+
+# Logical port 'foo1' in switch 'foo'.
+ADD_NAMESPACES(foo1)
+ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \
+         "192.168.1.1")
+ovn-nbctl lsp-add foo foo1 \
+-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2"
+
+ADD_NAMESPACES(foo16)
+ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:00:02:02:03", \
+         "fd11::1")
+OVS_WAIT_UNTIL([test "$(ip netns exec foo16 ip a | grep fd11::2 | grep tentative)" = ""])
+ovn-nbctl lsp-add foo foo16 \
+-- lsp-set-addresses foo16 "f0:00:00:02:02:03 fd11::2"
+
+# Logical port 'alice1' in switch 'alice'.
+ADD_NAMESPACES(alice1)
+ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \
+         "172.16.1.1")
+ovn-nbctl lsp-add alice alice1 \
+-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3"
+
+ADD_NAMESPACES(alice16)
+ADD_VETH(alice16, alice16, br-int, "fd30::3/64", "f0:00:00:02:02:04", \
+         "fd30::1")
+OVS_WAIT_UNTIL([test "$(ip netns exec alice16 ip a | grep fd30::3 | grep tentative)" = ""])
+ovn-nbctl lsp-add alice alice16 \
+-- lsp-set-addresses alice16 "f0:00:00:02:02:04 fd30::3"
+
+# Logical port 'bar1' in switch 'bar'.
+ADD_NAMESPACES(bar1)
+ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \
+"192.168.2.1")
+ovn-nbctl lsp-add bar bar1 \
+-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2"
+
+ADD_NAMESPACES(bar16)
+ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:00:02:02:05", \
+         "fd12::1")
+OVS_WAIT_UNTIL([test "$(ip netns exec bar16 ip a | grep fd12::2 | grep tentative)" = ""])
+ovn-nbctl lsp-add bar bar16 \
+-- lsp-set-addresses bar16 "f0:00:00:02:02:05 fd12::2"
+
+# Logical port 'bob1' in switch 'bob'.
+ADD_NAMESPACES(bob1)
+ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \
+         "172.16.1.2")
+ovn-nbctl lsp-add bob bob1 \
+-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4"
+
+ADD_NAMESPACES(bob16)
+ADD_VETH(bob16, bob16, br-int, "fd30::4/64", "f0:00:00:02:02:06", \
+         "fd30::2")
+OVS_WAIT_UNTIL([test "$(ip netns exec bob16 ip a | grep fd30::4 | grep tentative)" = ""])
+ovn-nbctl lsp-add bob bob16 \
+-- lsp-set-addresses bob16 "f0:00:00:02:02:06 fd30::4"
+
+# Router R2
+# Add a DNAT rule.
+ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \
+    external_ip=30.0.0.2 -- add logical_router R2 nat @nat
+ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \
+    external_ip='"fd40::2"' -- add logical_router R2 nat @nat
+
+# Add a SNAT rule
+ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.1.2 \
+    external_ip=30.0.0.1 -- add logical_router R2 nat @nat
+ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd11::2"' \
+    external_ip='"fd40::1"' -- add logical_router R2 nat @nat
+
+# Router R3
+# Add a DNAT rule.
+ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \
+    external_ip=30.0.0.3 -- add logical_router R3 nat @nat
+ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \
+    external_ip='"fd40::3"' -- add logical_router R3 nat @nat
+
+# Add a SNAT rule
+ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.2.2 \
+    external_ip=30.0.0.4 -- add logical_router R3 nat @nat
+ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd12::2"' \
+    external_ip='"fd40::4"' -- add logical_router R3 nat @nat
+
+# wait for ovn-controller to catch up.
+ovn-nbctl --wait=hv sync
+OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=fd40::4)'])
+OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=30.0.0.4)'])
+
+# North-South DNAT: 'alice1' should be able to ping 'foo1' via 30.0.0.2
+NS_CHECK_EXEC([alice1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.2 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# North-South DNAT: 'alice16' should be able to ping 'foo16' via fd30::2
+NS_CHECK_EXEC([alice16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::2 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# Check conntrack entries.
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.3) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmp,orig=(src=172.16.1.3,dst=30.0.0.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=0,code=0),zone=<cleared>
+])
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::3) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmpv6,orig=(src=fd30::3,dst=fd40::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::3,id=<cleared>,type=129,code=0),zone=<cleared>
+])
+
+# But foo1 should receive traffic from 20.0.0.2
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.2) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmp,orig=(src=172.16.1.3,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.2,id=<cleared>,type=0,code=0),zone=<cleared>
+])
+# But foo16 should receive traffic from fd20::2
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmpv6,orig=(src=fd30::3,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::2,id=<cleared>,type=129,code=0),zone=<cleared>
+])
+
+# North-South DNAT: 'bob1' should be able to ping 'foo1' via 30.0.0.3
+NS_CHECK_EXEC([bob1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.3 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# North-South DNAT: 'bob16' should be able to ping 'foo16' via fd40::3
+NS_CHECK_EXEC([bob16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::3 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# Check conntrack entries.
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.4) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmp,orig=(src=172.16.1.4,dst=30.0.0.3,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.4,id=<cleared>,type=0,code=0),zone=<cleared>
+])
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::4) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmpv6,orig=(src=fd30::4,dst=fd40::3,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::4,id=<cleared>,type=129,code=0),zone=<cleared>
+])
+
+# But foo1 should receive traffic from 20.0.0.3
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.3) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmp,orig=(src=172.16.1.4,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.3,id=<cleared>,type=0,code=0),zone=<cleared>
+])
+
+# But foo16 should receive traffic from fd20::3
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::3) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmpv6,orig=(src=fd30::4,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::3,id=<cleared>,type=129,code=0),zone=<cleared>
+])
+
+# South-North SNAT: 'bar1' pings 'bob1'. But 'bob1' receives traffic
+# from 30.0.0.4
+NS_CHECK_EXEC([bar1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.4 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+# South-North SNAT: 'bar16' pings 'bob16'. But 'bob16' receives traffic
+# from fd40::4
+NS_CHECK_EXEC([bar16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::4 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# We verify that SNAT indeed happened via 'dump-conntrack' command.
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.4) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmp,orig=(src=192.168.2.2,dst=172.16.1.4,id=<cleared>,type=8,code=0),reply=(src=172.16.1.4,dst=30.0.0.4,id=<cleared>,type=0,code=0),zone=<cleared>
+])
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::4) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmpv6,orig=(src=fd12::2,dst=fd30::4,id=<cleared>,type=128,code=0),reply=(src=fd30::4,dst=fd40::4,id=<cleared>,type=129,code=0),zone=<cleared>
+])
+
+# South-North SNAT: 'foo1' pings 'alice1'. But 'alice1' receives traffic
+# from 30.0.0.1
+NS_CHECK_EXEC([foo1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.3 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# South-North SNAT: 'foo16' pings 'alice16'. But 'alice16' receives traffic
+# from fd40::1
+NS_CHECK_EXEC([foo16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::3 | FORMAT_PING], \
+[0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+# We verify that SNAT indeed happened via 'dump-conntrack' command.
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmp,orig=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=8,code=0),reply=(src=172.16.1.3,dst=30.0.0.1,id=<cleared>,type=0,code=0),zone=<cleared>
+])
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::1) | \
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+icmpv6,orig=(src=fd11::2,dst=fd30::3,id=<cleared>,type=128,code=0),reply=(src=fd30::3,dst=fd40::1,id=<cleared>,type=129,code=0),zone=<cleared>
+])
+
+OVS_APP_EXIT_AND_WAIT([ovn-controller])
+
+as ovn-sb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as northd
+OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+as
+OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
+/connection dropped.*/d"])
+AT_CLEANUP
+
+
 AT_SETUP([ovn -- load-balancing])
 AT_KEYWORDS([ovnlb])
 
@@ -2405,6 +2722,230 @@  OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
 /connection dropped.*/d"])
 AT_CLEANUP
 
+AT_SETUP([ovn -- multiple gateway routers, load-balancing - Dual Stack])
+AT_KEYWORDS([ovnlb])
+
+CHECK_CONNTRACK()
+CHECK_CONNTRACK_NAT()
+ovn_start
+OVS_TRAFFIC_VSWITCHD_START()
+ADD_BR([br-int])
+
+# Set external-ids in br-int needed for ovn-controller
+ovs-vsctl \
+        -- set Open_vSwitch . external-ids:system-id=hv1 \
+        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
+        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
+        -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \
+        -- set bridge br-int fail-mode=secure other-config:disable-in-band=true
+
+# Start ovn-controller
+start_daemon ovn-controller
+
+# Logical network:
+# Three LRs - R1, R2 and R3 that are connected to each other via LS "join"
+# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24
+# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2
+# has alice (172.16.1.0/24 and fd72::/64) connected to it.  R3 has bob
+# (172.16.1.0/24 and fd72::/64) connected to it. Note how both alice and
+# bob have the same subnets behind them.  We are trying to simulate external
+# network via those 2 switches. In real world the switch ports of these
+# switches will have addresses set as "unknown" to make them learning switches.
+# Or those switches will be "localnet" ones.
+#
+#    foo -- R1 -- join - R2 -- alice
+#           |          |
+#    bar ----          - R3 --- bob
+
+ovn-nbctl create Logical_Router name=R1
+ovn-nbctl create Logical_Router name=R2 options:chassis=hv1
+ovn-nbctl create Logical_Router name=R3 options:chassis=hv1
+
+ovn-nbctl ls-add foo
+ovn-nbctl ls-add bar
+ovn-nbctl ls-add alice
+ovn-nbctl ls-add bob
+ovn-nbctl ls-add join
+
+# Connect foo to R1
+ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64
+ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \
+    type=router options:router-port=foo addresses=\"00:00:01:01:02:03\"
+
+# Connect bar to R1
+ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64
+ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \
+    type=router options:router-port=bar addresses=\"00:00:01:01:02:04\"
+
+# Connect alice to R2
+ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd72::1/64
+ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \
+    type=router options:router-port=alice addresses=\"00:00:02:01:02:03\"
+
+# Connect bob to R3
+ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd72::2/64
+ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \
+    type=router options:router-port=bob addresses=\"00:00:03:01:02:03\"
+
+# Connect R1 to join
+ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64
+ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \
+    type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"'
+
+# Connect R2 to join
+ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64
+ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \
+    type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"'
+
+# Connect R3 to join
+ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64
+ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \
+    type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"'
+
+# Install static routes with source ip address as the policy for routing.
+# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3.
+ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2
+ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3
+ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2
+ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3
+
+# Static routes.
+ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1
+ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1
+ovn-nbctl lr-route-add R2 fd11::/64 fd20::1
+ovn-nbctl lr-route-add R2 fd12::/64 fd20::1
+ovn-nbctl lr-route-add R3 fd11::/64 fd20::1
+ovn-nbctl lr-route-add R3 fd12::/64 fd20::1
+
+# For gateway routers R2 and R3, set a force SNAT rule.
+ovn-nbctl set logical_router R2 options:lb_force_snat_ip="20.0.0.2 fd20::2"
+ovn-nbctl set logical_router R3 options:lb_force_snat_ip="20.0.0.3 fd20::3"
+
+# Logical port 'foo1' in switch 'foo'.
+ADD_NAMESPACES(foo1)
+ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \
+         "192.168.1.1")
+ovn-nbctl lsp-add foo foo1 \
+-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2"
+
+# Logical port 'foo16' in switch 'foo'.
+ADD_NAMESPACES(foo16)
+ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:06:01:02:03", \
+         "fd11::1")
+ovn-nbctl lsp-add foo foo16 \
+-- lsp-set-addresses foo16 "f0:00:06:01:02:03 fd11::2"
+
+# Logical port 'alice1' in switch 'alice'.
+ADD_NAMESPACES(alice1)
+ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \
+         "172.16.1.1")
+ovn-nbctl lsp-add alice alice1 \
+-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3"
+
+# Logical port 'alice16' in switch 'alice'.
+ADD_NAMESPACES(alice16)
+ADD_VETH(alice16, alice16, br-int, "fd72::3/64", "f0:00:06:01:02:04", \
+         "fd72::1")
+ovn-nbctl lsp-add alice alice16 \
+-- lsp-set-addresses alice16 "f0:00:06:01:02:04 fd72::3"
+
+# Logical port 'bar1' in switch 'bar'.
+ADD_NAMESPACES(bar1)
+ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \
+"192.168.2.1")
+ovn-nbctl lsp-add bar bar1 \
+-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2"
+
+# Logical port 'bar16' in switch 'bar'.
+ADD_NAMESPACES(bar16)
+ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:06:01:02:05", \
+"fd12::1")
+ovn-nbctl lsp-add bar bar16 \
+-- lsp-set-addresses bar16 "f0:00:06:01:02:05 fd12::2"
+
+# Logical port 'bob1' in switch 'bob'.
+ADD_NAMESPACES(bob1)
+ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \
+         "172.16.1.2")
+ovn-nbctl lsp-add bob bob1 \
+-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4"
+
+# Logical port 'bob16' in switch 'bob'.
+ADD_NAMESPACES(bob16)
+ADD_VETH(bob16, bob16, br-int, "fd72::4/64", "f0:00:06:01:02:06", \
+         "fd72::2")
+ovn-nbctl lsp-add bob bob16 \
+-- lsp-set-addresses bob16 "f0:00:06:01:02:06 fd72::4"
+
+# Config OVN load-balancer with a VIP.
+uuid=`ovn-nbctl  create load_balancer vips:30.0.0.1="192.168.1.2,192.168.2.2" \
+vips:\"fd30::1\"=\"fd11::2,fd12::2\"`
+ovn-nbctl set logical_router R2 load_balancer=$uuid
+ovn-nbctl set logical_router R3 load_balancer=$uuid
+
+# Wait for ovn-controller to catch up.
+ovn-nbctl --wait=hv sync
+OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \
+grep 'nat(dst=192.168.2.2)'])
+OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \
+grep 'nat(dst=fd12::2)'])
+
+# Start webservers in 'foo1', 'foo16, 'bar1', and 'bar16'.
+OVS_START_L7([foo1], [http])
+OVS_START_L7([bar1], [http])
+OVS_START_L7([foo16], [http6])
+OVS_START_L7([bar16], [http6])
+
+dnl Should work with the virtual IP address through NAT
+for i in `seq 1 20`; do
+    echo Request $i
+    NS_CHECK_EXEC([alice1], [wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log])
+done
+
+for i in `seq 1 20`; do
+    echo Request ${i}_6
+    NS_CHECK_EXEC([alice16], [wget http://[[fd30::1]] -t 5 -T 1 --retry-connrefused -v -o wget${i}_6.log])
+done
+
+dnl Each server should have at least one connection.
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) |
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+])
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::1) | grep -v fe80 |
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+])
+
+dnl Force SNAT should have worked.
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0) |
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+tcp,orig=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+tcp,orig=(src=172.16.1.3,dst=192.168.2.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+])
+AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | grep -v fe80 |
+sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl
+tcp,orig=(src=fd72::3,dst=fd11::2,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+tcp,orig=(src=fd72::3,dst=fd12::2,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>)
+])
+OVS_APP_EXIT_AND_WAIT([ovn-controller])
+
+as ovn-sb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+as northd
+OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+as
+OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d
+/connection dropped.*/d"])
+AT_CLEANUP
+
 AT_SETUP([ovn -- load balancing in router with gateway router port])
 AT_KEYWORDS([ovnlb])