Message ID | 20200716180606.2820933-1-mmichels@redhat.com |
---|---|
State | Superseded |
Headers | show |
Series | [ovs-dev,ovn] Allow force_snat options to work for dual-stack routers. | expand |
On Thu, Jul 16, 2020 at 11:37 PM Mark Michelson <mmichels@redhat.com> wrote: > > The lb_force_snat and dnat_force_snat options could accept only a single > IP address. For routers that only route traffic of a single IP address > family, this is fine. However, if a router routes both IPv4 and IPv6 > traffic, then this limitation is a problem. > > This patch addresses this problem by allowing for these options to > specify both an IPv4 and IPv6 address. > > Signed-off-by: Mark Michelson <mmichels@redhat.com> Hi Mark, A couple of comments below. Thanks Numan > --- > northd/ovn-northd.c | 179 ++++++++------- > ovn-nb.xml | 24 +- > tests/system-ovn.at | 541 ++++++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 649 insertions(+), 95 deletions(-) > > diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c > index 192198272..2c05d1c2a 100644 > --- a/northd/ovn-northd.c > +++ b/northd/ovn-northd.c > @@ -7801,44 +7801,37 @@ op_put_v6_networks(struct ds *ds, const struct ovn_port *op) > ds_put_cstr(ds, "}"); > } > > -static const char * > +static bool > get_force_snat_ip(struct ovn_datapath *od, const char *key_type, > - struct v46_ip *ip) > + struct lport_addresses *laddrs) > { > char *key = xasprintf("%s_force_snat_ip", key_type); > - const char *ip_address = smap_get(&od->nbr->options, key); > + const char *addresses = smap_get(&od->nbr->options, key); > free(key); > > - if (ip_address) { > - ovs_be32 mask; > - ip->family = AF_INET; > - char *error = ip_parse_masked(ip_address, &ip->ipv4, &mask); > - if (error || mask != OVS_BE32_MAX) { > - free(error); > - struct in6_addr mask_v6, v6_exact = IN6ADDR_EXACT_INIT; > - ip->family = AF_INET6; > - error = ipv6_parse_masked(ip_address, &ip->ipv6, &mask_v6); > - if (error || memcmp(&mask_v6, &v6_exact, sizeof(mask_v6))) { > - static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); > - VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"", > - ip_address, UUID_ARGS(&od->key)); > - memset(ip, 0, sizeof *ip); > - ip->family = AF_UNSPEC; > - return NULL; > - } > - } > - return ip_address; > + if (!addresses) { > + return false; > } > > - memset(ip, 0, sizeof *ip); > - ip->family = AF_UNSPEC; > - return NULL; > + if (!extract_ip_addresses(addresses, laddrs) || > + laddrs->n_ipv4_addrs > 1 || > + laddrs->n_ipv6_addrs > 1 || > + (laddrs->n_ipv4_addrs && laddrs->ipv4_addrs[0].plen != 32) || > + (laddrs->n_ipv6_addrs && laddrs->ipv6_addrs[0].plen != 128)) { > + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); > + VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"", > + addresses, UUID_ARGS(&od->key)); > + destroy_lport_addresses(laddrs); > + return false; > + } > + > + return true; > } > > static void > add_router_lb_flow(struct hmap *lflows, struct ovn_datapath *od, > struct ds *match, struct ds *actions, int priority, > - const char *lb_force_snat_ip, struct lb_vip *lb_vip, > + bool lb_force_snat_ip, struct lb_vip *lb_vip, > const char *proto, struct nbrec_load_balancer *lb, > struct shash *meter_groups, struct sset *nat_entries) > { > @@ -8159,6 +8152,29 @@ build_lrouter_nd_flow(struct ovn_datapath *od, struct ovn_port *op, > ds_destroy(&actions); > } > > +static void > +build_lrouter_force_snat_flows(struct hmap *lflows, struct ovn_datapath *od, > + const char *ip_version, const char *ip_addr, > + const char *context) > +{ > + struct ds match = DS_EMPTY_INITIALIZER; > + struct ds actions = DS_EMPTY_INITIALIZER; > + ds_put_format(&match, "ip%s && ip%s.dst == %s", > + ip_version, ip_version, ip_addr); > + ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110, > + ds_cstr(&match), "ct_snat;"); > + > + /* Higher priority rules to force SNAT with the IP addresses > + * configured in the Gateway router. This only takes effect > + * when the packet has already been DNATed or load balanced once. */ > + ds_clear(&match); > + ds_put_format(&match, "flags.force_snat_for_%s == 1 && ip%s", > + context, ip_version); > + ds_put_format(&actions, "ct_snat(%s);", ip_addr); > + ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100, > + ds_cstr(&match), ds_cstr(&actions)); There is a memory leak here. I think you need to call ds_destroy for both 'match' and 'actions'. > +} > + > static void > build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, > struct hmap *lflows, struct shash *meter_groups, > @@ -8609,24 +8625,37 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, > } > } > > - /* A gateway router can have 2 SNAT IP addresses to force DNATed and > + /* A gateway router can have 4 SNAT IP addresses to force DNATed and > * LBed traffic respectively to be SNATed. In addition, there can be > * a number of SNAT rules in the NAT table. */ > struct v46_ip *snat_ips = xmalloc(sizeof *snat_ips > - * (op->od->nbr->n_nat + 2)); > + * (op->od->nbr->n_nat + 4)); > size_t n_snat_ips = 0; > + struct lport_addresses snat_addrs; > > - struct v46_ip snat_ip; > - const char *dnat_force_snat_ip = get_force_snat_ip(op->od, "dnat", > - &snat_ip); > - if (dnat_force_snat_ip) { > - snat_ips[n_snat_ips++] = snat_ip; > + if (get_force_snat_ip(op->od, "dnat", &snat_addrs)) { > + if (snat_addrs.n_ipv4_addrs) { > + snat_ips[n_snat_ips].family = AF_INET; > + snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr; > + } > + if (snat_addrs.n_ipv6_addrs) { > + snat_ips[n_snat_ips].family = AF_INET6; > + snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr; > + } > + destroy_lport_addresses(&snat_addrs); > } > > - const char *lb_force_snat_ip = get_force_snat_ip(op->od, "lb", > - &snat_ip); > - if (lb_force_snat_ip) { > - snat_ips[n_snat_ips++] = snat_ip; > + memset(&snat_addrs, 0, sizeof(snat_addrs)); > + if (get_force_snat_ip(op->od, "lb", &snat_addrs)) { > + if (snat_addrs.n_ipv4_addrs) { > + snat_ips[n_snat_ips].family = AF_INET; > + snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr; > + } > + if (snat_addrs.n_ipv6_addrs) { > + snat_ips[n_snat_ips].family = AF_INET6; > + snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr; > + } > + destroy_lport_addresses(&snat_addrs); > } > > for (size_t i = 0; i < op->od->nbr->n_nat; i++) { > @@ -8985,11 +9014,12 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, > > struct sset nat_entries = SSET_INITIALIZER(&nat_entries); > > - struct v46_ip snat_ip, lb_snat_ip; > - const char *dnat_force_snat_ip = get_force_snat_ip(od, "dnat", > - &snat_ip); > - const char *lb_force_snat_ip = get_force_snat_ip(od, "lb", > - &lb_snat_ip); > + struct lport_addresses dnat_force_snat_addrs; > + struct lport_addresses lb_force_snat_addrs; > + bool dnat_force_snat_ip = get_force_snat_ip(od, "dnat", > + &dnat_force_snat_addrs); > + bool lb_force_snat_ip = get_force_snat_ip(od, "lb", > + &lb_force_snat_addrs); 'dnat_force_snat_addrs and 'lb_force_snat_addrs' are destroyed below if the logical router doesn't have a gateway router port. What if there is a logical router with these options set and also has a gw router port. Even though its misconfiguration from the user, but ovn-northd will leak the memory. I think this should be handled properly. > > for (int i = 0; i < od->nbr->n_nat; i++) { > const struct nbrec_nat *nat; > @@ -9455,49 +9485,30 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, > } > > /* Handle force SNAT options set in the gateway router. */ > - if (dnat_force_snat_ip && !od->l3dgw_port) { > - /* If a packet with destination IP address as that of the > - * gateway router (as set in options:dnat_force_snat_ip) is seen, > - * UNSNAT it. */ > - ds_clear(&match); > - ds_put_format(&match, "ip && ip%s.dst == %s", > - snat_ip.family == AF_INET ? "4" : "6", > - dnat_force_snat_ip); > - ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110, > - ds_cstr(&match), "ct_snat;"); > - > - /* Higher priority rules to force SNAT with the IP addresses > - * configured in the Gateway router. This only takes effect > - * when the packet has already been DNATed once. */ > - ds_clear(&match); > - ds_put_format(&match, "flags.force_snat_for_dnat == 1 && ip"); > - ds_clear(&actions); > - ds_put_format(&actions, "ct_snat(%s);", dnat_force_snat_ip); > - ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100, > - ds_cstr(&match), ds_cstr(&actions)); > - } > - if (lb_force_snat_ip && !od->l3dgw_port) { > - /* If a packet with destination IP address as that of the > - * gateway router (as set in options:lb_force_snat_ip) is seen, > - * UNSNAT it. */ > - ds_clear(&match); > - ds_put_format(&match, "ip && ip%s.dst == %s", > - lb_snat_ip.family == AF_INET ? "4" : "6", > - lb_force_snat_ip); > - ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 100, > - ds_cstr(&match), "ct_snat;"); > - > - /* Load balanced traffic will have flags.force_snat_for_lb set. > - * Force SNAT it. */ > - ds_clear(&match); > - ds_put_format(&match, "flags.force_snat_for_lb == 1 && ip"); > - ds_clear(&actions); > - ds_put_format(&actions, "ct_snat(%s);", lb_force_snat_ip); > - ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100, > - ds_cstr(&match), ds_cstr(&actions)); > - } > - > if (!od->l3dgw_port) { > + if (dnat_force_snat_ip) { > + if (dnat_force_snat_addrs.n_ipv4_addrs) { > + build_lrouter_force_snat_flows(lflows, od, "4", > + dnat_force_snat_addrs.ipv4_addrs[0].addr_s, "dnat"); > + } > + if (dnat_force_snat_addrs.n_ipv6_addrs) { > + build_lrouter_force_snat_flows(lflows, od, "6", > + dnat_force_snat_addrs.ipv6_addrs[0].addr_s, "dnat"); > + } > + destroy_lport_addresses(&dnat_force_snat_addrs); > + } > + if (lb_force_snat_ip) { > + if (lb_force_snat_addrs.n_ipv4_addrs) { > + build_lrouter_force_snat_flows(lflows, od, "4", > + lb_force_snat_addrs.ipv4_addrs[0].addr_s, "lb"); > + } > + if (lb_force_snat_addrs.n_ipv6_addrs) { > + build_lrouter_force_snat_flows(lflows, od, "6", > + lb_force_snat_addrs.ipv6_addrs[0].addr_s, "lb"); > + } > + destroy_lport_addresses(&lb_force_snat_addrs); > + } > + > /* For gateway router, re-circulate every packet through > * the DNAT zone. This helps with the following. > * > diff --git a/ovn-nb.xml b/ovn-nb.xml > index db5908cd5..9f3da3563 100644 > --- a/ovn-nb.xml > +++ b/ovn-nb.xml > @@ -1817,27 +1817,29 @@ > </column> > <column name="options" key="dnat_force_snat_ip"> > <p> > - If set, indicates the IP address to use to force SNAT a packet > - that has already been DNATed in the gateway router. When multiple > - gateway routers are configured, a packet can potentially enter any > - of the gateway router, get DNATted and eventually reach the logical > - switch port. For the return traffic to go back to the same gateway > - router (for unDNATing), the packet needs a SNAT in the first place. > - This can be achieved by setting the above option with a gateway > - specific IP address. > + If set, indicates a set of IP addresses to use to force SNAT a > + packet that has already been DNATed in the gateway router. When > + multiple gateway routers are configured, a packet can potentially > + enter any of the gateway router, get DNATted and eventually reach the > + logical switch port. For the return traffic to go back to the same > + gateway router (for unDNATing), the packet needs a SNAT in the first > + place. This can be achieved by setting the above option with a > + gateway specific set of IP addresses. This option may have exactly > + one IPv4 and/or one IPv6 address on it, separated by a a space. > </p> > </column> > <column name="options" key="lb_force_snat_ip"> > <p> > - If set, indicates the IP address to use to force SNAT a packet > + If set, indicates a set of IP addresses to use to force SNAT a packet > that has already been load-balanced in the gateway router. When > multiple gateway routers are configured, a packet can potentially > enter any of the gateway routers, get DNATted as part of the load- > balancing and eventually reach the logical switch port. > For the return traffic to go back to the same gateway router (for > unDNATing), the packet needs a SNAT in the first place. This can be > - achieved by setting the above option with a gateway specific IP > - address. > + achieved by setting the above option with a gateway specific set of > + IP addresses. This option may have exactly one IPv4 and/or one IPv6 > + address on it, separated by a space character. > </p> > </column> > <column name="options" key="mcast_relay" type='{"type": "boolean"}'> > diff --git a/tests/system-ovn.at b/tests/system-ovn.at > index 2999e52fd..079d61975 100644 > --- a/tests/system-ovn.at > +++ b/tests/system-ovn.at > @@ -1026,6 +1026,323 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d > /connection dropped.*/d"]) > AT_CLEANUP > > +AT_SETUP([ovn -- multiple gateway routers, SNAT and DNAT - Dual Stack]) > +AT_KEYWORDS([ovnnat]) > + > +CHECK_CONNTRACK() > +CHECK_CONNTRACK_NAT() > +ovn_start > +OVS_TRAFFIC_VSWITCHD_START() > +ADD_BR([br-int]) > + > +# Set external-ids in br-int needed for ovn-controller > +ovs-vsctl \ > + -- set Open_vSwitch . external-ids:system-id=hv1 \ > + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ > + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ > + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ > + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true > + > +# Start ovn-controller > +start_daemon ovn-controller > + > +# Logical network: > +# Three LRs - R1, R2 and R3 that are connected to each other via LS "join" > +# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24 > +# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2 > +# has alice (172.16.1.0/24 and fd30::/64) connected to it. R3 has bob > +# (172.16.1.0/24 andfd30::/64) connected to it. Note how both alice and bob > +# have the same subnets behind them. We are trying to simulate external network > +# via those 2 switches. In real world the switch ports of these switches will > +# have addresses set as "unknown" to make them learning switches. Or those > +# switches will be "localnet" ones. > +# > +# foo -- R1 -- join - R2 -- alice > +# | | > +# bar ---- - R3 --- bob > + > +ovn-nbctl create Logical_Router name=R1 > +ovn-nbctl create Logical_Router name=R2 options:chassis=hv1 > +ovn-nbctl create Logical_Router name=R3 options:chassis=hv1 > + > +ovn-nbctl ls-add foo > +ovn-nbctl ls-add bar > +ovn-nbctl ls-add alice > +ovn-nbctl ls-add bob > +ovn-nbctl ls-add join > + > +# Connect foo to R1 > +ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64 > +ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \ > + type=router options:router-port=foo addresses=\"00:00:01:01:02:03\" > + > +# Connect bar to R1 > +ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64 > +ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \ > + type=router options:router-port=bar addresses=\"00:00:01:01:02:04\" > + > +# Connect alice to R2 > +ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd30::1/64 > +ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \ > + type=router options:router-port=alice addresses=\"00:00:02:01:02:03\" > + > +# Connect bob to R3 > +ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd30::2/64 > +ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \ > + type=router options:router-port=bob addresses=\"00:00:03:01:02:03\" > + > +# Connect R1 to join > +ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64 > +ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \ > + type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"' > + > +# Connect R2 to join > +ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64 > +ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \ > + type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"' > + > +# Connect R3 to join > +ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64 > +ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \ > + type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"' > + > +# Install static routes with source ip address as the policy for routing. > +# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3. > +ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2 > +ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3 > +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2 > +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3 > + > +# Static routes. > +ovn-nbctl lr-route-add R2 fd11::/64 fd20::1 > +ovn-nbctl lr-route-add R2 fd12::/64 fd20::1 > +ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1 > +ovn-nbctl lr-route-add R3 fd11::/64 fd20::1 > +ovn-nbctl lr-route-add R3 fd12::/64 fd20::1 > +ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1 > + > +# For gateway routers R2 and R3, set a force SNAT rule. > +ovn-nbctl set logical_router R2 options:dnat_force_snat_ip="20.0.0.2 fd20::2" > +ovn-nbctl set logical_router R3 options:dnat_force_snat_ip="20.0.0.3 fd20::3" > + > +# Logical port 'foo1' in switch 'foo'. > +ADD_NAMESPACES(foo1) > +ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \ > + "192.168.1.1") > +ovn-nbctl lsp-add foo foo1 \ > +-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2" > + > +ADD_NAMESPACES(foo16) > +ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:00:02:02:03", \ > + "fd11::1") > +OVS_WAIT_UNTIL([test "$(ip netns exec foo16 ip a | grep fd11::2 | grep tentative)" = ""]) > +ovn-nbctl lsp-add foo foo16 \ > +-- lsp-set-addresses foo16 "f0:00:00:02:02:03 fd11::2" > + > +# Logical port 'alice1' in switch 'alice'. > +ADD_NAMESPACES(alice1) > +ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \ > + "172.16.1.1") > +ovn-nbctl lsp-add alice alice1 \ > +-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3" > + > +ADD_NAMESPACES(alice16) > +ADD_VETH(alice16, alice16, br-int, "fd30::3/64", "f0:00:00:02:02:04", \ > + "fd30::1") > +OVS_WAIT_UNTIL([test "$(ip netns exec alice16 ip a | grep fd30::3 | grep tentative)" = ""]) > +ovn-nbctl lsp-add alice alice16 \ > +-- lsp-set-addresses alice16 "f0:00:00:02:02:04 fd30::3" > + > +# Logical port 'bar1' in switch 'bar'. > +ADD_NAMESPACES(bar1) > +ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \ > +"192.168.2.1") > +ovn-nbctl lsp-add bar bar1 \ > +-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2" > + > +ADD_NAMESPACES(bar16) > +ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:00:02:02:05", \ > + "fd12::1") > +OVS_WAIT_UNTIL([test "$(ip netns exec bar16 ip a | grep fd12::2 | grep tentative)" = ""]) > +ovn-nbctl lsp-add bar bar16 \ > +-- lsp-set-addresses bar16 "f0:00:00:02:02:05 fd12::2" > + > +# Logical port 'bob1' in switch 'bob'. > +ADD_NAMESPACES(bob1) > +ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \ > + "172.16.1.2") > +ovn-nbctl lsp-add bob bob1 \ > +-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4" > + > +ADD_NAMESPACES(bob16) > +ADD_VETH(bob16, bob16, br-int, "fd30::4/64", "f0:00:00:02:02:06", \ > + "fd30::2") > +OVS_WAIT_UNTIL([test "$(ip netns exec bob16 ip a | grep fd30::4 | grep tentative)" = ""]) > +ovn-nbctl lsp-add bob bob16 \ > +-- lsp-set-addresses bob16 "f0:00:00:02:02:06 fd30::4" > + > +# Router R2 > +# Add a DNAT rule. > +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \ > + external_ip=30.0.0.2 -- add logical_router R2 nat @nat > +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \ > + external_ip='"fd40::2"' -- add logical_router R2 nat @nat > + > +# Add a SNAT rule > +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.1.2 \ > + external_ip=30.0.0.1 -- add logical_router R2 nat @nat > +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd11::2"' \ > + external_ip='"fd40::1"' -- add logical_router R2 nat @nat > + > +# Router R3 > +# Add a DNAT rule. > +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \ > + external_ip=30.0.0.3 -- add logical_router R3 nat @nat > +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \ > + external_ip='"fd40::3"' -- add logical_router R3 nat @nat > + > +# Add a SNAT rule > +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.2.2 \ > + external_ip=30.0.0.4 -- add logical_router R3 nat @nat > +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd12::2"' \ > + external_ip='"fd40::4"' -- add logical_router R3 nat @nat > + > +# wait for ovn-controller to catch up. > +ovn-nbctl --wait=hv sync > +OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=fd40::4)']) > +OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=30.0.0.4)']) > + > +# North-South DNAT: 'alice1' should be able to ping 'foo1' via 30.0.0.2 > +NS_CHECK_EXEC([alice1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.2 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# North-South DNAT: 'alice16' should be able to ping 'foo16' via fd30::2 > +NS_CHECK_EXEC([alice16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::2 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# Check conntrack entries. > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.3) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmp,orig=(src=172.16.1.3,dst=30.0.0.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=0,code=0),zone=<cleared> > +]) > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::3) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmpv6,orig=(src=fd30::3,dst=fd40::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::3,id=<cleared>,type=129,code=0),zone=<cleared> > +]) > + > +# But foo1 should receive traffic from 20.0.0.2 > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.2) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmp,orig=(src=172.16.1.3,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.2,id=<cleared>,type=0,code=0),zone=<cleared> > +]) > +# But foo16 should receive traffic from fd20::2 > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmpv6,orig=(src=fd30::3,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::2,id=<cleared>,type=129,code=0),zone=<cleared> > +]) > + > +# North-South DNAT: 'bob1' should be able to ping 'foo1' via 30.0.0.3 > +NS_CHECK_EXEC([bob1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.3 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# North-South DNAT: 'bob16' should be able to ping 'foo16' via fd40::3 > +NS_CHECK_EXEC([bob16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::3 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# Check conntrack entries. > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.4) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmp,orig=(src=172.16.1.4,dst=30.0.0.3,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.4,id=<cleared>,type=0,code=0),zone=<cleared> > +]) > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::4) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmpv6,orig=(src=fd30::4,dst=fd40::3,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::4,id=<cleared>,type=129,code=0),zone=<cleared> > +]) > + > +# But foo1 should receive traffic from 20.0.0.3 > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.3) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmp,orig=(src=172.16.1.4,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.3,id=<cleared>,type=0,code=0),zone=<cleared> > +]) > + > +# But foo16 should receive traffic from fd20::3 > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::3) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmpv6,orig=(src=fd30::4,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::3,id=<cleared>,type=129,code=0),zone=<cleared> > +]) > + > +# South-North SNAT: 'bar1' pings 'bob1'. But 'bob1' receives traffic > +# from 30.0.0.4 > +NS_CHECK_EXEC([bar1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.4 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > +# South-North SNAT: 'bar16' pings 'bob16'. But 'bob16' receives traffic > +# from fd40::4 > +NS_CHECK_EXEC([bar16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::4 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# We verify that SNAT indeed happened via 'dump-conntrack' command. > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.4) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmp,orig=(src=192.168.2.2,dst=172.16.1.4,id=<cleared>,type=8,code=0),reply=(src=172.16.1.4,dst=30.0.0.4,id=<cleared>,type=0,code=0),zone=<cleared> > +]) > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::4) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmpv6,orig=(src=fd12::2,dst=fd30::4,id=<cleared>,type=128,code=0),reply=(src=fd30::4,dst=fd40::4,id=<cleared>,type=129,code=0),zone=<cleared> > +]) > + > +# South-North SNAT: 'foo1' pings 'alice1'. But 'alice1' receives traffic > +# from 30.0.0.1 > +NS_CHECK_EXEC([foo1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.3 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# South-North SNAT: 'foo16' pings 'alice16'. But 'alice16' receives traffic > +# from fd40::1 > +NS_CHECK_EXEC([foo16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::3 | FORMAT_PING], \ > +[0], [dnl > +3 packets transmitted, 3 received, 0% packet loss, time 0ms > +]) > + > +# We verify that SNAT indeed happened via 'dump-conntrack' command. > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmp,orig=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=8,code=0),reply=(src=172.16.1.3,dst=30.0.0.1,id=<cleared>,type=0,code=0),zone=<cleared> > +]) > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::1) | \ > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +icmpv6,orig=(src=fd11::2,dst=fd30::3,id=<cleared>,type=128,code=0),reply=(src=fd30::3,dst=fd40::1,id=<cleared>,type=129,code=0),zone=<cleared> > +]) > + > +OVS_APP_EXIT_AND_WAIT([ovn-controller]) > + > +as ovn-sb > +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) > + > +as ovn-nb > +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) > + > +as northd > +OVS_APP_EXIT_AND_WAIT([ovn-northd]) > + > +as > +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d > +/connection dropped.*/d"]) > +AT_CLEANUP > + > + > AT_SETUP([ovn -- load-balancing]) > AT_KEYWORDS([ovnlb]) > > @@ -2405,6 +2722,230 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d > /connection dropped.*/d"]) > AT_CLEANUP > > +AT_SETUP([ovn -- multiple gateway routers, load-balancing - Dual Stack]) > +AT_KEYWORDS([ovnlb]) > + > +CHECK_CONNTRACK() > +CHECK_CONNTRACK_NAT() > +ovn_start > +OVS_TRAFFIC_VSWITCHD_START() > +ADD_BR([br-int]) > + > +# Set external-ids in br-int needed for ovn-controller > +ovs-vsctl \ > + -- set Open_vSwitch . external-ids:system-id=hv1 \ > + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ > + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ > + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ > + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true > + > +# Start ovn-controller > +start_daemon ovn-controller > + > +# Logical network: > +# Three LRs - R1, R2 and R3 that are connected to each other via LS "join" > +# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24 > +# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2 > +# has alice (172.16.1.0/24 and fd72::/64) connected to it. R3 has bob > +# (172.16.1.0/24 and fd72::/64) connected to it. Note how both alice and > +# bob have the same subnets behind them. We are trying to simulate external > +# network via those 2 switches. In real world the switch ports of these > +# switches will have addresses set as "unknown" to make them learning switches. > +# Or those switches will be "localnet" ones. > +# > +# foo -- R1 -- join - R2 -- alice > +# | | > +# bar ---- - R3 --- bob > + > +ovn-nbctl create Logical_Router name=R1 > +ovn-nbctl create Logical_Router name=R2 options:chassis=hv1 > +ovn-nbctl create Logical_Router name=R3 options:chassis=hv1 > + > +ovn-nbctl ls-add foo > +ovn-nbctl ls-add bar > +ovn-nbctl ls-add alice > +ovn-nbctl ls-add bob > +ovn-nbctl ls-add join > + > +# Connect foo to R1 > +ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64 > +ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \ > + type=router options:router-port=foo addresses=\"00:00:01:01:02:03\" > + > +# Connect bar to R1 > +ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64 > +ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \ > + type=router options:router-port=bar addresses=\"00:00:01:01:02:04\" > + > +# Connect alice to R2 > +ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd72::1/64 > +ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \ > + type=router options:router-port=alice addresses=\"00:00:02:01:02:03\" > + > +# Connect bob to R3 > +ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd72::2/64 > +ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \ > + type=router options:router-port=bob addresses=\"00:00:03:01:02:03\" > + > +# Connect R1 to join > +ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64 > +ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \ > + type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"' > + > +# Connect R2 to join > +ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64 > +ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \ > + type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"' > + > +# Connect R3 to join > +ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64 > +ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \ > + type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"' > + > +# Install static routes with source ip address as the policy for routing. > +# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3. > +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2 > +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3 > +ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2 > +ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3 > + > +# Static routes. > +ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1 > +ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1 > +ovn-nbctl lr-route-add R2 fd11::/64 fd20::1 > +ovn-nbctl lr-route-add R2 fd12::/64 fd20::1 > +ovn-nbctl lr-route-add R3 fd11::/64 fd20::1 > +ovn-nbctl lr-route-add R3 fd12::/64 fd20::1 > + > +# For gateway routers R2 and R3, set a force SNAT rule. > +ovn-nbctl set logical_router R2 options:lb_force_snat_ip="20.0.0.2 fd20::2" > +ovn-nbctl set logical_router R3 options:lb_force_snat_ip="20.0.0.3 fd20::3" > + > +# Logical port 'foo1' in switch 'foo'. > +ADD_NAMESPACES(foo1) > +ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \ > + "192.168.1.1") > +ovn-nbctl lsp-add foo foo1 \ > +-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2" > + > +# Logical port 'foo16' in switch 'foo'. > +ADD_NAMESPACES(foo16) > +ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:06:01:02:03", \ > + "fd11::1") > +ovn-nbctl lsp-add foo foo16 \ > +-- lsp-set-addresses foo16 "f0:00:06:01:02:03 fd11::2" > + > +# Logical port 'alice1' in switch 'alice'. > +ADD_NAMESPACES(alice1) > +ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \ > + "172.16.1.1") > +ovn-nbctl lsp-add alice alice1 \ > +-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3" > + > +# Logical port 'alice16' in switch 'alice'. > +ADD_NAMESPACES(alice16) > +ADD_VETH(alice16, alice16, br-int, "fd72::3/64", "f0:00:06:01:02:04", \ > + "fd72::1") > +ovn-nbctl lsp-add alice alice16 \ > +-- lsp-set-addresses alice16 "f0:00:06:01:02:04 fd72::3" > + > +# Logical port 'bar1' in switch 'bar'. > +ADD_NAMESPACES(bar1) > +ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \ > +"192.168.2.1") > +ovn-nbctl lsp-add bar bar1 \ > +-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2" > + > +# Logical port 'bar16' in switch 'bar'. > +ADD_NAMESPACES(bar16) > +ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:06:01:02:05", \ > +"fd12::1") > +ovn-nbctl lsp-add bar bar16 \ > +-- lsp-set-addresses bar16 "f0:00:06:01:02:05 fd12::2" > + > +# Logical port 'bob1' in switch 'bob'. > +ADD_NAMESPACES(bob1) > +ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \ > + "172.16.1.2") > +ovn-nbctl lsp-add bob bob1 \ > +-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4" > + > +# Logical port 'bob16' in switch 'bob'. > +ADD_NAMESPACES(bob16) > +ADD_VETH(bob16, bob16, br-int, "fd72::4/64", "f0:00:06:01:02:06", \ > + "fd72::2") > +ovn-nbctl lsp-add bob bob16 \ > +-- lsp-set-addresses bob16 "f0:00:06:01:02:06 fd72::4" > + > +# Config OVN load-balancer with a VIP. > +uuid=`ovn-nbctl create load_balancer vips:30.0.0.1="192.168.1.2,192.168.2.2" \ > +vips:\"fd30::1\"=\"fd11::2,fd12::2\"` > +ovn-nbctl set logical_router R2 load_balancer=$uuid > +ovn-nbctl set logical_router R3 load_balancer=$uuid > + > +# Wait for ovn-controller to catch up. > +ovn-nbctl --wait=hv sync > +OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \ > +grep 'nat(dst=192.168.2.2)']) > +OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \ > +grep 'nat(dst=fd12::2)']) > + > +# Start webservers in 'foo1', 'foo16, 'bar1', and 'bar16'. > +OVS_START_L7([foo1], [http]) > +OVS_START_L7([bar1], [http]) > +OVS_START_L7([foo16], [http6]) > +OVS_START_L7([bar16], [http6]) > + > +dnl Should work with the virtual IP address through NAT > +for i in `seq 1 20`; do > + echo Request $i > + NS_CHECK_EXEC([alice1], [wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) > +done > + > +for i in `seq 1 20`; do > + echo Request ${i}_6 > + NS_CHECK_EXEC([alice16], [wget http://[[fd30::1]] -t 5 -T 1 --retry-connrefused -v -o wget${i}_6.log]) > +done > + > +dnl Each server should have at least one connection. > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +]) > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::1) | grep -v fe80 | > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +]) > + > +dnl Force SNAT should have worked. > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0) | > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +tcp,orig=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +tcp,orig=(src=172.16.1.3,dst=192.168.2.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +]) > +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | grep -v fe80 | > +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl > +tcp,orig=(src=fd72::3,dst=fd11::2,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +tcp,orig=(src=fd72::3,dst=fd12::2,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) > +]) > +OVS_APP_EXIT_AND_WAIT([ovn-controller]) > + > +as ovn-sb > +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) > + > +as ovn-nb > +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) > + > +as northd > +OVS_APP_EXIT_AND_WAIT([ovn-northd]) > + > +as > +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d > +/connection dropped.*/d"]) > +AT_CLEANUP > + > AT_SETUP([ovn -- load balancing in router with gateway router port]) > AT_KEYWORDS([ovnlb]) > > -- > 2.25.4 > > _______________________________________________ > dev mailing list > dev@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev >
diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c index 192198272..2c05d1c2a 100644 --- a/northd/ovn-northd.c +++ b/northd/ovn-northd.c @@ -7801,44 +7801,37 @@ op_put_v6_networks(struct ds *ds, const struct ovn_port *op) ds_put_cstr(ds, "}"); } -static const char * +static bool get_force_snat_ip(struct ovn_datapath *od, const char *key_type, - struct v46_ip *ip) + struct lport_addresses *laddrs) { char *key = xasprintf("%s_force_snat_ip", key_type); - const char *ip_address = smap_get(&od->nbr->options, key); + const char *addresses = smap_get(&od->nbr->options, key); free(key); - if (ip_address) { - ovs_be32 mask; - ip->family = AF_INET; - char *error = ip_parse_masked(ip_address, &ip->ipv4, &mask); - if (error || mask != OVS_BE32_MAX) { - free(error); - struct in6_addr mask_v6, v6_exact = IN6ADDR_EXACT_INIT; - ip->family = AF_INET6; - error = ipv6_parse_masked(ip_address, &ip->ipv6, &mask_v6); - if (error || memcmp(&mask_v6, &v6_exact, sizeof(mask_v6))) { - static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); - VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"", - ip_address, UUID_ARGS(&od->key)); - memset(ip, 0, sizeof *ip); - ip->family = AF_UNSPEC; - return NULL; - } - } - return ip_address; + if (!addresses) { + return false; } - memset(ip, 0, sizeof *ip); - ip->family = AF_UNSPEC; - return NULL; + if (!extract_ip_addresses(addresses, laddrs) || + laddrs->n_ipv4_addrs > 1 || + laddrs->n_ipv6_addrs > 1 || + (laddrs->n_ipv4_addrs && laddrs->ipv4_addrs[0].plen != 32) || + (laddrs->n_ipv6_addrs && laddrs->ipv6_addrs[0].plen != 128)) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "bad ip %s in options of router "UUID_FMT"", + addresses, UUID_ARGS(&od->key)); + destroy_lport_addresses(laddrs); + return false; + } + + return true; } static void add_router_lb_flow(struct hmap *lflows, struct ovn_datapath *od, struct ds *match, struct ds *actions, int priority, - const char *lb_force_snat_ip, struct lb_vip *lb_vip, + bool lb_force_snat_ip, struct lb_vip *lb_vip, const char *proto, struct nbrec_load_balancer *lb, struct shash *meter_groups, struct sset *nat_entries) { @@ -8159,6 +8152,29 @@ build_lrouter_nd_flow(struct ovn_datapath *od, struct ovn_port *op, ds_destroy(&actions); } +static void +build_lrouter_force_snat_flows(struct hmap *lflows, struct ovn_datapath *od, + const char *ip_version, const char *ip_addr, + const char *context) +{ + struct ds match = DS_EMPTY_INITIALIZER; + struct ds actions = DS_EMPTY_INITIALIZER; + ds_put_format(&match, "ip%s && ip%s.dst == %s", + ip_version, ip_version, ip_addr); + ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110, + ds_cstr(&match), "ct_snat;"); + + /* Higher priority rules to force SNAT with the IP addresses + * configured in the Gateway router. This only takes effect + * when the packet has already been DNATed or load balanced once. */ + ds_clear(&match); + ds_put_format(&match, "flags.force_snat_for_%s == 1 && ip%s", + context, ip_version); + ds_put_format(&actions, "ct_snat(%s);", ip_addr); + ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100, + ds_cstr(&match), ds_cstr(&actions)); +} + static void build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, struct hmap *lflows, struct shash *meter_groups, @@ -8609,24 +8625,37 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, } } - /* A gateway router can have 2 SNAT IP addresses to force DNATed and + /* A gateway router can have 4 SNAT IP addresses to force DNATed and * LBed traffic respectively to be SNATed. In addition, there can be * a number of SNAT rules in the NAT table. */ struct v46_ip *snat_ips = xmalloc(sizeof *snat_ips - * (op->od->nbr->n_nat + 2)); + * (op->od->nbr->n_nat + 4)); size_t n_snat_ips = 0; + struct lport_addresses snat_addrs; - struct v46_ip snat_ip; - const char *dnat_force_snat_ip = get_force_snat_ip(op->od, "dnat", - &snat_ip); - if (dnat_force_snat_ip) { - snat_ips[n_snat_ips++] = snat_ip; + if (get_force_snat_ip(op->od, "dnat", &snat_addrs)) { + if (snat_addrs.n_ipv4_addrs) { + snat_ips[n_snat_ips].family = AF_INET; + snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr; + } + if (snat_addrs.n_ipv6_addrs) { + snat_ips[n_snat_ips].family = AF_INET6; + snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr; + } + destroy_lport_addresses(&snat_addrs); } - const char *lb_force_snat_ip = get_force_snat_ip(op->od, "lb", - &snat_ip); - if (lb_force_snat_ip) { - snat_ips[n_snat_ips++] = snat_ip; + memset(&snat_addrs, 0, sizeof(snat_addrs)); + if (get_force_snat_ip(op->od, "lb", &snat_addrs)) { + if (snat_addrs.n_ipv4_addrs) { + snat_ips[n_snat_ips].family = AF_INET; + snat_ips[n_snat_ips++].ipv4 = snat_addrs.ipv4_addrs[0].addr; + } + if (snat_addrs.n_ipv6_addrs) { + snat_ips[n_snat_ips].family = AF_INET6; + snat_ips[n_snat_ips++].ipv6 = snat_addrs.ipv6_addrs[0].addr; + } + destroy_lport_addresses(&snat_addrs); } for (size_t i = 0; i < op->od->nbr->n_nat; i++) { @@ -8985,11 +9014,12 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, struct sset nat_entries = SSET_INITIALIZER(&nat_entries); - struct v46_ip snat_ip, lb_snat_ip; - const char *dnat_force_snat_ip = get_force_snat_ip(od, "dnat", - &snat_ip); - const char *lb_force_snat_ip = get_force_snat_ip(od, "lb", - &lb_snat_ip); + struct lport_addresses dnat_force_snat_addrs; + struct lport_addresses lb_force_snat_addrs; + bool dnat_force_snat_ip = get_force_snat_ip(od, "dnat", + &dnat_force_snat_addrs); + bool lb_force_snat_ip = get_force_snat_ip(od, "lb", + &lb_force_snat_addrs); for (int i = 0; i < od->nbr->n_nat; i++) { const struct nbrec_nat *nat; @@ -9455,49 +9485,30 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports, } /* Handle force SNAT options set in the gateway router. */ - if (dnat_force_snat_ip && !od->l3dgw_port) { - /* If a packet with destination IP address as that of the - * gateway router (as set in options:dnat_force_snat_ip) is seen, - * UNSNAT it. */ - ds_clear(&match); - ds_put_format(&match, "ip && ip%s.dst == %s", - snat_ip.family == AF_INET ? "4" : "6", - dnat_force_snat_ip); - ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 110, - ds_cstr(&match), "ct_snat;"); - - /* Higher priority rules to force SNAT with the IP addresses - * configured in the Gateway router. This only takes effect - * when the packet has already been DNATed once. */ - ds_clear(&match); - ds_put_format(&match, "flags.force_snat_for_dnat == 1 && ip"); - ds_clear(&actions); - ds_put_format(&actions, "ct_snat(%s);", dnat_force_snat_ip); - ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100, - ds_cstr(&match), ds_cstr(&actions)); - } - if (lb_force_snat_ip && !od->l3dgw_port) { - /* If a packet with destination IP address as that of the - * gateway router (as set in options:lb_force_snat_ip) is seen, - * UNSNAT it. */ - ds_clear(&match); - ds_put_format(&match, "ip && ip%s.dst == %s", - lb_snat_ip.family == AF_INET ? "4" : "6", - lb_force_snat_ip); - ovn_lflow_add(lflows, od, S_ROUTER_IN_UNSNAT, 100, - ds_cstr(&match), "ct_snat;"); - - /* Load balanced traffic will have flags.force_snat_for_lb set. - * Force SNAT it. */ - ds_clear(&match); - ds_put_format(&match, "flags.force_snat_for_lb == 1 && ip"); - ds_clear(&actions); - ds_put_format(&actions, "ct_snat(%s);", lb_force_snat_ip); - ovn_lflow_add(lflows, od, S_ROUTER_OUT_SNAT, 100, - ds_cstr(&match), ds_cstr(&actions)); - } - if (!od->l3dgw_port) { + if (dnat_force_snat_ip) { + if (dnat_force_snat_addrs.n_ipv4_addrs) { + build_lrouter_force_snat_flows(lflows, od, "4", + dnat_force_snat_addrs.ipv4_addrs[0].addr_s, "dnat"); + } + if (dnat_force_snat_addrs.n_ipv6_addrs) { + build_lrouter_force_snat_flows(lflows, od, "6", + dnat_force_snat_addrs.ipv6_addrs[0].addr_s, "dnat"); + } + destroy_lport_addresses(&dnat_force_snat_addrs); + } + if (lb_force_snat_ip) { + if (lb_force_snat_addrs.n_ipv4_addrs) { + build_lrouter_force_snat_flows(lflows, od, "4", + lb_force_snat_addrs.ipv4_addrs[0].addr_s, "lb"); + } + if (lb_force_snat_addrs.n_ipv6_addrs) { + build_lrouter_force_snat_flows(lflows, od, "6", + lb_force_snat_addrs.ipv6_addrs[0].addr_s, "lb"); + } + destroy_lport_addresses(&lb_force_snat_addrs); + } + /* For gateway router, re-circulate every packet through * the DNAT zone. This helps with the following. * diff --git a/ovn-nb.xml b/ovn-nb.xml index db5908cd5..9f3da3563 100644 --- a/ovn-nb.xml +++ b/ovn-nb.xml @@ -1817,27 +1817,29 @@ </column> <column name="options" key="dnat_force_snat_ip"> <p> - If set, indicates the IP address to use to force SNAT a packet - that has already been DNATed in the gateway router. When multiple - gateway routers are configured, a packet can potentially enter any - of the gateway router, get DNATted and eventually reach the logical - switch port. For the return traffic to go back to the same gateway - router (for unDNATing), the packet needs a SNAT in the first place. - This can be achieved by setting the above option with a gateway - specific IP address. + If set, indicates a set of IP addresses to use to force SNAT a + packet that has already been DNATed in the gateway router. When + multiple gateway routers are configured, a packet can potentially + enter any of the gateway router, get DNATted and eventually reach the + logical switch port. For the return traffic to go back to the same + gateway router (for unDNATing), the packet needs a SNAT in the first + place. This can be achieved by setting the above option with a + gateway specific set of IP addresses. This option may have exactly + one IPv4 and/or one IPv6 address on it, separated by a a space. </p> </column> <column name="options" key="lb_force_snat_ip"> <p> - If set, indicates the IP address to use to force SNAT a packet + If set, indicates a set of IP addresses to use to force SNAT a packet that has already been load-balanced in the gateway router. When multiple gateway routers are configured, a packet can potentially enter any of the gateway routers, get DNATted as part of the load- balancing and eventually reach the logical switch port. For the return traffic to go back to the same gateway router (for unDNATing), the packet needs a SNAT in the first place. This can be - achieved by setting the above option with a gateway specific IP - address. + achieved by setting the above option with a gateway specific set of + IP addresses. This option may have exactly one IPv4 and/or one IPv6 + address on it, separated by a space character. </p> </column> <column name="options" key="mcast_relay" type='{"type": "boolean"}'> diff --git a/tests/system-ovn.at b/tests/system-ovn.at index 2999e52fd..079d61975 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -1026,6 +1026,323 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d /connection dropped.*/d"]) AT_CLEANUP +AT_SETUP([ovn -- multiple gateway routers, SNAT and DNAT - Dual Stack]) +AT_KEYWORDS([ovnnat]) + +CHECK_CONNTRACK() +CHECK_CONNTRACK_NAT() +ovn_start +OVS_TRAFFIC_VSWITCHD_START() +ADD_BR([br-int]) + +# Set external-ids in br-int needed for ovn-controller +ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +# Start ovn-controller +start_daemon ovn-controller + +# Logical network: +# Three LRs - R1, R2 and R3 that are connected to each other via LS "join" +# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24 +# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2 +# has alice (172.16.1.0/24 and fd30::/64) connected to it. R3 has bob +# (172.16.1.0/24 andfd30::/64) connected to it. Note how both alice and bob +# have the same subnets behind them. We are trying to simulate external network +# via those 2 switches. In real world the switch ports of these switches will +# have addresses set as "unknown" to make them learning switches. Or those +# switches will be "localnet" ones. +# +# foo -- R1 -- join - R2 -- alice +# | | +# bar ---- - R3 --- bob + +ovn-nbctl create Logical_Router name=R1 +ovn-nbctl create Logical_Router name=R2 options:chassis=hv1 +ovn-nbctl create Logical_Router name=R3 options:chassis=hv1 + +ovn-nbctl ls-add foo +ovn-nbctl ls-add bar +ovn-nbctl ls-add alice +ovn-nbctl ls-add bob +ovn-nbctl ls-add join + +# Connect foo to R1 +ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64 +ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \ + type=router options:router-port=foo addresses=\"00:00:01:01:02:03\" + +# Connect bar to R1 +ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64 +ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \ + type=router options:router-port=bar addresses=\"00:00:01:01:02:04\" + +# Connect alice to R2 +ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd30::1/64 +ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \ + type=router options:router-port=alice addresses=\"00:00:02:01:02:03\" + +# Connect bob to R3 +ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd30::2/64 +ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \ + type=router options:router-port=bob addresses=\"00:00:03:01:02:03\" + +# Connect R1 to join +ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64 +ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \ + type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"' + +# Connect R2 to join +ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64 +ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \ + type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"' + +# Connect R3 to join +ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64 +ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \ + type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"' + +# Install static routes with source ip address as the policy for routing. +# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3. +ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2 +ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3 +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2 +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3 + +# Static routes. +ovn-nbctl lr-route-add R2 fd11::/64 fd20::1 +ovn-nbctl lr-route-add R2 fd12::/64 fd20::1 +ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1 +ovn-nbctl lr-route-add R3 fd11::/64 fd20::1 +ovn-nbctl lr-route-add R3 fd12::/64 fd20::1 +ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1 + +# For gateway routers R2 and R3, set a force SNAT rule. +ovn-nbctl set logical_router R2 options:dnat_force_snat_ip="20.0.0.2 fd20::2" +ovn-nbctl set logical_router R3 options:dnat_force_snat_ip="20.0.0.3 fd20::3" + +# Logical port 'foo1' in switch 'foo'. +ADD_NAMESPACES(foo1) +ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \ + "192.168.1.1") +ovn-nbctl lsp-add foo foo1 \ +-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2" + +ADD_NAMESPACES(foo16) +ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:00:02:02:03", \ + "fd11::1") +OVS_WAIT_UNTIL([test "$(ip netns exec foo16 ip a | grep fd11::2 | grep tentative)" = ""]) +ovn-nbctl lsp-add foo foo16 \ +-- lsp-set-addresses foo16 "f0:00:00:02:02:03 fd11::2" + +# Logical port 'alice1' in switch 'alice'. +ADD_NAMESPACES(alice1) +ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \ + "172.16.1.1") +ovn-nbctl lsp-add alice alice1 \ +-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3" + +ADD_NAMESPACES(alice16) +ADD_VETH(alice16, alice16, br-int, "fd30::3/64", "f0:00:00:02:02:04", \ + "fd30::1") +OVS_WAIT_UNTIL([test "$(ip netns exec alice16 ip a | grep fd30::3 | grep tentative)" = ""]) +ovn-nbctl lsp-add alice alice16 \ +-- lsp-set-addresses alice16 "f0:00:00:02:02:04 fd30::3" + +# Logical port 'bar1' in switch 'bar'. +ADD_NAMESPACES(bar1) +ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \ +"192.168.2.1") +ovn-nbctl lsp-add bar bar1 \ +-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2" + +ADD_NAMESPACES(bar16) +ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:00:02:02:05", \ + "fd12::1") +OVS_WAIT_UNTIL([test "$(ip netns exec bar16 ip a | grep fd12::2 | grep tentative)" = ""]) +ovn-nbctl lsp-add bar bar16 \ +-- lsp-set-addresses bar16 "f0:00:00:02:02:05 fd12::2" + +# Logical port 'bob1' in switch 'bob'. +ADD_NAMESPACES(bob1) +ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \ + "172.16.1.2") +ovn-nbctl lsp-add bob bob1 \ +-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4" + +ADD_NAMESPACES(bob16) +ADD_VETH(bob16, bob16, br-int, "fd30::4/64", "f0:00:00:02:02:06", \ + "fd30::2") +OVS_WAIT_UNTIL([test "$(ip netns exec bob16 ip a | grep fd30::4 | grep tentative)" = ""]) +ovn-nbctl lsp-add bob bob16 \ +-- lsp-set-addresses bob16 "f0:00:00:02:02:06 fd30::4" + +# Router R2 +# Add a DNAT rule. +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \ + external_ip=30.0.0.2 -- add logical_router R2 nat @nat +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \ + external_ip='"fd40::2"' -- add logical_router R2 nat @nat + +# Add a SNAT rule +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.1.2 \ + external_ip=30.0.0.1 -- add logical_router R2 nat @nat +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd11::2"' \ + external_ip='"fd40::1"' -- add logical_router R2 nat @nat + +# Router R3 +# Add a DNAT rule. +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip=192.168.1.2 \ + external_ip=30.0.0.3 -- add logical_router R3 nat @nat +ovn-nbctl -- --id=@nat create nat type="dnat" logical_ip='"fd11::2"' \ + external_ip='"fd40::3"' -- add logical_router R3 nat @nat + +# Add a SNAT rule +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=192.168.2.2 \ + external_ip=30.0.0.4 -- add logical_router R3 nat @nat +ovn-nbctl -- --id=@nat create nat type="snat" logical_ip='"fd12::2"' \ + external_ip='"fd40::4"' -- add logical_router R3 nat @nat + +# wait for ovn-controller to catch up. +ovn-nbctl --wait=hv sync +OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=fd40::4)']) +OVS_WAIT_UNTIL([ovs-ofctl dump-flows br-int | grep 'nat(src=30.0.0.4)']) + +# North-South DNAT: 'alice1' should be able to ping 'foo1' via 30.0.0.2 +NS_CHECK_EXEC([alice1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.2 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# North-South DNAT: 'alice16' should be able to ping 'foo16' via fd30::2 +NS_CHECK_EXEC([alice16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::2 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# Check conntrack entries. +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.3) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmp,orig=(src=172.16.1.3,dst=30.0.0.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=0,code=0),zone=<cleared> +]) +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::3) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmpv6,orig=(src=fd30::3,dst=fd40::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::3,id=<cleared>,type=129,code=0),zone=<cleared> +]) + +# But foo1 should receive traffic from 20.0.0.2 +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.2) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmp,orig=(src=172.16.1.3,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.2,id=<cleared>,type=0,code=0),zone=<cleared> +]) +# But foo16 should receive traffic from fd20::2 +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmpv6,orig=(src=fd30::3,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::2,id=<cleared>,type=129,code=0),zone=<cleared> +]) + +# North-South DNAT: 'bob1' should be able to ping 'foo1' via 30.0.0.3 +NS_CHECK_EXEC([bob1], [ping -q -c 3 -i 0.3 -w 2 30.0.0.3 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# North-South DNAT: 'bob16' should be able to ping 'foo16' via fd40::3 +NS_CHECK_EXEC([bob16], [ping -6 -q -c 3 -i 0.3 -w 2 fd40::3 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# Check conntrack entries. +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(172.16.1.4) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmp,orig=(src=172.16.1.4,dst=30.0.0.3,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=172.16.1.4,id=<cleared>,type=0,code=0),zone=<cleared> +]) +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::4) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmpv6,orig=(src=fd30::4,dst=fd40::3,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd30::4,id=<cleared>,type=129,code=0),zone=<cleared> +]) + +# But foo1 should receive traffic from 20.0.0.3 +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0.3) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmp,orig=(src=172.16.1.4,dst=192.168.1.2,id=<cleared>,type=8,code=0),reply=(src=192.168.1.2,dst=20.0.0.3,id=<cleared>,type=0,code=0),zone=<cleared> +]) + +# But foo16 should receive traffic from fd20::3 +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::3) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmpv6,orig=(src=fd30::4,dst=fd11::2,id=<cleared>,type=128,code=0),reply=(src=fd11::2,dst=fd20::3,id=<cleared>,type=129,code=0),zone=<cleared> +]) + +# South-North SNAT: 'bar1' pings 'bob1'. But 'bob1' receives traffic +# from 30.0.0.4 +NS_CHECK_EXEC([bar1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.4 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) +# South-North SNAT: 'bar16' pings 'bob16'. But 'bob16' receives traffic +# from fd40::4 +NS_CHECK_EXEC([bar16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::4 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# We verify that SNAT indeed happened via 'dump-conntrack' command. +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.4) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmp,orig=(src=192.168.2.2,dst=172.16.1.4,id=<cleared>,type=8,code=0),reply=(src=172.16.1.4,dst=30.0.0.4,id=<cleared>,type=0,code=0),zone=<cleared> +]) +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::4) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmpv6,orig=(src=fd12::2,dst=fd30::4,id=<cleared>,type=128,code=0),reply=(src=fd30::4,dst=fd40::4,id=<cleared>,type=129,code=0),zone=<cleared> +]) + +# South-North SNAT: 'foo1' pings 'alice1'. But 'alice1' receives traffic +# from 30.0.0.1 +NS_CHECK_EXEC([foo1], [ping -q -c 3 -i 0.3 -w 2 172.16.1.3 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# South-North SNAT: 'foo16' pings 'alice16'. But 'alice16' receives traffic +# from fd40::1 +NS_CHECK_EXEC([foo16], [ping -6 -q -c 3 -i 0.3 -w 2 fd30::3 | FORMAT_PING], \ +[0], [dnl +3 packets transmitted, 3 received, 0% packet loss, time 0ms +]) + +# We verify that SNAT indeed happened via 'dump-conntrack' command. +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmp,orig=(src=192.168.1.2,dst=172.16.1.3,id=<cleared>,type=8,code=0),reply=(src=172.16.1.3,dst=30.0.0.1,id=<cleared>,type=0,code=0),zone=<cleared> +]) +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd40::1) | \ +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +icmpv6,orig=(src=fd11::2,dst=fd30::3,id=<cleared>,type=128,code=0),reply=(src=fd30::3,dst=fd40::1,id=<cleared>,type=129,code=0),zone=<cleared> +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) +AT_CLEANUP + + AT_SETUP([ovn -- load-balancing]) AT_KEYWORDS([ovnlb]) @@ -2405,6 +2722,230 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d /connection dropped.*/d"]) AT_CLEANUP +AT_SETUP([ovn -- multiple gateway routers, load-balancing - Dual Stack]) +AT_KEYWORDS([ovnlb]) + +CHECK_CONNTRACK() +CHECK_CONNTRACK_NAT() +ovn_start +OVS_TRAFFIC_VSWITCHD_START() +ADD_BR([br-int]) + +# Set external-ids in br-int needed for ovn-controller +ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +# Start ovn-controller +start_daemon ovn-controller + +# Logical network: +# Three LRs - R1, R2 and R3 that are connected to each other via LS "join" +# in 20.0.0.0/24 and fd20::/64 networks. R1 has switches foo (192.168.1.0/24 +# and fd11::/64) and bar (192.168.2.0/24 and fd12::/64) connected to it. R2 +# has alice (172.16.1.0/24 and fd72::/64) connected to it. R3 has bob +# (172.16.1.0/24 and fd72::/64) connected to it. Note how both alice and +# bob have the same subnets behind them. We are trying to simulate external +# network via those 2 switches. In real world the switch ports of these +# switches will have addresses set as "unknown" to make them learning switches. +# Or those switches will be "localnet" ones. +# +# foo -- R1 -- join - R2 -- alice +# | | +# bar ---- - R3 --- bob + +ovn-nbctl create Logical_Router name=R1 +ovn-nbctl create Logical_Router name=R2 options:chassis=hv1 +ovn-nbctl create Logical_Router name=R3 options:chassis=hv1 + +ovn-nbctl ls-add foo +ovn-nbctl ls-add bar +ovn-nbctl ls-add alice +ovn-nbctl ls-add bob +ovn-nbctl ls-add join + +# Connect foo to R1 +ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24 fd11::1/64 +ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \ + type=router options:router-port=foo addresses=\"00:00:01:01:02:03\" + +# Connect bar to R1 +ovn-nbctl lrp-add R1 bar 00:00:01:01:02:04 192.168.2.1/24 fd12::1/64 +ovn-nbctl lsp-add bar rp-bar -- set Logical_Switch_Port rp-bar \ + type=router options:router-port=bar addresses=\"00:00:01:01:02:04\" + +# Connect alice to R2 +ovn-nbctl lrp-add R2 alice 00:00:02:01:02:03 172.16.1.1/24 fd72::1/64 +ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \ + type=router options:router-port=alice addresses=\"00:00:02:01:02:03\" + +# Connect bob to R3 +ovn-nbctl lrp-add R3 bob 00:00:03:01:02:03 172.16.1.2/24 fd72::2/64 +ovn-nbctl lsp-add bob rp-bob -- set Logical_Switch_Port rp-bob \ + type=router options:router-port=bob addresses=\"00:00:03:01:02:03\" + +# Connect R1 to join +ovn-nbctl lrp-add R1 R1_join 00:00:04:01:02:03 20.0.0.1/24 fd20::1/64 +ovn-nbctl lsp-add join r1-join -- set Logical_Switch_Port r1-join \ + type=router options:router-port=R1_join addresses='"00:00:04:01:02:03"' + +# Connect R2 to join +ovn-nbctl lrp-add R2 R2_join 00:00:04:01:02:04 20.0.0.2/24 fd20::2/64 +ovn-nbctl lsp-add join r2-join -- set Logical_Switch_Port r2-join \ + type=router options:router-port=R2_join addresses='"00:00:04:01:02:04"' + +# Connect R3 to join +ovn-nbctl lrp-add R3 R3_join 00:00:04:01:02:05 20.0.0.3/24 fd20::3/64 +ovn-nbctl lsp-add join r3-join -- set Logical_Switch_Port r3-join \ + type=router options:router-port=R3_join addresses='"00:00:04:01:02:05"' + +# Install static routes with source ip address as the policy for routing. +# We want traffic from 'foo' to go via R2 and traffic of 'bar' to go via R3. +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.1.0/24 20.0.0.2 +ovn-nbctl --policy="src-ip" lr-route-add R1 192.168.2.0/24 20.0.0.3 +ovn-nbctl --policy="src-ip" lr-route-add R1 fd11::/64 fd20::2 +ovn-nbctl --policy="src-ip" lr-route-add R1 fd12::/64 fd20::3 + +# Static routes. +ovn-nbctl lr-route-add R2 192.168.0.0/16 20.0.0.1 +ovn-nbctl lr-route-add R3 192.168.0.0/16 20.0.0.1 +ovn-nbctl lr-route-add R2 fd11::/64 fd20::1 +ovn-nbctl lr-route-add R2 fd12::/64 fd20::1 +ovn-nbctl lr-route-add R3 fd11::/64 fd20::1 +ovn-nbctl lr-route-add R3 fd12::/64 fd20::1 + +# For gateway routers R2 and R3, set a force SNAT rule. +ovn-nbctl set logical_router R2 options:lb_force_snat_ip="20.0.0.2 fd20::2" +ovn-nbctl set logical_router R3 options:lb_force_snat_ip="20.0.0.3 fd20::3" + +# Logical port 'foo1' in switch 'foo'. +ADD_NAMESPACES(foo1) +ADD_VETH(foo1, foo1, br-int, "192.168.1.2/24", "f0:00:00:01:02:03", \ + "192.168.1.1") +ovn-nbctl lsp-add foo foo1 \ +-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2" + +# Logical port 'foo16' in switch 'foo'. +ADD_NAMESPACES(foo16) +ADD_VETH(foo16, foo16, br-int, "fd11::2/64", "f0:00:06:01:02:03", \ + "fd11::1") +ovn-nbctl lsp-add foo foo16 \ +-- lsp-set-addresses foo16 "f0:00:06:01:02:03 fd11::2" + +# Logical port 'alice1' in switch 'alice'. +ADD_NAMESPACES(alice1) +ADD_VETH(alice1, alice1, br-int, "172.16.1.3/24", "f0:00:00:01:02:04", \ + "172.16.1.1") +ovn-nbctl lsp-add alice alice1 \ +-- lsp-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.3" + +# Logical port 'alice16' in switch 'alice'. +ADD_NAMESPACES(alice16) +ADD_VETH(alice16, alice16, br-int, "fd72::3/64", "f0:00:06:01:02:04", \ + "fd72::1") +ovn-nbctl lsp-add alice alice16 \ +-- lsp-set-addresses alice16 "f0:00:06:01:02:04 fd72::3" + +# Logical port 'bar1' in switch 'bar'. +ADD_NAMESPACES(bar1) +ADD_VETH(bar1, bar1, br-int, "192.168.2.2/24", "f0:00:00:01:02:05", \ +"192.168.2.1") +ovn-nbctl lsp-add bar bar1 \ +-- lsp-set-addresses bar1 "f0:00:00:01:02:05 192.168.2.2" + +# Logical port 'bar16' in switch 'bar'. +ADD_NAMESPACES(bar16) +ADD_VETH(bar16, bar16, br-int, "fd12::2/64", "f0:00:06:01:02:05", \ +"fd12::1") +ovn-nbctl lsp-add bar bar16 \ +-- lsp-set-addresses bar16 "f0:00:06:01:02:05 fd12::2" + +# Logical port 'bob1' in switch 'bob'. +ADD_NAMESPACES(bob1) +ADD_VETH(bob1, bob1, br-int, "172.16.1.4/24", "f0:00:00:01:02:06", \ + "172.16.1.2") +ovn-nbctl lsp-add bob bob1 \ +-- lsp-set-addresses bob1 "f0:00:00:01:02:06 172.16.1.4" + +# Logical port 'bob16' in switch 'bob'. +ADD_NAMESPACES(bob16) +ADD_VETH(bob16, bob16, br-int, "fd72::4/64", "f0:00:06:01:02:06", \ + "fd72::2") +ovn-nbctl lsp-add bob bob16 \ +-- lsp-set-addresses bob16 "f0:00:06:01:02:06 fd72::4" + +# Config OVN load-balancer with a VIP. +uuid=`ovn-nbctl create load_balancer vips:30.0.0.1="192.168.1.2,192.168.2.2" \ +vips:\"fd30::1\"=\"fd11::2,fd12::2\"` +ovn-nbctl set logical_router R2 load_balancer=$uuid +ovn-nbctl set logical_router R3 load_balancer=$uuid + +# Wait for ovn-controller to catch up. +ovn-nbctl --wait=hv sync +OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \ +grep 'nat(dst=192.168.2.2)']) +OVS_WAIT_UNTIL([ovs-ofctl -O OpenFlow13 dump-groups br-int | \ +grep 'nat(dst=fd12::2)']) + +# Start webservers in 'foo1', 'foo16, 'bar1', and 'bar16'. +OVS_START_L7([foo1], [http]) +OVS_START_L7([bar1], [http]) +OVS_START_L7([foo16], [http6]) +OVS_START_L7([bar16], [http6]) + +dnl Should work with the virtual IP address through NAT +for i in `seq 1 20`; do + echo Request $i + NS_CHECK_EXEC([alice1], [wget 30.0.0.1 -t 5 -T 1 --retry-connrefused -v -o wget$i.log]) +done + +for i in `seq 1 20`; do + echo Request ${i}_6 + NS_CHECK_EXEC([alice16], [wget http://[[fd30::1]] -t 5 -T 1 --retry-connrefused -v -o wget${i}_6.log]) +done + +dnl Each server should have at least one connection. +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(30.0.0.1) | +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +tcp,orig=(src=172.16.1.3,dst=30.0.0.1,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=172.16.1.3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +]) +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd30::1) | grep -v fe80 | +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +tcp,orig=(src=fd72::3,dst=fd30::1,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd72::3,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +]) + +dnl Force SNAT should have worked. +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(20.0.0) | +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +tcp,orig=(src=172.16.1.3,dst=192.168.1.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.1.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +tcp,orig=(src=172.16.1.3,dst=192.168.2.2,sport=<cleared>,dport=<cleared>),reply=(src=192.168.2.2,dst=20.0.0.2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +]) +AT_CHECK([ovs-appctl dpctl/dump-conntrack | FORMAT_CT(fd20::2) | grep -v fe80 | +sed -e 's/zone=[[0-9]]*/zone=<cleared>/'], [0], [dnl +tcp,orig=(src=fd72::3,dst=fd11::2,sport=<cleared>,dport=<cleared>),reply=(src=fd11::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +tcp,orig=(src=fd72::3,dst=fd12::2,sport=<cleared>,dport=<cleared>),reply=(src=fd12::2,dst=fd20::2,sport=<cleared>,dport=<cleared>),zone=<cleared>,protoinfo=(state=<cleared>) +]) +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) +AT_CLEANUP + AT_SETUP([ovn -- load balancing in router with gateway router port]) AT_KEYWORDS([ovnlb])
The lb_force_snat and dnat_force_snat options could accept only a single IP address. For routers that only route traffic of a single IP address family, this is fine. However, if a router routes both IPv4 and IPv6 traffic, then this limitation is a problem. This patch addresses this problem by allowing for these options to specify both an IPv4 and IPv6 address. Signed-off-by: Mark Michelson <mmichels@redhat.com> --- northd/ovn-northd.c | 179 ++++++++------- ovn-nb.xml | 24 +- tests/system-ovn.at | 541 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 649 insertions(+), 95 deletions(-)