[ovs-dev,v11,2/4] northd: Add IP routing and ARP resolution flows for NAT/LB addresses.

diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c
index c10361a17..3cf821e9b 100644
--- a/northd/ovn-northd.c
+++ b/northd/ovn-northd.c
@@ -1405,6 +1405,19 @@  build_datapaths(struct northd_context *ctx, struct hmap *datapaths,
     }
 }
 
+/* Structure representing logical router port
+ * routable addresses. This includes DNAT and Load Balancer
+ * addresses. This structure will only be filled in if the
+ * router port is a gateway router port. Otherwise, all pointers
+ * will be NULL and n_addrs will be 0.
+ */
+struct ovn_port_routable_addresses {
+    /* The parsed routable addresses */
+    struct lport_addresses *laddrs;
+    /* Number of items in the laddrs array */
+    size_t n_addrs;
+};
+
 /* A logical switch port or logical router port.
  *
  * In steady state, an ovn_port points to a northbound Logical_Switch_Port
@@ -1448,6 +1461,8 @@  struct ovn_port {
 
     struct lport_addresses lrp_networks;
 
+    struct ovn_port_routable_addresses routables;
+
     /* Logical port multicast data. */
     struct mcast_port_info mcast_info;
 
@@ -1474,6 +1489,46 @@  struct ovn_port {
     struct ovs_list list;       /* In list of similar records. */
 };
 
+static void
+destroy_routable_addresses(struct ovn_port_routable_addresses *ra)
+{
+    for (size_t i = 0; i < ra->n_addrs; i++) {
+        destroy_lport_addresses(&ra->laddrs[i]);
+    }
+    free(ra->laddrs);
+}
+
+static char **get_nat_addresses(const struct ovn_port *op, size_t *n);
+
+static void
+assign_routable_addresses(struct ovn_port *op)
+{
+    size_t n;
+    char **nats = get_nat_addresses(op, &n);
+
+    if (!nats) {
+        return;
+    }
+
+    struct lport_addresses *laddrs = xcalloc(n, sizeof(*laddrs));
+    size_t n_addrs = 0;
+    for (size_t i = 0; i < n; i++) {
+        int ofs;
+        if (!extract_addresses(nats[i], &laddrs[n_addrs], &ofs)) {
+            free(nats[i]);
+            continue;
+        }
+        n_addrs++;
+        free(nats[i]);
+    }
+    free(nats);
+
+    /* Everything seems to have worked out */
+    op->routables.laddrs = laddrs;
+    op->routables.n_addrs = n_addrs;
+}
+
+
 static void
 ovn_port_set_nb(struct ovn_port *op,
                 const struct nbrec_logical_switch_port *nbsp,
@@ -1523,6 +1578,8 @@  ovn_port_destroy(struct hmap *ports, struct ovn_port *port)
         }
         free(port->ps_addrs);
 
+        destroy_routable_addresses(&port->routables);
+
         destroy_lport_addresses(&port->lrp_networks);
         free(port->json_key);
         free(port->key);
@@ -2430,6 +2487,8 @@  join_logical_ports(struct northd_context *ctx,
                      * use during flow creation. */
                     od->l3dgw_port = op;
                     od->l3redirect_port = crp;
+
+                    assign_routable_addresses(op);
                 }
             }
         }
@@ -2513,7 +2572,7 @@  get_nat_addresses(const struct ovn_port *op, size_t *n)
 {
     size_t n_nats = 0;
     struct eth_addr mac;
-    if (!op->nbrp || !op->od || !op->od->nbr
+    if (!op || !op->nbrp || !op->od || !op->od->nbr
         || (!op->od->nbr->n_nat && !op->od->nbr->n_load_balancer)
         || !eth_addr_from_string(op->nbrp->mac, &mac)) {
         *n = n_nats;
@@ -3094,7 +3153,6 @@  ovn_port_update_sbrec(struct northd_context *ctx,
             } else {
                 sbrec_port_binding_set_options(op->sb, NULL);
             }
-
             const char *nat_addresses = smap_get(&op->nbsp->options,
                                            "nat-addresses");
             size_t n_nats = 0;
@@ -3150,6 +3208,7 @@  ovn_port_update_sbrec(struct northd_context *ctx,
             if (add_router_port_garp) {
                 struct ds garp_info = DS_EMPTY_INITIALIZER;
                 ds_put_format(&garp_info, "%s", op->peer->lrp_networks.ea_s);
+
                 for (size_t i = 0; i < op->peer->lrp_networks.n_ipv4_addrs;
                      i++) {
                     ds_put_format(&garp_info, " %s",
@@ -3166,7 +3225,6 @@  ovn_port_update_sbrec(struct northd_context *ctx,
                 nats[n_nats - 1] = ds_steal_cstr(&garp_info);
                 ds_destroy(&garp_info);
             }
-
             sbrec_port_binding_set_nat_addresses(op->sb,
                                                  (const char **) nats, n_nats);
             for (size_t i = 0; i < n_nats; i++) {
@@ -10031,7 +10089,7 @@  build_ND_RA_flows_for_lrouter(struct ovn_datapath *od, struct hmap *lflows)
  */
 static void
 build_ip_routing_flows_for_lrouter_port(
-        struct ovn_port *op, struct hmap *lflows)
+        struct ovn_port *op, struct hmap *ports,struct hmap *lflows)
 {
     if (op->nbrp) {
 
@@ -10048,6 +10106,31 @@  build_ip_routing_flows_for_lrouter_port(
                       op->lrp_networks.ipv6_addrs[i].plen, NULL, false,
                       &op->nbrp->header_, false);
         }
+    } else if (lsp_is_router(op->nbsp)) {
+        struct ovn_port *peer = ovn_port_get_peer(ports, op);
+        if (!peer || !peer->nbrp || !peer->lrp_networks.n_ipv4_addrs) {
+            return;
+        }
+
+        for (int i = 0; i < op->od->n_router_ports; i++) {
+            struct ovn_port *router_port = ovn_port_get_peer(
+                    ports, op->od->router_ports[i]);
+            if (!router_port || !router_port->nbrp || router_port == peer) {
+                continue;
+            }
+
+            struct ovn_port_routable_addresses *ra = &router_port->routables;
+            for (size_t j = 0; j < ra->n_addrs; j++) {
+                struct lport_addresses *laddrs = &ra->laddrs[j];
+                for (size_t k = 0; k < laddrs->n_ipv4_addrs; k++) {
+                    add_route(lflows, peer->od, peer,
+                              peer->lrp_networks.ipv4_addrs[0].addr_s,
+                              laddrs->ipv4_addrs[k].network_s,
+                              laddrs->ipv4_addrs[k].plen, NULL, false,
+                              &peer->nbrp->header_, false);
+                }
+            }
+        }
     }
 }
 
@@ -10226,6 +10309,37 @@  build_arp_resolve_flows_for_lrouter(
     }
 }
 
+static void
+routable_addresses_to_lflows(struct hmap *lflows, struct ovn_port *router_port,
+                             struct ovn_port *peer, struct ds *match,
+                             struct ds *actions)
+{
+    struct ovn_port_routable_addresses *ra = &router_port->routables;
+    if (!ra->n_addrs) {
+        return;
+    }
+
+    for (size_t i = 0; i < ra->n_addrs; i++) {
+        ds_clear(match);
+        ds_put_format(match, "outport == %s && "REG_NEXT_HOP_IPV4" == {",
+                      peer->json_key);
+        bool first = true;
+        for (size_t j = 0; j < ra->laddrs[i].n_ipv4_addrs; j++) {
+            if (!first) {
+                ds_put_cstr(match, ", ");
+            }
+            ds_put_cstr(match, ra->laddrs[i].ipv4_addrs[j].addr_s);
+            first = false;
+        }
+        ds_put_cstr(match, "}");
+
+        ds_clear(actions);
+        ds_put_format(actions, "eth.dst = %s; next;", ra->laddrs[i].ea_s);
+        ovn_lflow_add(lflows, peer->od, S_ROUTER_IN_ARP_RESOLVE, 100,
+                      ds_cstr(match), ds_cstr(actions));
+    }
+}
+
 /* Local router ingress table ARP_RESOLVE: ARP Resolution.
  *
  * Any unicast packet that reaches this table is an IP packet whose
@@ -10548,6 +10662,12 @@  build_arp_resolve_flows_for_lrouter_port(
                                         ds_cstr(match), ds_cstr(actions),
                                         &op->nbsp->header_);
             }
+
+            if (smap_get(&peer->od->nbr->options, "chassis") ||
+                (peer->od->l3dgw_port && peer == peer->od->l3dgw_port)) {
+                routable_addresses_to_lflows(lflows, router_port, peer,
+                                             match, actions);
+            }
         }
     }
 
@@ -12084,7 +12204,7 @@  build_lswitch_and_lrouter_iterate_by_op(struct ovn_port *op,
                                           &lsi->actions);
     build_neigh_learning_flows_for_lrouter_port(op, lsi->lflows, &lsi->match,
                                                 &lsi->actions);
-    build_ip_routing_flows_for_lrouter_port(op, lsi->lflows);
+    build_ip_routing_flows_for_lrouter_port(op, lsi->ports, lsi->lflows);
     build_ND_RA_flows_for_lrouter_port(op, lsi->lflows, &lsi->match,
                                        &lsi->actions);
     build_arp_resolve_flows_for_lrouter_port(op, lsi->lflows, lsi->ports,
diff --git a/northd/ovn_northd.dl b/northd/ovn_northd.dl
index e27c944a0..8afc93b5d 100644
--- a/northd/ovn_northd.dl
+++ b/northd/ovn_northd.dl
@@ -23,6 +23,7 @@  import multicast
 import helpers
 import ipam
 import vec
+import set
 
 index Logical_Flow_Index() on sb::Out_Logical_Flow()
 
@@ -6507,6 +6508,45 @@  Route(key, dst.port, dst.src_ip, Some{dst.nexthop}) :-
     dsts.size() == 1,
     Some{var dst} = dsts.nth(0).
 
+/* Create routes from peer to port's routable addresses */
+Route(key, peer, src_ip, None) :-
+    RouterPortRoutableAddresses(port, addresses),
+    FirstHopRouterPortRoutableAddresses(port, peer_uuid),
+    peer_lrp in &nb::Logical_Router_Port(._uuid = peer_uuid),
+    peer in &RouterPort(.lrp = peer_lrp, .networks = networks),
+    Some{var src} = networks.ipv4_addrs.first(),
+    var src_ip = IPv4{src.addr},
+    var addr = FlatMap(addresses),
+    var ip4_addr = FlatMap(addr.ipv4_addrs),
+    var key = RouteKey{DstIp, IPv4{ip4_addr.addr}, ip4_addr.plen}.
+
+/* This relation indicates that logical router port "port" has routable
+ * addresses (i.e. DNAT and Load Balancer VIPs) and that logical router
+ * port "peer" is reachable via a hop across a single logical switch.
+ */
+relation FirstHopRouterPortRoutableAddresses(
+    port: uuid,
+    peer: uuid)
+FirstHopRouterPortRoutableAddresses(port_uuid, peer_uuid) :-
+    FirstHopLogicalRouter(r1, ls),
+    FirstHopLogicalRouter(r2, ls),
+    r1 != r2,
+    LogicalRouterPort(port_uuid, r1),
+    LogicalRouterPort(peer_uuid, r2),
+    RouterPortRoutableAddresses(.rport = port_uuid),
+    lrp in &nb::Logical_Router_Port(._uuid = port_uuid),
+    peer_lrp in &nb::Logical_Router_Port(._uuid = peer_uuid),
+    LogicalSwitchRouterPort(_, lrp.name, ls),
+    LogicalSwitchRouterPort(_, peer_lrp.name, ls).
+
+relation RouterPortRoutableAddresses(
+    rport: uuid,
+    addresses: Set<lport_addresses>)
+RouterPortRoutableAddresses(port.lrp._uuid, addresses) :-
+    port in &RouterPort(.is_redirect = true),
+    var addresses = get_nat_addresses(port).filter_map(extract_addresses),
+    addresses != set_empty().
+
 /* Return a vector of pairs (1, set[0]), ... (n, set[n - 1]). */
 function numbered_vec(set: Set<'A>) : Vec<(bit<16>, 'A)> = {
     var vec = vec_with_capacity(set.size());
@@ -6994,6 +7034,23 @@  Flow(.logical_datapath = lr_uuid,
     snat_ips.contains_key(IPv6{addr.addr}),
     var match_ips = "${addr.addr}".group_by((lr_uuid, lrp_uuid)).to_vec().
 
+/* Create ARP resolution flows for NAT and LB addresses for first hop
+ * logical routers
+ */
+Flow(.logical_datapath = peer.router._uuid,
+     .stage = s_ROUTER_IN_ARP_RESOLVE(),
+     .priority = 100,
+     .__match = "outport == ${peer.json_name} && " ++ rEG_NEXT_HOP() ++ " == {${ips}}",
+     .actions = "eth.dst = ${addr.ea}; next;",
+     .external_ids = stage_hint(lrp._uuid)) :-
+     RouterPortRoutableAddresses(port, addresses),
+     FirstHopRouterPortRoutableAddresses(port, peer_uuid),
+     peer in &RouterPort(.lrp = lrp),
+     lrp._uuid == peer_uuid,
+     not peer.router.options.get_bool_def("dynamic_neigh_routers", false),
+     var addr = FlatMap(addresses),
+     var ips = addr.ipv4_addrs.map(|a| a.addr.to_string()).join(", ").
+
 /* This is a logical switch port that backs a VM or a container.
  * Extract its addresses. For each of the address, go through all
  * the router ports attached to the switch (to which this port
diff --git a/tests/ovn-northd.at b/tests/ovn-northd.at
index 02640d163..389393909 100644
--- a/tests/ovn-northd.at
+++ b/tests/ovn-northd.at
@@ -3787,3 +3787,217 @@  AT_CHECK([grep -w "ls_in_dhcp_options" sw0flows | sort], [0], [dnl
 
 AT_CLEANUP
 ])
+
+# XXX This test currently only runs for ovn-northd.c. The test fails
+# with ovn-northd-ddlog because of the section where 2 HA_Chassis_Groups
+# are used by 2 routers. For some reason, this causes ovn-northd-ddlog
+# to stop processing new changes to the northbound database and to
+# seemingly infinitely loop. This issue has been reported, but there is
+# currently no fix for it. Once this issue is fixed, we can run this
+# test for both C and DDLog versions of northd.
+AT_SETUP([ovn -- NAT and Load Balancer flows])
+
+# Determine if expected flows are present. The only parameter to this
+# function is the number of expected flows per NAT destination address.
+# This should always be either 0 or 1. 0 means that we do not expect
+# lflows to be present. 1 means we expect an lflow to be present
+check_lflows() {
+    expected=$1
+    ro1_flows=$(ovn-sbctl lflow-list ro1)
+
+    ro1_ip_routing=$(grep lr_in_ip_routing <<< "$ro1_flows")
+    match=$(grep -c "match=(ip4.dst == 20.0.0.100/32)" <<< "$ro1_ip_routing")
+    AT_CHECK([test "$expected" = "$match"])
+
+    ro1_arp_resolve=$(grep lr_in_arp_resolve <<< "$ro1_flows")
+    match=$(grep -c 'match=(outport == "ro1-sw" && reg0 == {20.0.0.100})' <<< "$ro1_arp_resolve")
+    AT_CHECK([test "$expected" = "$match"])
+
+    ro2_flows=$(ovn-sbctl lflow-list ro2)
+
+    ro2_ip_routing=$(grep lr_in_ip_routing <<< "$ro2_flows")
+    match=$(grep -c "match=(ip4.dst == 10.0.0.100/32)" <<< "$ro2_ip_routing")
+    AT_CHECK([test "$expected" = "$match"])
+
+    ro2_arp_resolve=$(grep lr_in_arp_resolve <<< "$ro2_flows")
+    match=$(grep -c 'match=(outport == "ro2-sw" && reg0 == {10.0.0.100})' <<< "$ro2_arp_resolve")
+    AT_CHECK([test "$expected" = "$match"])
+}
+
+ovn_start
+
+AS_BOX([Setting up the logical network])
+
+check ovn-nbctl ls-add sw
+
+check ovn-nbctl lr-add ro1
+check ovn-nbctl lrp-add ro1 ro1-sw 00:00:00:00:00:01 10.0.0.1/24
+check ovn-nbctl lsp-add sw sw-ro1
+
+check ovn-nbctl lr-add ro2
+check ovn-nbctl lrp-add ro2 ro2-sw 00:00:00:00:00:02 20.0.0.1/24
+check ovn-nbctl --wait=sb lsp-add sw sw-ro2
+
+check ovn-nbctl ls-add ls1
+check ovn-nbctl lsp-add ls1 vm1
+check ovn-nbctl lsp-set-addresses vm1 "00:00:00:00:01:02 192.168.1.2"
+check ovn-nbctl lrp-add ro1 ro1-ls1 00:00:00:00:01:01 192.168.1.1/24
+check ovn-nbctl lsp-add ls1 ls1-ro1
+check ovn-nbctl lsp-set-type ls1-ro1 router
+check ovn-nbctl lsp-set-addresses ls1-ro1 router
+check ovn-nbctl lsp-set-options ls1-ro1 router-port=ro1-ls1
+
+check ovn-nbctl ls-add ls2
+check ovn-nbctl lsp-add ls2 vm2
+check ovn-nbctl lsp-set-addresses vm2 "00:00:00:00:02:02 192.168.2.2"
+check ovn-nbctl lrp-add ro2 ro2-ls2 00:00:00:00:02:01 192.168.2.1/24
+check ovn-nbctl lsp-add ls2 ls2-ro2
+check ovn-nbctl lsp-set-type ls2-ro2 router
+check ovn-nbctl lsp-set-addresses ls2-ro2 router
+check ovn-nbctl lsp-set-options ls2-ro2 router-port=ro2-ls2
+
+check ovn-nbctl ha-chassis-group-add grp1
+check ovn-nbctl ha-chassis-group-add-chassis grp1 hv1 100
+grp1_uuid=$(ovn-nbctl --columns=_uuid --bare find HA_Chassis_group name=grp1)
+
+check ovn-nbctl ha-chassis-group-add grp2
+check ovn-nbctl ha-chassis-group-add-chassis grp2 hv2 100
+grp2_uuid=$(ovn-nbctl --columns=_uuid --bare find HA_Chassis_group name=grp2)
+
+AS_BOX([Checking that unconnected logical switch ports generate no lflows])
+
+check_lflows 0
+
+AS_BOX([Checking that connected logical switch ports have no lflows for non-gateway ports])
+
+check ovn-nbctl lsp-set-type sw-ro1 router
+check ovn-nbctl lsp-set-addresses sw-ro1 router
+check ovn-nbctl lsp-set-options sw-ro1 router-port=ro1-sw
+
+check ovn-nbctl lsp-set-type sw-ro2 router
+check ovn-nbctl lsp-set-addresses sw-ro2 router
+check ovn-nbctl --wait=sb lsp-set-options sw-ro2 router-port=ro2-sw
+
+check_lflows 0
+
+AS_BOX([Checking that NAT flows are not installed for non-gateway routers])
+
+check ovn-nbctl lr-nat-add ro1 dnat 10.0.0.100 192.168.1.100
+check ovn-nbctl lr-nat-add ro2 dnat 20.0.0.100 192.168.2.100
+
+check_lflows 0
+
+AS_BOX([Checking that NAT flows are installed for gateway routers])
+
+check ovn-nbctl lrp-set-gateway-chassis ro1-sw hv1 100
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis ro2-sw hv2 100
+
+check_lflows 1
+
+AS_BOX([Checking that NAT flows are not installed for routers with gateway chassis removed])
+
+check ovn-nbctl lrp-del-gateway-chassis ro1-sw hv1
+check ovn-nbctl --wait=sb lrp-del-gateway-chassis ro2-sw hv2
+
+check_lflows 0
+
+AS_BOX([Checking that NAT flows are installed for routers with HA_Chassis_Group])
+
+check ovn-nbctl set logical_router_port ro1-sw ha_chassis_group="$grp1_uuid"
+check ovn-nbctl --wait=sb set logical_router_port ro2-sw ha_chassis_group="$grp2_uuid"
+
+check_lflows 1
+
+AS_BOX([Checking that NAT flows are not installed for routers with HA_Chassis_Group removed])
+
+check ovn-nbctl clear logical_router_port ro1-sw ha_chassis_group
+check ovn-nbctl --wait=sb clear logical_router_port ro2-sw ha_chassis_group
+
+check_lflows 0
+
+AS_BOX([Checking that Floating IP NAT flows are not installed with no gateway port set])
+
+check ovn-nbctl lr-nat-del ro1
+check ovn-nbctl lr-nat-del ro2
+
+check ovn-nbctl lr-nat-add ro1 dnat_and_snat 10.0.0.100 192.168.1.2 vm1 00:00:00:00:00:01
+check ovn-nbctl lr-nat-add ro2 dnat_and_snat 20.0.0.100 192.168.2.2 vm2 00:00:00:00:00:02
+
+check_lflows 0
+
+AS_BOX([Checking that Floating IP NAT flows are installed for gateway routers])
+
+check ovn-nbctl lrp-set-gateway-chassis ro1-sw hv1 100
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis ro2-sw hv2 100
+
+check_lflows 1
+
+AS_BOX([Checking that Floating IP NAT flows are not installed for routers with gateway chassis removed])
+
+check ovn-nbctl lrp-del-gateway-chassis ro1-sw hv1
+check ovn-nbctl --wait=sb lrp-del-gateway-chassis ro2-sw hv2
+
+check_lflows 0
+
+AS_BOX([Checking that Floating IP NAT flows are installed for routers with ha_chassis_group])
+
+grp1_uuid=$(ovn-nbctl --columns=_uuid --bare find HA_Chassis_group name=grp1)
+check ovn-nbctl set logical_router_port ro1-sw ha_chassis_group="$grp1_uuid"
+
+grp2_uuid=$(ovn-nbctl --columns=_uuid --bare find HA_Chassis_group name=grp2)
+check ovn-nbctl --wait=sb set logical_router_port ro2-sw ha_chassis_group="$grp2_uuid"
+
+check_lflows 1
+
+AS_BOX([Checking that Floating IP NAT flows are not installed for routers with HA_Chassis_Group removed])
+
+check ovn-nbctl clear logical_router_port ro1-sw ha_chassis_group
+check ovn-nbctl --wait=sb clear logical_router_port ro2-sw ha_chassis_group
+
+check_lflows 0
+
+AS_BOX([Checking that Load Balancer VIP flows are not installed for routers with no gateway port])
+
+check ovn-nbctl lr-nat-del ro1
+check ovn-nbctl lr-nat-del ro2
+
+check ovn-nbctl lb-add lb1 10.0.0.100 192.168.1.2
+check ovn-nbctl lr-lb-add ro1 lb1
+
+check ovn-nbctl lb-add lb2 20.0.0.100 192.168.2.2
+check ovn-nbctl lr-lb-add ro2 lb2
+
+check_lflows 0
+
+AS_BOX([Checking that Load Balancer VIP flows are installed for gateway routers])
+
+check ovn-nbctl lrp-set-gateway-chassis ro1-sw hv1 100
+check ovn-nbctl --wait=sb lrp-set-gateway-chassis ro2-sw hv2 100
+
+check_lflows 1
+
+AS_BOX([Checking that Load Balancer VIP flows are not installed for routers with gateway chassis removed])
+
+check ovn-nbctl lrp-del-gateway-chassis ro1-sw hv1
+check ovn-nbctl --wait=sb lrp-del-gateway-chassis ro2-sw hv2
+
+check_lflows 0
+
+AS_BOX([Checking that Load Balancer VIP flows are installed for routers with ha_chassis_group])
+
+grp1_uuid=$(ovn-nbctl --columns=_uuid --bare find HA_Chassis_group name=grp1)
+check ovn-nbctl set logical_router_port ro1-sw ha_chassis_group="$grp1_uuid"
+
+grp2_uuid=$(ovn-nbctl --columns=_uuid --bare find HA_Chassis_group name=grp2)
+check ovn-nbctl --wait=sb set logical_router_port ro2-sw ha_chassis_group="$grp2_uuid"
+
+check_lflows 1
+
+AS_BOX([Checking that Load Balancer VIP flows are not iinstalled for routers with HA_Chassis_Group removed])
+
+check ovn-nbctl clear logical_router_port ro1-sw ha_chassis_group
+check ovn-nbctl --wait=sb clear logical_router_port ro2-sw ha_chassis_group
+
+check_lflows 0
+
+AT_CLEANUP