@@ -355,6 +355,14 @@ consider_local_datapath(struct controller_ctx *ctx,
add_local_datapath(ldatapaths, lports, binding_rec->datapath,
false, local_datapaths);
}
+ } else if (!strcmp(binding_rec->type, "patch")) {
+ const char *chassis_id = smap_get(&binding_rec->options,
+ "redirect-chassis");
+ our_chassis = chassis_id && !strcmp(chassis_id, chassis_rec->name);
+ if (our_chassis) {
+ add_local_datapath(ldatapaths, lports, binding_rec->datapath,
+ false, local_datapaths);
+ }
} else if (!strcmp(binding_rec->type, "l3gateway")) {
const char *chassis_id = smap_get(&binding_rec->options,
"l3gateway-chassis");
@@ -163,6 +163,10 @@ update_sb_monitors(struct ovsdb_idl *ovnsb_idl,
sbrec_port_binding_add_clause_options(&pb, OVSDB_F_INCLUDES, &l2);
const struct smap l3 = SMAP_CONST1(&l3, "l3gateway-chassis", id);
sbrec_port_binding_add_clause_options(&pb, OVSDB_F_INCLUDES, &l3);
+ const struct smap redirect = SMAP_CONST1(&redirect,
+ "redirect-chassis", id);
+ sbrec_port_binding_add_clause_options(&pb, OVSDB_F_INCLUDES,
+ &redirect);
}
if (local_ifaces) {
const char *name;
@@ -324,6 +324,40 @@ consider_port_binding(enum mf_field_id mff_ovn_geneve,
put_local_common_flows(dp_key, port_key, false, &binding_zones,
ofpacts_p, flow_table);
+ if (binding->chassis && binding->chassis != chassis) {
+ /* Table 32, priority 100.
+ * =======================
+ *
+ * Redirect traffic with MLF_FORCE_CHASSIS_REDIRECT set and output
+ * port of type "patch" that is a distributed gateway port,
+ * through a tunnel to binding->chassis.
+ */
+ const struct chassis_tunnel *tun = chassis_tunnel_find(
+ binding->chassis->name);
+ if (tun) {
+ ofp_port_t ofport = tun->ofport;
+
+ match_init_catchall(&match);
+ ofpbuf_clear(ofpacts_p);
+
+ /* Match MFF_LOG_DATAPATH, MFF_LOG_OUTPORT, and
+ * MLF_FORCE_CHASSIS_REDIRECT. */
+ match_set_metadata(&match, htonll(dp_key));
+ match_set_reg(&match, MFF_LOG_OUTPORT - MFF_REG0, port_key);
+ match_set_reg_masked(&match, MFF_LOG_FLAGS - MFF_REG0,
+ MLF_FORCE_CHASSIS_REDIRECT,
+ MLF_FORCE_CHASSIS_REDIRECT);
+
+ put_encapsulation(mff_ovn_geneve, tun, binding->datapath,
+ port_key, ofpacts_p);
+
+ /* Output to tunnel. */
+ ofpact_put_OUTPUT(ofpacts_p)->port = ofport;
+ ofctrl_add_flow(flow_table, OFTABLE_REMOTE_OUTPUT, 100, 0,
+ &match, ofpacts_p);
+ }
+ }
+
match_init_catchall(&match);
ofpbuf_clear(ofpacts_p);
match_set_metadata(&match, htonll(dp_key));
@@ -96,6 +96,10 @@ ovn_init_symtab(struct shash *symtab)
MLF_FORCE_SNAT_FOR_LB_BIT);
expr_symtab_add_subfield(symtab, "flags.force_snat_for_lb", NULL,
flags_str);
+ snprintf(flags_str, sizeof flags_str, "flags[%d]",
+ MLF_FORCE_CHASSIS_REDIRECT_BIT);
+ expr_symtab_add_subfield(symtab, "flags.force_chassis_redirect", NULL,
+ flags_str);
/* Connection tracking state. */
expr_symtab_add_field(symtab, "ct_mark", MFF_CT_MARK, NULL, false);
@@ -49,6 +49,7 @@ enum mff_log_flags_bits {
MLF_RCV_FROM_VXLAN_BIT = 1,
MLF_FORCE_SNAT_FOR_DNAT_BIT = 2,
MLF_FORCE_SNAT_FOR_LB_BIT = 3,
+ MLF_FORCE_CHASSIS_REDIRECT_BIT = 4,
};
/* MFF_LOG_FLAGS_REG flag assignments */
@@ -69,6 +70,11 @@ enum mff_log_flags {
/* Indicate that a packet needs a force SNAT in the gateway router when
* load-balancing has taken place. */
MLF_FORCE_SNAT_FOR_LB = (1 << MLF_FORCE_SNAT_FOR_LB_BIT),
+
+ /* Indicate that the packet needs to be redirected to the chassis
+ * associated with the port_binding. This only takes effect when the
+ * port_binding type is "patch". */
+ MLF_FORCE_CHASSIS_REDIRECT = (1 << MLF_FORCE_CHASSIS_REDIRECT_BIT),
};
#endif /* ovn/lib/logical-fields.h */
@@ -740,9 +740,21 @@ output;
</li>
<li>
- One priority-50 flow that matches each known Ethernet address against
- <code>eth.dst</code> and outputs the packet to the single associated
- output port.
+ <p>
+ One priority-50 flow that matches each known Ethernet address against
+ <code>eth.dst</code> and outputs the packet to the single associated
+ output port.
+ </p>
+
+ <p>
+ For the Ethernet address on a logical switch port of type
+ <code>router</code>, when that logical switch port's
+ <ref column="addresses" table="Logical_Switch_Port"
+ db="OVN_Northbound"/> column is set to <code>router</code> and
+ the connected logical router port specifies a
+ <code>redirect-chassis</code>, the flow is only programmed on the
+ <code>redirect-chassis</code>.
+ </p>
</li>
<li>
@@ -862,10 +874,21 @@ output;
</li>
<li>
- For each enabled router port <var>P</var> with Ethernet address
- <var>E</var>, a priority-50 flow that matches <code>inport ==
- <var>P</var> && (eth.mcast || eth.dst ==
- <var>E</var></code>), with action <code>next;</code>.
+ <p>
+ For each enabled router port <var>P</var> with Ethernet address
+ <var>E</var>, a priority-50 flow that matches <code>inport ==
+ <var>P</var> && (eth.mcast || eth.dst ==
+ <var>E</var></code>), with action <code>next;</code>.
+ </p>
+
+ <p>
+ For the gateway port on a distributed logical router (where
+ one of the logical router ports specifies a
+ <code>redirect-chassis</code>), the above flow matching
+ <code>eth.dst == <var>E</var></code> is only programmed on
+ the gateway port instance on the
+ <code>redirect-chassis</code>.
+ </p>
</li>
</ul>
@@ -980,6 +1003,17 @@ outport = <var>P</var>;
flags.loopback = 1;
output;
</pre>
+
+ <p>
+ For the gateway port on a distributed logical router (where
+ one of the logical router ports specifies a
+ <code>redirect-chassis</code>), the above flows are only
+ programmed on the gateway port instance on the
+ <code>redirect-chassis</code>. This behavior avoids generation
+ of multiple ARP responses from different chassis, and allows
+ upstream MAC learning to point to the
+ <code>redirect-chassis</code>.
+ </p>
</li>
<li>
@@ -1040,6 +1074,17 @@ nd_na {
output;
};
</pre>
+
+ <p>
+ For the gateway port on a distributed logical router (where
+ one of the logical router ports specifies a
+ <code>redirect-chassis</code>), the above flows replying to
+ IPv6 Neighbor Solicitations are only programmed on the
+ gateway port instance on the <code>redirect-chassis</code>.
+ This behavior avoids generation of multiple replies from
+ different chassis, and allows upstream MAC learning to point
+ to the <code>redirect-chassis</code>.
+ </p>
</li>
<li>
@@ -1485,7 +1530,40 @@ next;
</li>
</ul>
- <h3>Ingress Table 7: ARP Request</h3>
+ <h3>Ingress Table 7: Gateway Redirect</h3>
+
+ <p>
+ For distributed logical routers where one of the logical router
+ ports specifies a <code>redirect-chassis</code>, this table redirects
+ certain packets to the distributed gateway port instance on the
+ <code>redirect-chassis</code>. This table has the following flows:
+ </p>
+
+ <ul>
+ <li>
+ A priority-150 logical flow with match
+ <code>outport == <var>GW</var> &&
+ eth.dst == 00:00:00:00:00:00</code> has actions
+ <code>flags.force_chassis_redirect = 1; next;</code>, where
+ <var>GW</var> is the logical router distributed gateway
+ port.
+ </li>
+
+ <li>
+ A priority-50 logical flow with match
+ <code>outport == <var>GW</var></code> has actions
+ <code>flags.force_chassis_redirect = 1; next;</code>, where
+ <var>GW</var> is the logical router distributed gateway
+ port.
+ </li>
+
+ <li>
+ A priority-0 logical flow with match <code>1</code> has actions
+ <code>next;</code>.
+ </li>
+ </ul>
+
+ <h3>Ingress Table 8: ARP Request</h3>
<p>
In the common case where the Ethernet destination has been resolved, this
@@ -132,7 +132,8 @@ enum ovn_stage {
PIPELINE_STAGE(ROUTER, IN, DNAT, 4, "lr_in_dnat") \
PIPELINE_STAGE(ROUTER, IN, IP_ROUTING, 5, "lr_in_ip_routing") \
PIPELINE_STAGE(ROUTER, IN, ARP_RESOLVE, 6, "lr_in_arp_resolve") \
- PIPELINE_STAGE(ROUTER, IN, ARP_REQUEST, 7, "lr_in_arp_request") \
+ PIPELINE_STAGE(ROUTER, IN, GW_REDIRECT, 7, "lr_in_gw_redirect") \
+ PIPELINE_STAGE(ROUTER, IN, ARP_REQUEST, 8, "lr_in_arp_request") \
\
/* Logical router egress stages. */ \
PIPELINE_STAGE(ROUTER, OUT, SNAT, 0, "lr_out_snat") \
@@ -382,6 +383,13 @@ struct ovn_datapath {
/* IPAM data. */
struct hmap ipam;
+
+ /* OVN northd only needs to know about the logical router gateway port
+ * when there is a "redirect-chassis" specified for one of the ports on
+ * the logical router, indicating that the port is a "distributed gateway
+ * port". This port is used for NAT on a distributed router.
+ * Otherwise this will be NULL. */
+ struct ovn_port *l3dgw_port;
};
struct macam_node {
@@ -1299,6 +1307,34 @@ join_logical_ports(struct northd_context *ctx,
op->lrp_networks = lrp_networks;
op->od = od;
ipam_add_port_addresses(op->od, op);
+
+ const char *redirect_chassis = smap_get(&op->nbrp->options,
+ "redirect-chassis");
+ if (redirect_chassis) {
+ const char *gw_chassis = smap_get(&op->od->nbr->options,
+ "chassis");
+ if (gw_chassis) {
+ static struct vlog_rate_limit rl
+ = VLOG_RATE_LIMIT_INIT(1, 1);
+ VLOG_WARN_RL(&rl, "Bad configuration: "
+ "redirect-chassis configured on port %s "
+ "on L3 gateway router", nbrp->name);
+ continue;
+ }
+
+ /* Set l3dgw_port in od, for later use during flow
+ * creation. */
+ if (od->l3dgw_port) {
+ static struct vlog_rate_limit rl
+ = VLOG_RATE_LIMIT_INIT(1, 1);
+ VLOG_WARN_RL(&rl, "Bad configuration: multiple ports "
+ "with redirect-chassis on same logical "
+ "router %s", od->nbr->name);
+ continue;
+ } else {
+ od->l3dgw_port = op;
+ }
+ }
}
}
}
@@ -1366,10 +1402,13 @@ ovn_port_update_sbrec(const struct ovn_port *op,
/* If the router is for l3 gateway, it resides on a chassis
* and its port type is "l3gateway". */
const char *chassis = smap_get(&op->od->nbr->options, "chassis");
+ const char *redirect_chassis = NULL;
if (chassis) {
sbrec_port_binding_set_type(op->sb, "l3gateway");
} else {
sbrec_port_binding_set_type(op->sb, "patch");
+ redirect_chassis = smap_get(&op->nbrp->options,
+ "redirect-chassis");
}
const char *peer = op->peer ? op->peer->key : "<error>";
@@ -1378,6 +1417,8 @@ ovn_port_update_sbrec(const struct ovn_port *op,
smap_add(&new, "peer", peer);
if (chassis) {
smap_add(&new, "l3gateway-chassis", chassis);
+ } else if (redirect_chassis) {
+ smap_add(&new, "redirect-chassis", redirect_chassis);
}
sbrec_port_binding_set_options(op->sb, &new);
smap_destroy(&new);
@@ -3145,6 +3186,14 @@ build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
ds_clear(&match);
ds_put_format(&match, "eth.dst == "ETH_ADDR_FMT,
ETH_ADDR_ARGS(mac));
+ if (op->peer->od->l3dgw_port
+ && op->peer == op->peer->od->l3dgw_port) {
+ /* The destination lookup flow for the router's
+ * distributed gateway port MAC address should only be
+ * programmed on the "redirect-chassis". */
+ ds_put_format(&match, " && is_chassis_resident(%s)",
+ op->peer->od->l3dgw_port->json_key);
+ }
ds_clear(&actions);
ds_put_format(&actions, "outport = %s; output;", op->json_key);
@@ -3584,8 +3633,19 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
}
ds_clear(&match);
- ds_put_format(&match, "(eth.mcast || eth.dst == %s) && inport == %s",
+ ds_put_format(&match, "eth.mcast && inport == %s", op->json_key);
+ ovn_lflow_add(lflows, op->od, S_ROUTER_IN_ADMISSION, 50,
+ ds_cstr(&match), "next;");
+
+ ds_clear(&match);
+ ds_put_format(&match, "eth.dst == %s && inport == %s",
op->lrp_networks.ea_s, op->json_key);
+ if (op->od->l3dgw_port && op == op->od->l3dgw_port) {
+ /* Traffic with eth.dst = l3dgw_port->lrp_networks.ea_s
+ * should only be received on the "redirect-chassis". */
+ ds_put_format(&match, " && is_chassis_resident(%s)",
+ op->od->l3dgw_port->json_key);
+ }
ovn_lflow_add(lflows, op->od, S_ROUTER_IN_ADMISSION, 50,
ds_cstr(&match), "next;");
}
@@ -3687,6 +3747,15 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
ds_put_format(&match,
"inport == %s && arp.tpa == %s && arp.op == 1",
op->json_key, op->lrp_networks.ipv4_addrs[i].addr_s);
+ if (op->od->l3dgw_port && op == op->od->l3dgw_port) {
+ /* Traffic with eth.src = l3dgw_port->lrp_networks.ea_s
+ * should only be sent from the "redirect-chassis", so that
+ * upstream MAC learning points to the "redirect-chassis".
+ * Also need to avoid generation of multiple ARP responses
+ * from different chassis. */
+ ds_put_format(&match, " && is_chassis_resident(%s)",
+ op->od->l3dgw_port->json_key);
+ }
ds_clear(&actions);
ds_put_format(&actions,
@@ -3918,6 +3987,15 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
op->lrp_networks.ipv6_addrs[i].addr_s,
op->lrp_networks.ipv6_addrs[i].sn_addr_s,
op->lrp_networks.ipv6_addrs[i].addr_s);
+ if (op->od->l3dgw_port && op == op->od->l3dgw_port) {
+ /* Traffic with eth.src = l3dgw_port->lrp_networks.ea_s
+ * should only be sent from the "redirect-chassis", so that
+ * upstream MAC learning points to the "redirect-chassis".
+ * Also need to avoid generation of multiple ND replies
+ * from different chassis. */
+ ds_put_format(&match, " && is_chassis_resident(%s)",
+ op->od->l3dgw_port->json_key);
+ }
ds_clear(&actions);
ds_put_format(&actions,
@@ -4427,7 +4505,46 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
"get_nd(outport, xxreg0); next;");
}
- /* Local router ingress table 7: ARP request.
+ /* Logical router ingress table 7: Gateway redirect.
+ *
+ * For traffic with outport equal to the l3dgw_port on a
+ * distributed router, this table redirects a subset of the
+ * traffic to the central instance of the l3dgw_port by
+ * setting MLF_FORCE_CHASSIS_REDIRECT.
+ */
+ HMAP_FOR_EACH (od, key_node, datapaths) {
+ if (!od->nbr) {
+ continue;
+ }
+ if (od->l3dgw_port) {
+ /* For traffic with outport == l3dgw_port, if the
+ * packet did not match any higher priority redirect
+ * rule, then the traffic is redirected to the central
+ * instance of the l3dgw_port. */
+ ds_clear(&match);
+ ds_put_format(&match, "outport == %s",
+ od->l3dgw_port->json_key);
+ ovn_lflow_add(lflows, od, S_ROUTER_IN_GW_REDIRECT, 50,
+ ds_cstr(&match),
+ "flags.force_chassis_redirect = 1; next;");
+
+ /* If the Ethernet destination has not been resolved,
+ * redirect to the central instance of the l3dgw_port.
+ * Such traffic will be replaced by an ARP request or ND
+ * Neighbor Solicitation in the ARP request ingress
+ * table, before being redirected to the central instance.
+ */
+ ds_put_format(&match, " && eth.dst == 00:00:00:00:00:00");
+ ovn_lflow_add(lflows, od, S_ROUTER_IN_GW_REDIRECT, 150,
+ ds_cstr(&match),
+ "flags.force_chassis_redirect = 1; next;");
+ }
+
+ /* Packets are allowed by default. */
+ ovn_lflow_add(lflows, od, S_ROUTER_IN_GW_REDIRECT, 0, "1", "next;");
+ }
+
+ /* Local router ingress table 8: ARP request.
*
* In the common case where the Ethernet destination has been resolved,
* this table outputs the packet (priority 0). Otherwise, it composes
@@ -1139,8 +1139,8 @@
</p>
<p>
- The following section describes an exception, where logical routers
- and logical patch ports are associated with a physical location.
+ The following sections describe two exceptions, where logical routers
+ and/or logical patch ports are associated with a physical location.
</p>
<h3>Gateway Routers</h3>
@@ -1183,6 +1183,130 @@
one-to-many SNAT (aka IP masquerading).
</p>
+ <h3>Distributed Gateway Ports</h3>
+
+ <p>
+ <dfn>Distributed gateway ports</dfn> are logical router patch ports
+ that directly connect distributed logical routers to logical
+ switches with localnet ports.
+ </p>
+
+ <p>
+ The primary design goal of distributed gateway ports is to allow as
+ much traffic as possible to be handled locally on the hypervisor
+ where a VM or container resides. Whenever possible, packets from
+ the VM or container to the outside world should be processed
+ completely on that VM's or container's hypervisor, eventually
+ traversing a localnet port instance on that hypervisor to the
+ physical network. Whenever possible, packets from the outside
+ world to a VM or container should be directed through the physical
+ network directly to the VM's or container's hypervisor, where the
+ packet will enter the integration bridge through a localnet port.
+ </p>
+
+ <p>
+ In order to allow for the distributed processing of packets
+ described in the paragraph above, distributed gateway ports need to
+ be logical patch ports that effectively reside on every hypervisor,
+ rather than <code>l3gateway</code> ports that are bound to a
+ particular chassis. However, the flows associated with distributed
+ gateway ports often need to be associated with physical locations,
+ for the following reasons:
+ </p>
+
+ <ul>
+ <li>
+ <p>
+ The physical network that the localnet port is attached to
+ typically uses L2 learning. Any Ethernet address used over the
+ distributed gateway port must be restricted to a single physical
+ location so that upstream L2 learning is not confused. Traffic
+ sent out the distributed gateway port towards the localnet port
+ with a specific Ethernet address must be sent out one specific
+ instance of the distributed gateway port on one specific
+ chassis. Traffic received from the localnet port (or from a VIF
+ on the same logical switch as the localnet port) with a specific
+ Ethernet address must be directed to the logical switch's patch
+ port instance on that specific chassis.
+ </p>
+
+ <p>
+ Due to the implications of L2 learning, the Ethernet address and
+ IP address of the distributed gateway port need to be restricted
+ to a single physical location. For this reason, the user must
+ specify one chassis associated with the distributed gateway
+ port. Note that traffic traversing the distributed gateway port
+ using other Ethernet addresses and IP addresses (e.g. one-to-one
+ NAT) is not restricted to this chassis.
+ </p>
+
+ <p>
+ Replies to ARP and ND requests must be restricted to a single
+ physical location, where the Ethernet address in the reply
+ resides. This includes ARP and ND replies for the IP address
+ of the distributed gateway port, which are restricted to the
+ chassis that the user associated with the distributed gateway
+ port.
+ </p>
+ </li>
+
+ <li>
+ In order to support one-to-many SNAT (aka IP masquerading), where
+ multiple logical IP addresses spread across multiple chassis are
+ mapped to a single external IP address, it will be necessary to
+ handle some of the logical router processing on a specific chassis
+ in a centralized manner. Since the SNAT external IP address is
+ typically the distributed gateway port IP address, and for
+ simplicity, the same chassis associated with the distributed
+ gateway port is used.
+ </li>
+ </ul>
+
+ <p>
+ The details of flow restrictions to specific chassis are described
+ in the <code>ovn-northd</code> documentation.
+ </p>
+
+ <p>
+ While most of the physical location dependent aspects of distributed
+ gateway ports can be handled by restricting some flows to specific
+ chassis, one additional mechanism is required. When a packet
+ leaves the ingress pipeline and the logical egress port is the
+ distributed gateway port, one of two different sets of actions is
+ required at table 32:
+ </p>
+
+ <ul>
+ <li>
+ If the packet can be handled locally on the sender's hypervisor
+ (e.g. one-to-one NAT traffic), then the packet should just be
+ resubmitted locally to table 33, in the normal manner for
+ distributed logical patch ports.
+ </li>
+
+ <li>
+ However, if the packet needs to be handled on the chassis
+ associated with the distributed gateway port (e.g. one-to-many
+ SNAT traffic or non-NAT traffic), then table 32 must send the
+ packet on a tunnel port to that chassis.
+ </li>
+ </ul>
+
+ <p>
+ In order to trigger the second set of actions, the
+ MLF_FORCE_CHASSIS_REDIRECT flag is added. For port_bindings with
+ type "patch", when a "redirect-chassis" is specified, a flow is
+ added to table 32 that matches when the logical egress port is
+ the distributed gateway port and MLF_FORCE_CHASSIS_REDIRECT is
+ set. This flow sends the packet through a tunnel to the
+ "redirect-chassis", in the same way that table 32 directs packets
+ whose logical egress port is a VIF or a type "l3gateway" port to
+ different chassis. When the logical egress port is the
+ distributed gateway port and MLF_FORCE_CHASSIS_REDIRECT is
+ cleared, the packet will fall through to the table 32 priority 0
+ fallback flow and be resubmitted to table 33 locally.
+ </p>
+
<h2>Life Cycle of a VTEP gateway</h2>
<p>
@@ -1,7 +1,7 @@
{
"name": "OVN_Northbound",
- "version": "5.4.1",
- "cksum": "3485560318 13777",
+ "version": "5.5.0",
+ "cksum": "379266191 13990",
"tables": {
"NB_Global": {
"columns": {
@@ -191,6 +191,11 @@
"Logical_Router_Port": {
"columns": {
"name": {"type": "string"},
+ "options": {
+ "type": {"key": "string",
+ "value": "string",
+ "min": 0,
+ "max": "unlimited"}},
"networks": {"type": {"key": "string",
"min": 1,
"max": "unlimited"}},
@@ -1093,6 +1093,39 @@
port has all ingress and egress traffic dropped.
</column>
+ <group title="Options">
+ <p>
+ Additional options for the logical router port.
+ </p>
+
+ <column name="options" key="redirect-chassis">
+ <p>
+ If set, this indicates that this logical router port represents
+ a distributed gateway port that connects this router to a logical
+ switch with a localnet port. There may be at most one such
+ logical router port on each logical router.
+ </p>
+
+ <p>
+ Even when a <code>redirect-chassis</code> is specified, the
+ logical router port still effectively resides on each chassis.
+ However, due to the implications of the use of L2 learning in the
+ physical network, as well as the need to support advanced features
+ such as one-to-many NAT (aka IP masquerading), a subset of the
+ logical router processing is handled in a centralized manner on
+ the specified <code>redirect-chassis</code>.
+ </p>
+
+ <p>
+ When this option is specified, the peer logical switch port's
+ <ref column="addresses" table="Logical_Switch_Port"/> should be
+ set to <code>router</code>, so that the corresponding logical
+ switch destination lookup flow is only programmed on the
+ <code>redirect-chassis</code>.
+ </p>
+ </column>
+ </group>
+
<group title="Attachment">
<p>
A given router port serves one of two purposes:
@@ -832,7 +832,7 @@
<li><code>reg0</code>...<code>reg9</code></li>
<li><code>xxreg0</code> <code>xxreg1</code></li>
<li><code>inport</code> <code>outport</code></li>
- <li><code>flags.loopback</code></li>
+ <li><code>flags.loopback</code> <code>flags.force_chassis_redirect</code></li>
<li><code>eth.src</code> <code>eth.dst</code> <code>eth.type</code></li>
<li><code>vlan.tci</code> <code>vlan.vid</code> <code>vlan.pcp</code> <code>vlan.present</code></li>
<li><code>ip.proto</code> <code>ip.dscp</code> <code>ip.ecn</code> <code>ip.ttl</code> <code>ip.frag</code></li>
@@ -1707,6 +1707,19 @@ tcp.flags = RST;
This is populated by <code>ovn-controller</code> based on the value
of the <code>options:l2gateway-chassis</code> column in this table.
</dd>
+
+ <dt>patch</dt>
+ <dd>
+ The physical location of the centralized aspects of a patch port
+ representing a L3 distributed gateway port. Even when the chassis
+ is specified, the patch port still effectively resides on every
+ chassis. However, a subset of the logical router processing is
+ centralized at the patch port instance on the specified chassis.
+ To successfully identify a chassis, this column must be a
+ <ref table="Chassis"/> record. This is populated by
+ <code>ovn-controller</code> based on the value of the
+ <code>options:redirect-chassis</code> column in this table.
+ </dd>
</dl>
</column>
@@ -1808,6 +1821,18 @@ tcp.flags = RST;
ports must have reversed <ref column="logical_port"/> and
<code>peer</code> values.
</column>
+
+ <column name="options" key="redirect-chassis">
+ The <code>chassis</code> associated with this <code>patch</code> port,
+ if any. This is taken from <ref table="Logical_Router_Port"
+ column="options" key="redirect-chassis" db="OVN_Northbound"/>
+ in the OVN_Northbound database's <ref table="Logical_Router_Port"
+ db="OVN_Northbound"/> table. Even when a
+ <code>redirect-chassis</code> is specified, the <code>patch</code>
+ port still effectively resides on every hypervisor. However, a
+ subset of the logical router processing is centralized at the
+ <code>patch</code> port instance on the <code>redirect-chassis</code>.
+ </column>
</group>
<group title="L3 Gateway Options">
@@ -6148,3 +6148,329 @@ OVS_APP_EXIT_AND_WAIT([ovs-vswitchd])
OVS_APP_EXIT_AND_WAIT([ovsdb-server])
AT_CLEANUP
+
+AT_SETUP([ovn -- 1 LR with distributed router gateway port])
+AT_SKIP_IF([test $HAVE_PYTHON = no])
+ovn_start
+
+# Logical network:
+# One LR R1 that has switches foo (192.168.1.0/24) and
+# alice (172.16.1.0/24) connected to it. The logical port
+# between R1 and alice has a "redirect-chassis" specified,
+# i.e. it is the distributed router gateway port.
+# Switch alice also has a localnet port defined.
+# An additional switch outside has a localnet port and the
+# same subnet as alice (172.16.1.0/24).
+
+# Physical network:
+# Three hypervisors hv[123].
+# hv1 hosts vif foo1.
+# hv2 is the "redirect-chassis" that hosts the distributed
+# router gateway port.
+# hv3 hosts vif outside1.
+# In order to show that connectivity works only through hv2,
+# an initial round of tests is run without any bridge-mapping
+# defined for the localnet on hv2. These tests are expected
+# to fail.
+# Subsequent tests are run after defining the bridge-mapping
+# for the localnet on hv2. These tests are expected to succeed.
+
+# Create three hypervisors and create OVS ports corresponding
+to logical ports.
+net_add n1
+
+sim_add hv1
+as hv1
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.1
+ovs-vsctl -- add-port br-int hv1-vif1 -- \
+ set interface hv1-vif1 external-ids:iface-id=foo1 \
+ options:tx_pcap=hv1/vif1-tx.pcap \
+ options:rxq_pcap=hv1/vif1-rx.pcap \
+ ofport-request=1
+
+sim_add hv2
+as hv2
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.2
+
+sim_add hv3
+as hv3
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.3
+ovs-vsctl -- add-port br-int hv3-vif1 -- \
+ set interface hv3-vif1 external-ids:iface-id=outside1 \
+ options:tx_pcap=hv3/vif1-tx.pcap \
+ options:rxq_pcap=hv3/vif1-rx.pcap \
+ ofport-request=1
+
+# Pre-populate the hypervisors' ARP tables so that we don't lose any
+# packets for ARP resolution (native tunneling doesn't queue packets
+# for ARP resolution).
+ovn_populate_arp
+
+ovn-nbctl create Logical_Router name=R1
+
+ovn-nbctl ls-add foo
+ovn-nbctl ls-add alice
+ovn-nbctl ls-add outside
+
+# Connect foo to R1
+ovn-nbctl lrp-add R1 foo 00:00:01:01:02:03 192.168.1.1/24
+ovn-nbctl lsp-add foo rp-foo -- set Logical_Switch_Port rp-foo \
+ type=router options:router-port=foo \
+ -- lsp-set-addresses rp-foo router
+
+# Connect alice to R1 as distributed router gateway port on hv2
+ovn-nbctl lrp-add R1 alice 00:00:02:01:02:03 172.16.1.1/24 \
+ -- set Logical_Router_Port alice options:redirect-chassis="hv2"
+ovn-nbctl lsp-add alice rp-alice -- set Logical_Switch_Port rp-alice \
+ type=router options:router-port=alice \
+ -- lsp-set-addresses rp-alice router
+
+# Create logical port foo1 in foo
+ovn-nbctl lsp-add foo foo1 \
+-- lsp-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2"
+
+# Create logical port outside1 in outside
+ovn-nbctl lsp-add outside outside1 \
+-- lsp-set-addresses outside1 "f0:00:00:01:02:04 172.16.1.3"
+
+# Create localnet port in alice
+ovn-nbctl lsp-add alice ln-alice
+ovn-nbctl lsp-set-addresses ln-alice unknown
+ovn-nbctl lsp-set-type ln-alice localnet
+ovn-nbctl lsp-set-options ln-alice network_name=phys
+
+# Create localnet port in outside
+ovn-nbctl lsp-add outside ln-outside
+ovn-nbctl lsp-set-addresses ln-outside unknown
+ovn-nbctl lsp-set-type ln-outside localnet
+ovn-nbctl lsp-set-options ln-outside network_name=phys
+
+# Create bridge-mappings on hv1 and hv3, leaving hv2 for later
+as hv1 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
+as hv3 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
+
+
+# Allow some time for ovn-northd and ovn-controller to catch up.
+# XXX This should be more systematic.
+sleep 2
+
+echo "---------NB dump-----"
+ovn-nbctl show
+echo "---------------------"
+ovn-nbctl list logical_router
+echo "---------------------"
+ovn-nbctl list logical_router_port
+echo "---------------------"
+
+echo "---------SB dump-----"
+ovn-sbctl list datapath_binding
+echo "---------------------"
+ovn-sbctl list port_binding
+echo "---------------------"
+ovn-sbctl dump-flows
+echo "---------------------"
+ovn-sbctl list chassis
+ovn-sbctl list encap
+echo "---------------------"
+
+echo "------ hv1 dump ----------"
+as hv1 ovs-ofctl show br-int
+as hv1 ovs-ofctl dump-flows br-int
+echo "------ hv2 dump ----------"
+as hv2 ovs-ofctl show br-int
+as hv2 ovs-ofctl dump-flows br-int
+echo "------ hv3 dump ----------"
+as hv3 ovs-ofctl show br-int
+as hv3 ovs-ofctl dump-flows br-int
+echo "--------------------------"
+
+# Check that hv1 has an entry in table 32 sending some distributed gateway
+# port traffic through a tunnel
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=32 | grep =0x2,metadata=0x1 | grep output | wc -l], [0], [1
+])
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int table=32 | grep =0x2,metadata=0x1 | wc -l], [0], [0
+])
+# Check that arp reply on distributed gateway port is only programmed on hv2
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | grep arp | grep =0x2,metadata=0x1 | wc -l], [0], [0
+])
+AT_CHECK([as hv2 ovs-ofctl dump-flows br-int | grep arp | grep =0x2,metadata=0x1 | wc -l], [0], [1
+])
+
+
+ip_to_hex() {
+ printf "%02x%02x%02x%02x" "$@"
+}
+
+
+: > hv2-vif1.expected
+: > hv3-vif1.expected
+
+# test_arp INPORT SHA SPA TPA [REPLY_HA]
+#
+# Causes a packet to be received on INPORT. The packet is an ARP
+# request with SHA, SPA, and TPA as specified. If REPLY_HA is provided, then
+# it should be the hardware address of the target to expect to receive in an
+# ARP reply; otherwise no reply is expected.
+#
+# INPORT is an logical switch port number, e.g. 11 for vif11.
+# SHA and REPLY_HA are each 12 hex digits.
+# SPA and TPA are each 8 hex digits.
+test_arp() {
+ local hv=$1 inport=$2 sha=$3 spa=$4 tpa=$5 reply_ha=$6
+ local request=ffffffffffff${sha}08060001080006040001${sha}${spa}ffffffffffff${tpa}
+ as hv$hv ovs-appctl netdev-dummy/receive hv${hv}-vif$inport $request
+
+ if test X$reply_ha != X; then
+ # Expect to receive the reply, if any.
+ local reply=${sha}${reply_ha}08060001080006040002${reply_ha}${tpa}${sha}${spa}
+ echo $reply >> hv${hv}-vif$inport.expected
+ fi
+}
+
+rtr_ip=$(ip_to_hex 172 16 1 1)
+foo_ip=$(ip_to_hex 192 168 1 2)
+outside_ip=$(ip_to_hex 172 16 1 3)
+
+echo $rtr_ip
+echo $foo_ip
+echo $outside_ip
+
+# ARP for router IP address from outside1, no response expected
+test_arp 3 1 f00000010204 $outside_ip $rtr_ip
+
+# Now check the packets actually received against the ones expected.
+OVN_CHECK_PACKETS([hv3/vif1-tx.pcap], [hv3-vif1.expected])
+
+# Send ip packet between foo1 and outside1
+src_mac="f00000010203"
+dst_mac="000001010203"
+src_ip=`ip_to_hex 192 168 1 2`
+dst_ip=`ip_to_hex 172 16 1 3`
+packet=${dst_mac}${src_mac}08004500001c0000000040110000${src_ip}${dst_ip}0035111100080000
+
+as hv1 ovs-appctl netdev-dummy/receive hv1-vif1 $packet
+
+# Now check the packets actually received against the ones expected.
+OVN_CHECK_PACKETS([hv3/vif1-tx.pcap], [hv3-vif1.expected])
+
+# Now add bridge-mappings on hv2, which should make everything work
+as hv2 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
+
+# Allow some time for ovn-northd and ovn-controller to catch up.
+# XXX This should be more systematic.
+sleep 2
+
+# ARP for router IP address from outside1
+test_arp 3 1 f00000010204 $outside_ip $rtr_ip 000002010203
+
+# Now check the packets actually received against the ones expected.
+OVN_CHECK_PACKETS([hv3/vif1-tx.pcap], [hv3-vif1.expected])
+
+# ARP request packet to expect at outside1
+src_mac="000002010203"
+src_ip=`ip_to_hex 172 16 1 1`
+arp_request=ffffffffffff${src_mac}08060001080006040001${src_mac}${src_ip}000000000000${dst_ip}
+
+# Resend packet from foo1 to outside1
+as hv1 ovs-appctl netdev-dummy/receive hv1-vif1 $packet
+
+echo $arp_request >> hv3-vif1.expected
+OVN_CHECK_PACKETS([hv3/vif1-tx.pcap], [hv3-vif1.expected])
+
+# Send ARP reply from outside1 back to the router
+reply_mac="f00000010204"
+arp_reply=${src_mac}${reply_mac}08060001080006040002${reply_mac}${dst_ip}${src_mac}${src_ip}
+
+as hv3 ovs-appctl netdev-dummy/receive hv3-vif1 $arp_reply
+
+# Allow some time for ovn-northd and ovn-controller to catch up.
+# XXX This should be more systematic.
+sleep 1
+
+# Packet to Expect at outside1
+src_mac="000002010203"
+dst_mac="f00000010204"
+src_ip=`ip_to_hex 192 168 1 2`
+dst_ip=`ip_to_hex 172 16 1 3`
+expected=${dst_mac}${src_mac}08004500001c000000003f110100${src_ip}${dst_ip}0035111100080000
+
+# Resend packet from foo1 to outside1
+as hv1 ovs-appctl netdev-dummy/receive hv1-vif1 $packet
+
+echo "------ hv1 dump ----------"
+as hv1 ovs-ofctl show br-int
+as hv1 ovs-ofctl dump-flows br-int
+echo "------ hv2 dump ----------"
+as hv2 ovs-ofctl show br-int
+as hv2 ovs-ofctl dump-flows br-int
+echo "------ hv3 dump ----------"
+as hv3 ovs-ofctl show br-int
+as hv3 ovs-ofctl dump-flows br-int
+echo "----------------------------"
+
+echo $expected >> hv3-vif1.expected
+OVN_CHECK_PACKETS([hv3/vif1-tx.pcap], [hv3-vif1.expected])
+
+#Check ovn-trace over distributed gateway port
+AT_CAPTURE_FILE([trace])
+ovn_trace () {
+ ovn-trace --all "$@" | tee trace | sed '1,/Minimal trace/d'
+}
+
+echo 'ip.ttl--;' > expout
+echo 'eth.src = 00:00:02:01:02:03;' >> expout
+echo 'eth.dst = f0:00:00:01:02:04;' >> expout
+echo 'output("ln-alice");' >> expout
+AT_CHECK_UNQUOTED([ovn_trace foo 'inport == "foo1" && eth.src == f0:00:00:01:02:03 && eth.dst == 00:00:01:01:02:03 && ip4.src == 192.168.1.2 && ip4.dst == 172.16.1.3 && ip.ttl == 0xff'], [0], [expout])
+
+# Create logical port alice1 in alice on hv1
+as hv1 ovs-vsctl -- add-port br-int hv1-vif2 -- \
+ set interface hv1-vif2 external-ids:iface-id=alice1 \
+ options:tx_pcap=hv1/vif2-tx.pcap \
+ options:rxq_pcap=hv1/vif2-rx.pcap \
+ ofport-request=1
+
+ovn-nbctl lsp-add alice alice1 \
+-- lsp-set-addresses alice1 "f0:00:00:01:02:05 172.16.1.4"
+
+# Create logical port foo2 in foo on hv2
+as hv2 ovs-vsctl -- add-port br-int hv2-vif1 -- \
+ set interface hv2-vif1 external-ids:iface-id=foo2 \
+ options:tx_pcap=hv2/vif1-tx.pcap \
+ options:rxq_pcap=hv2/vif1-rx.pcap \
+ ofport-request=1
+
+ovn-nbctl lsp-add foo foo2 \
+-- lsp-set-addresses foo2 "f0:00:00:01:02:06 192.168.1.3"
+
+# Allow some time for ovn-northd and ovn-controller to catch up.
+# XXX This should be more systematic.
+sleep 1
+
+: > hv1-vif2.expected
+
+# Send ip packet between alice1 and foo2
+src_mac="f00000010205"
+dst_mac="000002010203"
+src_ip=`ip_to_hex 172 16 1 4`
+dst_ip=`ip_to_hex 192 168 1 3`
+packet=${dst_mac}${src_mac}08004500001c0000000040110000${src_ip}${dst_ip}0035111100080000
+
+as hv1 ovs-appctl netdev-dummy/receive hv1-vif2 $packet
+
+# Packet to Expect at foo2
+src_mac="000001010203"
+dst_mac="f00000010206"
+src_ip=`ip_to_hex 172 16 1 4`
+dst_ip=`ip_to_hex 192 168 1 3`
+expected=${dst_mac}${src_mac}08004500001c000000003f110100${src_ip}${dst_ip}0035111100080000
+
+echo $expected >> hv2-vif1.expected
+OVN_CHECK_PACKETS([hv2/vif1-tx.pcap], [hv2-vif1.expected])
+
+OVN_CLEANUP([hv1],[hv2],[hv3])
+
+AT_CLEANUP
Currently OVN distributed logical routers achieve reachability to physical networks by passing through a "join" logical switch to a centralized gateway router, which then connects to another logical switch that has a localnet port connecting to the physical network. This patch adds logical port and port binding abstractions that allow an OVN distributed logical router to connect directly to a logical switch that has a localnet port connecting to the physical network. In this patch, this logical router port is called a "distributed gateway port". The primary design goal of distributed gateway ports is to allow as much traffic as possible to be handled locally on the hypervisor where a VM or container resides. Whenever possible, packets from the VM or container to the outside world should be processed completely on that VM's or container's hypervisor, eventually traversing a localnet port instance on that hypervisor to the physical network. Whenever possible, packets from the outside world to a VM or container should be directed through the physical network directly to the VM's or container's hypervisor, where the packet will enter the integration bridge through a localnet port. However, due to the implications of the use of L2 learning in the physical network, as well as the need to support advanced features such as one-to-many NAT (aka IP masquerading), where multiple logical IP addresses spread across multiple chassis are mapped to one external IP address, it will be necessary to handle some of the logical router processing on a specific chassis in a centralized manner. For this reason, the user must associate a "redirect-chassis" with each distributed gateway port. In order to allow for the distributed processing of some packets, distributed gateway ports need to be logical patch ports that effectively reside on every hypervisor, rather than "l3gateway" ports that are bound to a particular chassis. However, the flows associated with distributed gateway ports often need to be associated with physical locations. This is implemented in this patch (and subsequent patches) by adding "is_chassis_resident()" match conditions to several logical router flows. While most of the physical location dependent aspects of distributed gateway ports can be handled by restricting some flows to specific chassis, one additional mechanism is required. When a packet leaves the ingress pipeline and the logical egress port is the distributed gateway port, one of two different sets of actions is required at table 32: - If the packet can be handled locally on the sender's hypervisor (e.g. one-to-one NAT traffic), then the packet should just be resubmitted locally to table 33, in the normal manner for distributed logical patch ports. - However, if the packet needs to be handled on the chassis associated with the distributed gateway port (e.g. one-to-many SNAT traffic or non-NAT traffic), then table 32 must send the packet on a tunnel port to that chassis. In order to trigger the second set of actions, the MLF_FORCE_CHASSIS_REDIRECT flag is added. For port_bindings with type "patch", when a "redirect-chassis" is specified, a flow is added to table 32 that matches when the logical egress port is the distributed gateway port and MLF_FORCE_CHASSIS_REDIRECT is set. This flow sends the packet through a tunnel to the "redirect-chassis", in the same way that table 32 directs packets whose logical egress port is a VIF or a type "l3gateway" port to different chassis. When the logical egress port is the distributed gateway port and MLF_FORCE_CHASSIS_REDIRECT is cleared, the packet will fall through to the table 32 priority 0 fallback flow and be resubmitted to table 33 locally. A port_binding of type "patch" is associated with a chassis in a similar manner as a "l3gateway" port. However, unlike "l3gateway" ports, "patch" ports are effectively resident on each hypervisor (subject to conditional monitoring constraints) even when there is a "redirect-chassis" specified. The effect of associating a "redirect-chassis" with a logical router port is to cause the additional table 32 flow to be created (through the southbound port_binding), and to restrict some flows to the "redirect-chassis" through "is_chassis_resident()" match conditions. Signed-off-by: Mickey Spiegel <mickeys.dev@gmail.com> --- ovn/controller/binding.c | 8 + ovn/controller/ovn-controller.c | 4 + ovn/controller/physical.c | 34 +++++ ovn/lib/logical-fields.c | 4 + ovn/lib/logical-fields.h | 6 + ovn/northd/ovn-northd.8.xml | 94 +++++++++++- ovn/northd/ovn-northd.c | 123 ++++++++++++++- ovn/ovn-architecture.7.xml | 128 +++++++++++++++- ovn/ovn-nb.ovsschema | 9 +- ovn/ovn-nb.xml | 33 ++++ ovn/ovn-sb.xml | 27 +++- tests/ovn.at | 326 ++++++++++++++++++++++++++++++++++++++++ 12 files changed, 780 insertions(+), 16 deletions(-)