[ovs-dev,RFC,1/8] system-traffic: Load balancing.

diff --git a/lib/ofp-actions.c b/lib/ofp-actions.c
index becf02d..8f6e02e 100644
--- a/lib/ofp-actions.c
+++ b/lib/ofp-actions.c
@@ -5913,7 +5913,8 @@  ofpacts_execute_action_set(struct ofpbuf *action_list,
      * not be sent anywhere. */
     if (!ofpacts_copy_last(action_list, action_set, OFPACT_GROUP) &&
         !ofpacts_copy_last(action_list, action_set, OFPACT_OUTPUT) &&
-        !ofpacts_copy_last(action_list, action_set, OFPACT_RESUBMIT)) {
+        !ofpacts_copy_last(action_list, action_set, OFPACT_RESUBMIT) &&
+        !ofpacts_copy_last(action_list, action_set, OFPACT_CT)) {
         ofpbuf_clear(action_list);
     }
 }
diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
index 57d877f..ce7d8e1 100644
--- a/ofproto/ofproto-dpif-xlate.c
+++ b/ofproto/ofproto-dpif-xlate.c
@@ -3293,18 +3293,37 @@  xlate_group_stats(struct xlate_ctx *ctx, struct group_dpif *group,
 }
 
 static void
+group_report_valist(struct xlate_in *xin OVS_UNUSED, int recurse,
+                    const char *format, va_list args)
+{
+    struct ds ds = DS_EMPTY_INITIALIZER;
+
+    ds_put_char_multiple(&ds, ' ', recurse);
+    ds_put_format_valist(&ds, format, args);
+    VLOG_INFO("Group execution: %s", ds_cstr(&ds));
+    ds_destroy(&ds);
+}
+
+static void
 xlate_group_bucket(struct xlate_ctx *ctx, struct ofputil_bucket *bucket)
 {
     uint64_t action_list_stub[1024 / 8];
     struct ofpbuf action_list, action_set;
     struct flow old_flow = ctx->xin->flow;
     bool old_was_mpls = ctx->was_mpls;
+    void (*old_report_hook)(struct xlate_in *, int recurse,
+                            const char *format, va_list args) = ctx->xin->report_hook;
+    ctx->xin->report_hook = group_report_valist;
 
+    xlate_report_actions(ctx, "Considering bucket actions: ",
+                         bucket->ofpacts, bucket->ofpacts_len);
     ofpbuf_use_const(&action_set, bucket->ofpacts, bucket->ofpacts_len);
     ofpbuf_use_stub(&action_list, action_list_stub, sizeof action_list_stub);
 
     ofpacts_execute_action_set(&action_list, &action_set);
     ctx->recurse++;
+    xlate_report_actions(ctx, "Executing group actions: ",
+                         action_list.data, action_list.size);
     do_xlate_actions(action_list.data, action_list.size, ctx);
     ctx->recurse--;
 
@@ -3342,6 +3361,8 @@  xlate_group_bucket(struct xlate_ctx *ctx, struct ofputil_bucket *bucket)
      * actions after the group action must continue processing with the
      * original, not the recirculated packet! */
     ctx->exit = false;
+
+    ctx->xin->report_hook = old_report_hook;
 }
 
 static void
diff --git a/tests/system-traffic.at b/tests/system-traffic.at
index 74ca0a0..150009a 100644
--- a/tests/system-traffic.at
+++ b/tests/system-traffic.at
@@ -1956,3 +1956,228 @@  tcp,orig=(src=fc00::2,dst=fc00::240,sport=<cleared>,dport=<cleared>),reply=(src=
 
 OVS_TRAFFIC_VSWITCHD_STOP
 AT_CLEANUP
+
+
+AT_SETUP([conntrack - DNAT load balancing])
+CHECK_CONNTRACK()
+OVS_TRAFFIC_VSWITCHD_START()
+
+ADD_NAMESPACES(at_ns1, at_ns2, at_ns3, at_ns4)
+
+ADD_VETH(p1, at_ns1, br0, "10.1.1.1/24")
+ADD_VETH(p2, at_ns2, br0, "10.1.1.2/24")
+ADD_VETH(p3, at_ns3, br0, "10.1.1.3/24")
+ADD_VETH(p4, at_ns4, br0, "10.1.1.4/24")
+NS_CHECK_EXEC([at_ns1], [ip link set dev p1 address 80:88:88:88:88:11])
+NS_CHECK_EXEC([at_ns2], [ip link set dev p2 address 80:88:88:88:88:22])
+NS_CHECK_EXEC([at_ns3], [ip link set dev p3 address 80:88:88:88:88:33])
+NS_CHECK_EXEC([at_ns4], [ip link set dev p4 address 80:88:88:88:88:44])
+
+dnl Select group for load balancing.  One bucket per server.  Each bucket
+dnl tracks and NATs the connection and recirculates to table 4 for egress
+dnl routing.  Packets of existing connections are always NATted based on
+dnl connection state, only new connections are NATted according to the
+dnl specific NAT parameters in each bucket.
+AT_CHECK([ovs-ofctl -O OpenFlow15 -vwarn add-group br0 "group_id=234,type=select,bucket=weight=100,ct(nat(dst=10.1.1.2),commit,table=4),bucket=weight=100,ct(nat(dst=10.1.1.3),commit,table=4),bucket=weight=100,ct(nat(dst=10.1.1.4),commit,table=4)"])
+
+AT_DATA([flows.txt], [dnl
+dnl Track connections to the virtual IP address.
+table=0 priority=100 ip nw_dst=10.1.1.64 action=group:234
+dnl All other IP traffic is allowed but the connection state is no commited.
+table=0 priority=90 ip action=ct(table=4,nat)
+dnl
+dnl Allow ARP, but generate responses for virtual addresses
+table=0 priority=100 arp arp_op=1 action=move:OXM_OF_ARP_TPA[[]]->NXM_NX_REG2[[]],resubmit(,8),goto_table:10
+table=0 priority=10 arp action=normal
+table=0 priority=0 action=drop
+dnl
+dnl Routing table
+dnl
+table=4,ip,nw_dst=10.1.1.1 action=mod_dl_dst:80:88:88:88:88:11,output:1
+table=4,ip,nw_dst=10.1.1.2 action=mod_dl_dst:80:88:88:88:88:22,output:2
+table=4,ip,nw_dst=10.1.1.3 action=mod_dl_dst:80:88:88:88:88:33,output:3
+table=4,ip,nw_dst=10.1.1.4 action=mod_dl_dst:80:88:88:88:88:44,output:4
+table=4 priority=0 action=drop
+dnl
+dnl MAC resolution table for IP in reg2, stores mac in OXM_OF_PKT_REG0
+table=8,reg2=0x0a010140,action=load:0x808888888888->OXM_OF_PKT_REG0[[]]
+dnl Zero result means not found.
+table=8,priority=0,action=load:0->OXM_OF_PKT_REG0[[]]
+dnl ARP responder mac filled in at OXM_OF_PKT_REG0, or 0 for normal action.
+dnl TPA IP in reg2.
+table=10 priority=100 arp xreg0=0 action=normal
+dnl Swaps the fields of the ARP message to turn a query to a response.
+table=10 priority=10,arp,arp_op=1,action=load:2->OXM_OF_ARP_OP[[]],move:OXM_OF_ARP_SHA[[]]->OXM_OF_ARP_THA[[]],move:OXM_OF_PKT_REG0[[0..47]]->OXM_OF_ARP_SHA[[]],move:OXM_OF_ARP_SPA[[]]->OXM_OF_ARP_TPA[[]],move:NXM_NX_REG2[[]]->OXM_OF_ARP_SPA[[]],move:NXM_OF_ETH_SRC[[]]->NXM_OF_ETH_DST[[]],move:OXM_OF_PKT_REG0[[0..47]]->NXM_OF_ETH_SRC[[]],move:NXM_OF_IN_PORT[[]]->NXM_NX_REG3[[0..15]],load:0->NXM_OF_IN_PORT[[]],output:NXM_NX_REG3[[0..15]]
+table=10 priority=0 action=controller
+])
+
+AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt])
+
+dnl Start web servers
+NETNS_DAEMONIZE([at_ns2], [[$PYTHON $srcdir/test-l7.py]], [http2.pid])
+NETNS_DAEMONIZE([at_ns3], [[$PYTHON $srcdir/test-l7.py]], [http3.pid])
+NETNS_DAEMONIZE([at_ns4], [[$PYTHON $srcdir/test-l7.py]], [http4.pid])
+
+on_exit 'ovs-ofctl -O OpenFlow15 dump-flows br0'
+on_exit 'ovs-appctl revalidator/purge'
+on_exit 'ovs-appctl dpif/dump-flows br0'
+
+dnl Should work with the virtual IP address through NAT
+for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
+    echo Request $i
+    NS_CHECK_EXEC([at_ns1], [wget 10.1.1.64 -t 5 -T 1 --retry-connrefused -v -o wget$i.log])
+done
+
+dnl Each server should have at least one connection.
+AT_CHECK([conntrack -L 2>&1 | FORMAT_CT(10.1.1.64) | sort], [0], [dnl
+TIME_WAIT src=10.1.1.1 dst=10.1.1.64 sport=<cleared> dport=<cleared> src=10.1.1.2 dst=10.1.1.1 sport=<cleared> dport=<cleared> [[ASSURED]] mark=0 use=1
+TIME_WAIT src=10.1.1.1 dst=10.1.1.64 sport=<cleared> dport=<cleared> src=10.1.1.3 dst=10.1.1.1 sport=<cleared> dport=<cleared> [[ASSURED]] mark=0 use=1
+TIME_WAIT src=10.1.1.1 dst=10.1.1.64 sport=<cleared> dport=<cleared> src=10.1.1.4 dst=10.1.1.1 sport=<cleared> dport=<cleared> [[ASSURED]] mark=0 use=1
+])
+
+ovs-appctl dpif/dump-flows br0
+ovs-appctl revalidator/purge
+ovs-ofctl -O OpenFlow15 dump-flows br0
+ovs-ofctl -O OpenFlow15 dump-group-stats br0
+
+OVS_TRAFFIC_VSWITCHD_STOP
+AT_CLEANUP
+
+
+AT_SETUP([conntrack - DNAT load balancing with NC])
+CHECK_CONNTRACK()
+OVS_TRAFFIC_VSWITCHD_START()
+
+ADD_NAMESPACES(at_ns1, at_ns2, at_ns3, at_ns4, at_ns5)
+
+ADD_VETH(p1, at_ns1, br0, "10.1.1.1/24")
+ADD_VETH(p2, at_ns2, br0, "10.1.1.2/24")
+ADD_VETH(p3, at_ns3, br0, "10.1.1.3/24")
+ADD_VETH(p4, at_ns4, br0, "10.1.1.4/24")
+ADD_VETH(p5, at_ns5, br0, "10.1.1.5/24")
+NS_CHECK_EXEC([at_ns1], [ip link set dev p1 address 80:88:88:88:88:11])
+NS_CHECK_EXEC([at_ns2], [ip link set dev p2 address 80:88:88:88:88:22])
+NS_CHECK_EXEC([at_ns3], [ip link set dev p3 address 80:88:88:88:88:33])
+NS_CHECK_EXEC([at_ns4], [ip link set dev p4 address 80:88:88:88:88:44])
+NS_CHECK_EXEC([at_ns5], [ip link set dev p5 address 80:88:88:88:88:55])
+
+dnl Select group for load balancing.  One bucket per server.  Each bucket
+dnl tracks and NATs the connection and recirculates to table 4 for egress
+dnl routing.  Packets of existing connections are always NATted based on
+dnl connection state, only new connections are NATted according to the
+dnl specific NAT parameters in each bucket.
+AT_CHECK([ovs-ofctl -O OpenFlow15 -vwarn add-group br0 "group_id=234,type=select,bucket=weight=100,ct(nat(dst=10.1.1.2),commit,table=4),bucket=weight=100,ct(nat(dst=10.1.1.3),commit,table=4),bucket=weight=100,ct(nat(dst=10.1.1.4),commit,table=4)"])
+
+AT_DATA([flows.txt], [dnl
+dnl Track connections to the virtual IP address.
+table=0 priority=100 ip nw_dst=10.1.1.64 action=group:234
+dnl All other IP traffic is allowed but the connection state is no commited.
+table=0 priority=90 ip action=ct(table=4,nat)
+dnl
+dnl Allow ARP, but generate responses for virtual addresses
+table=0 priority=100 arp arp_op=1 action=move:OXM_OF_ARP_TPA[[]]->NXM_NX_REG2[[]],resubmit(,8),goto_table:10
+table=0 priority=10 arp action=normal
+table=0 priority=0 action=drop
+dnl
+dnl Routing table
+dnl
+table=4,ip,nw_dst=10.1.1.1 action=mod_dl_dst:80:88:88:88:88:11,output:1
+table=4,ip,nw_dst=10.1.1.2 action=mod_dl_dst:80:88:88:88:88:22,output:2
+table=4,ip,nw_dst=10.1.1.3 action=mod_dl_dst:80:88:88:88:88:33,output:3
+table=4,ip,nw_dst=10.1.1.4 action=mod_dl_dst:80:88:88:88:88:44,output:4
+table=4,ip,nw_dst=10.1.1.5 action=mod_dl_dst:80:88:88:88:88:55,output:5
+table=4 priority=0 action=drop
+dnl
+dnl MAC resolution table for IP in reg2, stores mac in OXM_OF_PKT_REG0
+table=8,reg2=0x0a010140,action=load:0x808888888888->OXM_OF_PKT_REG0[[]]
+dnl Zero result means not found.
+table=8,priority=0,action=load:0->OXM_OF_PKT_REG0[[]]
+dnl ARP responder mac filled in at OXM_OF_PKT_REG0, or 0 for normal action.
+dnl TPA IP in reg2.
+table=10 priority=100 arp xreg0=0 action=normal
+dnl Swaps the fields of the ARP message to turn a query to a response.
+table=10 priority=10,arp,arp_op=1,action=load:2->OXM_OF_ARP_OP[[]],move:OXM_OF_ARP_SHA[[]]->OXM_OF_ARP_THA[[]],move:OXM_OF_PKT_REG0[[0..47]]->OXM_OF_ARP_SHA[[]],move:OXM_OF_ARP_SPA[[]]->OXM_OF_ARP_TPA[[]],move:NXM_NX_REG2[[]]->OXM_OF_ARP_SPA[[]],move:NXM_OF_ETH_SRC[[]]->NXM_OF_ETH_DST[[]],move:OXM_OF_PKT_REG0[[0..47]]->NXM_OF_ETH_SRC[[]],move:NXM_OF_IN_PORT[[]]->NXM_NX_REG3[[0..15]],load:0->NXM_OF_IN_PORT[[]],output:NXM_NX_REG3[[0..15]]
+table=10 priority=0 action=controller
+])
+
+AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt])
+
+dnl Start web servers
+NETNS_DAEMONIZE([at_ns2], [[$PYTHON $srcdir/test-l7.py]], [http2.pid])
+NETNS_DAEMONIZE([at_ns3], [[$PYTHON $srcdir/test-l7.py]], [http3.pid])
+NETNS_DAEMONIZE([at_ns4], [[$PYTHON $srcdir/test-l7.py]], [http4.pid])
+
+on_exit 'ovs-ofctl -O OpenFlow15 dump-flows br0'
+on_exit 'ovs-appctl revalidator/purge'
+on_exit 'ovs-appctl dpif/dump-flows br0'
+
+sleep 5
+
+dnl Should work with the virtual IP address through NAT
+for i in 1 2 3 4 5 6 7 8 9; do
+    echo Request $i
+    NS_CHECK_EXEC([at_ns1], [echo "TEST1" | nc -p 4100$i 10.1.1.64 80 > nc-1-$i.log])
+    NS_CHECK_EXEC([at_ns5], [echo "TEST5" | nc -p 4100$i 10.1.1.64 80 > nc-5-$i.log])
+done
+
+conntrack -L 2>&1
+
+ovs-appctl dpif/dump-flows br0
+ovs-appctl revalidator/purge
+ovs-ofctl -O OpenFlow15 dump-flows br0
+ovs-ofctl -O OpenFlow15 dump-group-stats br0
+
+OVS_TRAFFIC_VSWITCHD_STOP
+AT_CLEANUP
+
+
+AT_SETUP([conntrack - SNAT with overlapping source ports])
+CHECK_CONNTRACK()
+OVS_TRAFFIC_VSWITCHD_START()
+
+ADD_NAMESPACES(at_ns0, at_ns1)
+
+ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24")
+NS_CHECK_EXEC([at_ns0], [ip link set dev p0 address 80:88:88:88:88:88])
+NS_CHECK_EXEC([at_ns0], [ip addr add 10.1.1.11/24 dev p0])
+ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24")
+
+dnl Allow any traffic from ns0->ns1. Only allow nd, return traffic from ns1->ns0.
+AT_DATA([flows.txt], [dnl
+in_port=1,ip,action=ct(commit,zone=1,nat(src=10.1.1.240)),2
+in_port=2,ct_state=-trk,ip,action=ct(table=0,zone=1,nat)
+in_port=2,ct_state=+trk,ct_zone=1,ip,action=1
+dnl
+dnl ARP
+priority=100 arp arp_op=1 action=move:OXM_OF_ARP_TPA[[]]->NXM_NX_REG2[[]],resubmit(,8),goto_table:10
+priority=10 arp action=normal
+priority=0,action=drop
+dnl
+dnl MAC resolution table for IP in reg2, stores mac in OXM_OF_PKT_REG0
+table=8,reg2=0x0a0101f0/0xfffffff0,action=load:0x808888888888->OXM_OF_PKT_REG0[[]]
+table=8,priority=0,action=load:0->OXM_OF_PKT_REG0[[]]
+dnl ARP responder mac filled in at OXM_OF_PKT_REG0, or 0 for normal action.
+dnl TPA IP in reg2.
+dnl Swaps the fields of the ARP message to turn a query to a response.
+table=10 priority=100 arp xreg0=0 action=normal
+table=10 priority=10,arp,arp_op=1,action=load:2->OXM_OF_ARP_OP[[]],move:OXM_OF_ARP_SHA[[]]->OXM_OF_ARP_THA[[]],move:OXM_OF_PKT_REG0[[0..47]]->OXM_OF_ARP_SHA[[]],move:OXM_OF_ARP_SPA[[]]->OXM_OF_ARP_TPA[[]],move:NXM_NX_REG2[[]]->OXM_OF_ARP_SPA[[]],move:NXM_OF_ETH_SRC[[]]->NXM_OF_ETH_DST[[]],move:OXM_OF_PKT_REG0[[0..47]]->NXM_OF_ETH_SRC[[]],move:NXM_OF_IN_PORT[[]]->NXM_NX_REG3[[0..15]],load:0->NXM_OF_IN_PORT[[]],output:NXM_NX_REG3[[0..15]]
+table=10 priority=0 action=drop
+])
+
+AT_CHECK([ovs-ofctl --bundle add-flows br0 flows.txt])
+
+dnl HTTP requests from p0->p1 should work fine.
+NETNS_DAEMONIZE([at_ns1], [[$PYTHON $srcdir/test-l7.py]], [http0.pid])
+
+NS_CHECK_EXEC([at_ns0], [wget --bind-address 10.1.1.11 10.1.1.2 -t 5 -T 1 --retry-connrefused -v -o wget0.log])
+NS_CHECK_EXEC([at_ns0], [echo "TEST1" | nc -p 41001 -s 10.1.1.1 10.1.1.2 80 > nc-1.log])
+NS_CHECK_EXEC([at_ns0], [echo "TEST1" | nc -p 41001 -s 10.1.1.11 10.1.1.2 80 > nc-11.log])
+
+conntrack -L 2>&1
+
+AT_CHECK([conntrack -L 2>&1 | FORMAT_CT(10.1.1.2) | sed -e 's/dst=10.1.1.2[[45]][[0-9]]/dst=10.1.1.2XX/'], [0], [dnl
+TIME_WAIT src=10.1.1.11 dst=10.1.1.2 sport=<cleared> dport=<cleared> src=10.1.1.2 dst=10.1.1.2XX sport=<cleared> dport=<cleared> [[ASSURED]] mark=0 zone=1 use=1
+])
+
+OVS_TRAFFIC_VSWITCHD_STOP
+AT_CLEANUP