[ovs-dev,RFC] dp-packet-gso: Add GSO support.

diff --git a/lib/automake.mk b/lib/automake.mk
index 39afbff9d1a0..57f504d52f5c 100644
--- a/lib/automake.mk
+++ b/lib/automake.mk
@@ -104,6 +104,8 @@  lib_libopenvswitch_la_SOURCES = \
 	lib/dpctl.h \
 	lib/dp-packet.h \
 	lib/dp-packet.c \
+	lib/dp-packet-gso.h \
+	lib/dp-packet-gso.c \
 	lib/dpdk.h \
 	lib/dpif-netdev-lookup.h \
 	lib/dpif-netdev-lookup.c \
diff --git a/lib/dp-packet-gso.c b/lib/dp-packet-gso.c
new file mode 100644
index 000000000000..5ae7c88298a5
--- /dev/null
+++ b/lib/dp-packet-gso.c
@@ -0,0 +1,149 @@ 
+/*
+ * Copyright (c) 2021 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <config.h>
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "coverage.h"
+#include "csum.h"
+#include "dp-packet.h"
+#include "dp-packet-gso.h"
+#include "dpif-netdev.h"
+#include "openvswitch/compiler.h"
+#include "openvswitch/dynamic-string.h"
+#include "openvswitch/vlog.h"
+#include "packets.h"
+#include "util.h"
+
+VLOG_DEFINE_THIS_MODULE(dp_packet_gso);
+static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
+
+/* Update ip header's total len, and id and update tcp header's
+ * sent sequence number.  In the end, update ip and tcp csum.
+ */
+static void
+update_ipv4_tcp_headers(const struct dp_packet *src, struct dp_packet **pkts,
+                        uint16_t nb_segs)
+{
+    struct tcp_header *tcp;
+    struct ip_header *ip;
+    struct dp_packet *p;
+    uint32_t tcp_seq;
+    uint16_t ipid;
+    int i;
+
+    ip = dp_packet_l3(src);
+    ipid = ntohs(ip->ip_id);
+    tcp = dp_packet_l4(src);
+    tcp_seq = ntohl(get_16aligned_be32(&tcp->tcp_seq));
+
+    for (i = 0; i < nb_segs; i++) {
+        p = pkts[i];
+
+        ip = dp_packet_l3(p);
+        ip->ip_tot_len = htons(dp_packet_l3_size(p));
+        ip->ip_id = htons(ipid);
+        ip->ip_csum = 0;
+        ip->ip_csum = csum(ip, sizeof *ip);
+
+        tcp = dp_packet_l4(p);
+        put_16aligned_be32(&tcp->tcp_seq, htonl(tcp_seq));
+        packet_csum_tcpudp(p);
+
+        ipid += 1;
+        tcp_seq += (const char *) dp_packet_tail(p) -
+                   (const char *) dp_packet_l4(p) -
+                   TCP_OFFSET(tcp->tcp_ctl) * 4;
+    }
+}
+
+static void
+hdr_segment_init(struct dp_packet *dst, const struct dp_packet *src)
+{
+    /* Copy the following fields into the returned buffer: l2_pad_size,
+     * l2_5_ofs, l3_ofs, l4_ofs, cutlen, packet_type and md. */
+    memcpy(&dst->l2_pad_size, &src->l2_pad_size,
+           sizeof(struct dp_packet) -
+           offsetof(struct dp_packet, l2_pad_size));
+
+    *dp_packet_ol_flags_ptr(dst) = 0;
+}
+
+static int
+gso_do_segment(const struct dp_packet *p, uint16_t hdr_offset,
+               uint16_t pyld_unit_size, struct dp_packet **pout,
+               uint16_t nb_pout)
+{
+    uint16_t nb_segs = 0;
+    struct dp_packet *pkt;
+    uint16_t seg_size;
+    uint16_t pos = hdr_offset;
+    int bytes_remaining = dp_packet_size(p) - hdr_offset;
+
+    while (bytes_remaining > 0) {
+
+        seg_size = (bytes_remaining >= pyld_unit_size) ?
+                   pyld_unit_size : bytes_remaining;
+
+        /* Create a new dp_packet, put payload, push header. */
+        pkt = dp_packet_new_with_headroom(seg_size, hdr_offset);
+        hdr_segment_init(pkt, p);
+        dp_packet_put(pkt, (char *) dp_packet_data(p) + pos, seg_size);
+        dp_packet_push(pkt, dp_packet_data(p), hdr_offset);
+
+        pos += seg_size;
+        bytes_remaining -= seg_size;
+        pout[nb_segs] = pkt;
+        nb_segs++;
+
+        if (nb_segs > nb_pout) {
+            VLOG_WARN_RL(&rl, "Not enough memory to process GSO.");
+            nb_segs = -1;
+            /* need to free dp_packet. */
+            break;
+        }
+    }
+    return nb_segs;
+}
+
+int
+gso_tcp4_segment(struct dp_packet *p, uint16_t gso_size,
+                 struct dp_packet **pout, uint16_t nb_pout)
+{
+    uint16_t pyld_unit_size, hdr_offset;
+    int nb_segs;
+
+    hdr_offset = (char *) dp_packet_get_tcp_payload(p) -
+                 (char *) dp_packet_eth(p);
+    pyld_unit_size = gso_size - hdr_offset;
+
+    if (OVS_UNLIKELY(dp_packet_size(p) < ETH_PAYLOAD_MAX)) {
+        VLOG_WARN_RL(&rl, "Packet size %u bytes too small for GSO.",
+                     dp_packet_size(p));
+        return -EINVAL;
+    }
+
+    nb_segs = gso_do_segment(p, hdr_offset, pyld_unit_size, pout, nb_pout);
+    if (nb_segs > 0) {
+        /* Update TCP checksum. */
+        update_ipv4_tcp_headers(p, pout, nb_segs);
+    }
+
+    return nb_segs;
+}
diff --git a/lib/dp-packet-gso.h b/lib/dp-packet-gso.h
new file mode 100644
index 000000000000..d33d904c9e22
--- /dev/null
+++ b/lib/dp-packet-gso.h
@@ -0,0 +1,27 @@ 
+/*
+ * Copyright (c) 2021 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef DP_PACKET_GSO_H
+#define DP_PACKET_GSO_H 1
+
+#include <stdint.h>
+#include <stdbool.h>
+
+int gso_tcp4_segment(struct dp_packet *p, uint16_t gso_size,
+                     struct dp_packet **pouts, uint16_t nb_pouts);
+int gso_udp4_segment(struct dp_packet *p, uint16_t gso_size,
+                     struct dp_packet **pouts, uint16_t nb_pouts);
+#endif /* dp-packet-gso.h */
diff --git a/lib/netdev-afxdp.c b/lib/netdev-afxdp.c
index 482400d8d135..bf7b85d73a63 100644
--- a/lib/netdev-afxdp.c
+++ b/lib/netdev-afxdp.c
@@ -47,6 +47,7 @@ 
 #include "ovs-numa.h"
 #include "packets.h"
 #include "socket-util.h"
+#include "userspace-tso.h"
 #include "util.h"
 
 #ifndef SOL_XDP
@@ -867,6 +868,7 @@  netdev_afxdp_rxq_recv(struct netdev_rxq *rxq_, struct dp_packet_batch *batch,
                             FRAME_SIZE - FRAME_HEADROOM,
                             OVS_XDP_HEADROOM);
         dp_packet_set_size(packet, len);
+        *dp_packet_ol_flags_ptr(packet) = 0;
 
         /* Add packet into batch, increase batch->count. */
         dp_packet_batch_add(batch, packet);
@@ -1187,6 +1189,10 @@  netdev_afxdp_construct(struct netdev *netdev)
     dev->xsks = NULL;
     dev->tx_locks = NULL;
 
+    if (userspace_tso_enabled()) {
+        netdev->ol_flags = 0;
+    }
+
     netdev_request_reconfigure(netdev);
     return 0;
 }
diff --git a/lib/netdev.c b/lib/netdev.c
index 91e91955c09b..691ce81a01be 100644
--- a/lib/netdev.c
+++ b/lib/netdev.c
@@ -34,6 +34,7 @@ 
 #include "cmap.h"
 #include "coverage.h"
 #include "dpif.h"
+#include "dp-packet-gso.h"
 #include "dp-packet.h"
 #include "openvswitch/dynamic-string.h"
 #include "fatal-signal.h"
@@ -797,7 +798,6 @@  netdev_send_prepare_packet(const uint64_t netdev_flags,
     if (dp_packet_hwol_is_tso(packet)
         && !(netdev_flags & NETDEV_TX_OFFLOAD_TCP_TSO)) {
             /* Fall back to GSO in software. */
-            VLOG_ERR_BUF(errormsg, "No TSO support");
             return false;
     }
 
@@ -806,8 +806,8 @@  netdev_send_prepare_packet(const uint64_t netdev_flags,
         if (dp_packet_hwol_l4_is_tcp(packet)) {
             if (!(netdev_flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) {
                 /* Fall back to TCP csum in software. */
-                VLOG_ERR_BUF(errormsg, "No TCP checksum support");
-                return false;
+                packet_csum_tcpudp(packet);
+                return true;
             }
         } else if (dp_packet_hwol_l4_is_udp(packet)) {
             if (!(netdev_flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) {
@@ -835,7 +835,8 @@  netdev_send_prepare_packet(const uint64_t netdev_flags,
  * otherwise either fall back to software implementation or drop it. */
 static void
 netdev_send_prepare_batch(const struct netdev *netdev,
-                          struct dp_packet_batch *batch)
+                          struct dp_packet_batch *batch,
+                          struct dp_packet_batch *gso_batch)
 {
     struct dp_packet *packet;
     size_t i, size = dp_packet_batch_size(batch);
@@ -846,11 +847,16 @@  netdev_send_prepare_batch(const struct netdev *netdev,
         if (netdev_send_prepare_packet(netdev->ol_flags, packet, &errormsg)) {
             dp_packet_batch_refill(batch, packet, i);
         } else {
-            dp_packet_delete(packet);
-            COVERAGE_INC(netdev_send_prepare_drops);
-            VLOG_WARN_RL(&rl, "%s: Packet dropped: %s",
-                         netdev_get_name(netdev), errormsg);
-            free(errormsg);
+            if (dp_packet_hwol_is_tso(packet) &&
+                !(netdev->ol_flags & NETDEV_TX_OFFLOAD_TCP_TSO)) {
+                dp_packet_batch_add(gso_batch, packet);
+            } else {
+                dp_packet_delete(packet);
+                COVERAGE_INC(netdev_send_prepare_drops);
+                VLOG_WARN_RL(&rl, "%s: Packet dropped: %s",
+                             netdev_get_name(netdev), errormsg);
+                free(errormsg);
+            }
         }
     }
 }
@@ -884,17 +890,67 @@  int
 netdev_send(struct netdev *netdev, int qid, struct dp_packet_batch *batch,
             bool concurrent_txq)
 {
-    int error;
+    struct dp_packet_batch *gso_batch_ptr;
+    struct dp_packet_batch gso_batch;
+    struct dp_packet **gso_pkts;
+    struct dp_packet *packet;
+    uint16_t gso_pkts_len, nb_segs;
+    int error = 0;
 
-    netdev_send_prepare_batch(netdev, batch);
-    if (OVS_UNLIKELY(dp_packet_batch_is_empty(batch))) {
-        return 0;
+    dp_packet_batch_init(&gso_batch);
+    netdev_send_prepare_batch(netdev, batch, &gso_batch);
+
+    if (!dp_packet_batch_is_empty(batch)) {
+        error = netdev->netdev_class->send(netdev, qid, batch, concurrent_txq);
+        if (!error) {
+            COVERAGE_INC(netdev_sent);
+        }
     }
 
-    error = netdev->netdev_class->send(netdev, qid, batch, concurrent_txq);
-    if (!error) {
-        COVERAGE_INC(netdev_sent);
+    if (dp_packet_batch_is_empty(&gso_batch)) {
+        return error;
     }
+    gso_batch_ptr = &gso_batch;
+    DP_PACKET_BATCH_FOR_EACH (i, packet, gso_batch_ptr) {
+        struct dp_packet_batch seg_batch;
+        uint16_t gso_size = 1000; /* How to decide gso_size? */
+
+        gso_pkts_len = 2 * NETDEV_MAX_BURST;
+        gso_pkts = xmalloc(gso_pkts_len * sizeof(struct dp_packet *));
+
+        nb_segs = gso_tcp4_segment(packet, gso_size, gso_pkts, gso_pkts_len);
+        if (nb_segs <= 0) {
+            VLOG_WARN("GSO tcp4 segment failed");
+            dp_packet_delete_batch(gso_batch_ptr, true);
+            return EINVAL;
+        }
+        dp_packet_batch_init(&seg_batch);
+
+        for (i = 0; i < nb_segs; i++) {
+            dp_packet_batch_add(&seg_batch, gso_pkts[i]);
+
+            if (dp_packet_batch_is_full(&seg_batch)) {
+                /* Send the first batch when full. */
+                error = netdev->netdev_class->send(netdev, qid, &seg_batch,
+                                                   concurrent_txq);
+                if (!error) {
+                    COVERAGE_INC(netdev_sent);
+                }
+                dp_packet_batch_init(&seg_batch);
+            }
+        }
+        if (!dp_packet_batch_is_empty(&seg_batch)) {
+            /* Send the rest. */
+            error = netdev->netdev_class->send(netdev, qid, &seg_batch,
+                                               concurrent_txq);
+            if (!error) {
+                COVERAGE_INC(netdev_sent);
+            }
+        }
+
+    }
+    free(gso_pkts);
+
     return error;
 }
 
diff --git a/lib/packets.c b/lib/packets.c
index 4a7643c5dd3a..20702d25c2af 100644
--- a/lib/packets.c
+++ b/lib/packets.c
@@ -1887,3 +1887,38 @@  IP_ECN_set_ce(struct dp_packet *pkt, bool is_ipv6)
         }
     }
 }
+
+void
+packet_csum_tcpudp(struct dp_packet *p)
+{
+    struct eth_header *eth;
+    struct ip_header *ip;
+    struct tcp_header *tcp;
+    struct udp_header *udp;
+    uint32_t pseudo_hdr_csum;
+    uint8_t l4proto;
+    size_t l4_size;
+
+    eth = dp_packet_eth(p);
+    if (eth->eth_type != htons(ETH_TYPE_IP)) {
+        return;
+    }
+
+    ip = dp_packet_l3(p);
+    l4proto = ip->ip_proto;
+    l4_size = dp_packet_l4_size(p);
+
+    if (l4proto == IPPROTO_TCP) {
+        pseudo_hdr_csum = packet_csum_pseudoheader(ip);
+        tcp = dp_packet_l4(p);
+        tcp->tcp_csum = 0;
+        tcp->tcp_csum = csum_finish(csum_continue(pseudo_hdr_csum,
+                                                  tcp, l4_size));
+    } else if (l4proto == IPPROTO_UDP) {
+        pseudo_hdr_csum = packet_csum_pseudoheader(ip);
+        udp = dp_packet_l4(p);
+        udp->udp_csum = 0;
+        udp->udp_csum = csum_finish(csum_continue(pseudo_hdr_csum,
+                                                  udp, l4_size));
+    }
+}
diff --git a/lib/packets.h b/lib/packets.h
index 481bc22fa1fe..108087f916ac 100644
--- a/lib/packets.h
+++ b/lib/packets.h
@@ -1635,6 +1635,7 @@  void packet_put_ra_prefix_opt(struct dp_packet *,
                               const ovs_be128 router_prefix);
 uint32_t packet_csum_pseudoheader(const struct ip_header *);
 void IP_ECN_set_ce(struct dp_packet *pkt, bool is_ipv6);
+void packet_csum_tcpudp(struct dp_packet *p);
 
 #define DNS_HEADER_LEN 12
 struct dns_header {
diff --git a/tests/system-afxdp.at b/tests/system-afxdp.at
index 0d09906fb6c8..3c6a7708435c 100644
--- a/tests/system-afxdp.at
+++ b/tests/system-afxdp.at
@@ -45,3 +45,35 @@  NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0],
 
 OVS_TRAFFIC_VSWITCHD_STOP
 AT_CLEANUP
+
+dnl p0 at at_ns0 sends TSO packet to ovs-p0 at OVS.
+dnl ovs-p1 attached to OVS as type=afxdp
+AT_SETUP([AF_XDP - enable userspace TSO])
+AT_KEYWORDS([afxdp tso])
+OVS_TRAFFIC_VSWITCHD_START()
+
+AT_CHECK([ovs-vsctl set Open_vSwitch . other_config:userspace-tso-enable=true])
+AT_CHECK([ovs-ofctl add-flow br0 "actions=normal"])
+
+ADD_NAMESPACES(at_ns0, at_ns1)
+
+dnl Create and add ovs-p0 as system port
+ADD_VETH(p0, at_ns0, br0, "10.1.1.1/24")
+AT_CHECK([ovs-vsctl del-port ovs-p0])
+AT_CHECK([ovs-vsctl add-port br0 ovs-p0])
+dnl Enable tx offload at p0, so ovs-p0 sees TSO packets
+NS_CHECK_EXEC([at_ns0], [ethtool -K p0 tx on > /dev/null 2>&1])
+
+dnl Create and add ovs-p1 as afxdp port
+ADD_VETH(p1, at_ns1, br0, "10.1.1.2/24")
+
+NS_CHECK_EXEC([at_ns0], [ping -q -c 3 -i 0.3 -w 2 10.1.1.2 | FORMAT_PING], [0], [dnl
+3 packets transmitted, 3 received, 0% packet loss, time 0ms
+])
+
+dnl Send a TSO from ns0 to ns1
+NETNS_DAEMONIZE([at_ns1], [iperf -s], [iperf.pid])
+NS_CHECK_EXEC([at_ns0], [iperf -c 10.1.1.2 -t1 1> /dev/null], [0])
+
+OVS_TRAFFIC_VSWITCHD_STOP
+AT_CLEANUP