@@ -16,6 +16,9 @@
#include <config.h>
+#include <errno.h>
+#include <stdarg.h>
+
#include <rte_malloc.h>
#include <rte_mbuf.h>
#include <rte_mempool.h>
@@ -30,9 +33,12 @@
#include "ovs-thread.h"
#include "smap.h"
#include "sset.h"
+#include "unaligned.h"
+#include "userspace-tso.h"
#include "util.h"
VLOG_DEFINE_THIS_MODULE(netdev_dpdk_common);
+static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
static bool per_port_memory = false; /* Status of per port memory support */
@@ -544,6 +550,49 @@ static const struct dpdk_qos_ops *const qos_confs[] = {
NULL
};
+void
+netdev_dpdk_common_vlog_rl(enum vlog_level level, const char *format, ...)
+{
+ va_list args;
+
+ if (vlog_should_drop(&this_module, level, &rl)) {
+ return;
+ }
+
+ va_start(args, format);
+ vlog_valist(&this_module, level, format, args);
+ va_end(args);
+}
+
+void
+netdev_dpdk_mbuf_dump(const char *prefix, const char *message,
+ const struct rte_mbuf *mbuf)
+{
+ static struct vlog_rate_limit dump_rl = VLOG_RATE_LIMIT_INIT(5, 5);
+ char *response = NULL;
+ FILE *stream;
+ size_t size;
+
+ if (VLOG_DROP_DBG(&dump_rl)) {
+ return;
+ }
+
+ stream = open_memstream(&response, &size);
+ if (!stream) {
+ VLOG_ERR("Unable to open memstream for mbuf dump: %s.",
+ ovs_strerror(errno));
+ return;
+ }
+
+ rte_pktmbuf_dump(stream, mbuf, rte_pktmbuf_pkt_len(mbuf));
+
+ fclose(stream);
+
+ VLOG_DBG(prefix ? "%s: %s:\n%s" : "%s%s:\n%s",
+ prefix ? prefix : "", message, response);
+ free(response);
+}
+
static bool
srtcm_policer_pkt_handle(struct rte_meter_srtcm *meter,
struct rte_meter_srtcm_profile *profile,
@@ -601,6 +650,185 @@ dpdk_qos_ingress_policer_run(struct dpdk_qos_ingress_policer *policer,
return cnt;
}
+uint32_t
+netdev_dpdk_extbuf_size(uint32_t data_len)
+{
+ uint32_t buf_len = data_len;
+
+ buf_len += sizeof(struct rte_mbuf_ext_shared_info) + sizeof(uintptr_t);
+ buf_len = RTE_ALIGN_CEIL(buf_len, sizeof(uintptr_t));
+
+ return buf_len;
+}
+
+void *
+netdev_dpdk_extbuf_allocate(uint32_t buf_len)
+{
+ return rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE);
+}
+
+static void
+netdev_dpdk_extbuf_free(void *addr OVS_UNUSED, void *opaque)
+{
+ rte_free(opaque);
+}
+
+void
+netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf, uint32_t data_len)
+{
+ struct rte_mbuf *pkt = (struct rte_mbuf *) b;
+ struct rte_mbuf_ext_shared_info *shinfo;
+ uint16_t buf_len = data_len;
+
+ shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
+ netdev_dpdk_extbuf_free,
+ buf);
+ ovs_assert(shinfo != NULL);
+
+ if (RTE_MBUF_HAS_EXTBUF(pkt)) {
+ rte_pktmbuf_detach_extbuf(pkt);
+ }
+ rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len,
+ shinfo);
+ /* OVS only supports mono segment.
+ * Packet size did not change, restore the current segment length. */
+ pkt->data_len = pkt->pkt_len;
+}
+
+static struct rte_mbuf *
+dpdk_pktmbuf_attach_extbuf(struct rte_mbuf *pkt, uint32_t data_len)
+{
+ uint32_t total_len = RTE_PKTMBUF_HEADROOM + data_len;
+ struct rte_mbuf_ext_shared_info *shinfo = NULL;
+ uint16_t buf_len;
+ void *buf;
+
+ total_len = netdev_dpdk_extbuf_size(total_len);
+ if (OVS_UNLIKELY(total_len > UINT16_MAX)) {
+ VLOG_ERR("Can't copy packet: too big %u", total_len);
+ return NULL;
+ }
+
+ buf_len = total_len;
+ buf = netdev_dpdk_extbuf_allocate(buf_len);
+ if (OVS_UNLIKELY(buf == NULL)) {
+ VLOG_ERR("Failed to allocate memory using rte_malloc: %u", buf_len);
+ return NULL;
+ }
+
+ /* Initialize shinfo. */
+ shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
+ netdev_dpdk_extbuf_free,
+ buf);
+ if (OVS_UNLIKELY(shinfo == NULL)) {
+ netdev_dpdk_extbuf_free(NULL, buf);
+ VLOG_ERR("Failed to initialize shared info for mbuf while "
+ "attempting to attach an external buffer.");
+ return NULL;
+ }
+
+ rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len,
+ shinfo);
+ rte_pktmbuf_reset_headroom(pkt);
+
+ return pkt;
+}
+
+struct rte_mbuf *
+dpdk_pktmbuf_alloc(struct rte_mempool *mp, uint32_t data_len)
+{
+ struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
+
+ if (OVS_UNLIKELY(!pkt)) {
+ return NULL;
+ }
+
+ if (rte_pktmbuf_tailroom(pkt) >= data_len) {
+ return pkt;
+ }
+
+ if (dpdk_pktmbuf_attach_extbuf(pkt, data_len)) {
+ return pkt;
+ }
+
+ rte_pktmbuf_free(pkt);
+
+ return NULL;
+}
+
+struct dp_packet *
+dpdk_copy_dp_packet_to_mbuf(struct rte_mempool *mp, struct dp_packet *pkt_orig)
+{
+ struct rte_mbuf *mbuf_dest;
+ struct dp_packet *pkt_dest;
+ uint32_t pkt_len;
+
+ pkt_len = dp_packet_size(pkt_orig);
+ mbuf_dest = dpdk_pktmbuf_alloc(mp, pkt_len);
+ if (OVS_UNLIKELY(mbuf_dest == NULL)) {
+ return NULL;
+ }
+
+ pkt_dest = CONTAINER_OF(mbuf_dest, struct dp_packet, mbuf);
+ memcpy(dp_packet_data(pkt_dest), dp_packet_data(pkt_orig), pkt_len);
+ dp_packet_set_size(pkt_dest, pkt_len);
+
+ mbuf_dest->tx_offload = pkt_orig->mbuf.tx_offload;
+ mbuf_dest->packet_type = pkt_orig->mbuf.packet_type;
+ mbuf_dest->ol_flags |= (pkt_orig->mbuf.ol_flags &
+ ~(RTE_MBUF_F_EXTERNAL | RTE_MBUF_F_INDIRECT));
+ mbuf_dest->tso_segsz = pkt_orig->mbuf.tso_segsz;
+
+ memcpy(&pkt_dest->l2_pad_size, &pkt_orig->l2_pad_size,
+ sizeof(struct dp_packet) - offsetof(struct dp_packet, l2_pad_size));
+
+ if (dp_packet_l3(pkt_dest)) {
+ if (dp_packet_eth(pkt_dest)) {
+ mbuf_dest->l2_len = (char *) dp_packet_l3(pkt_dest)
+ - (char *) dp_packet_eth(pkt_dest);
+ } else {
+ mbuf_dest->l2_len = 0;
+ }
+ if (dp_packet_l4(pkt_dest)) {
+ mbuf_dest->l3_len = (char *) dp_packet_l4(pkt_dest)
+ - (char *) dp_packet_l3(pkt_dest);
+ } else {
+ mbuf_dest->l3_len = 0;
+ }
+ }
+
+ return pkt_dest;
+}
+
+/* Replace packets in a 'batch' with their corresponding copies using
+ * DPDK memory.
+ *
+ * Returns the number of good packets in the batch. */
+size_t
+dpdk_copy_batch_to_mbuf(struct netdev_dpdk_common *common,
+ struct dp_packet_batch *batch)
+{
+ size_t i, size = dp_packet_batch_size(batch);
+ struct dp_packet *packet;
+
+ DP_PACKET_BATCH_REFILL_FOR_EACH (i, size, packet, batch) {
+ if (OVS_UNLIKELY(packet->source == DPBUF_DPDK)) {
+ dp_packet_batch_add(batch, packet);
+ } else {
+ struct dp_packet *pktcopy;
+
+ pktcopy = dpdk_copy_dp_packet_to_mbuf(common->dpdk_mp->mp, packet);
+ if (pktcopy) {
+ dp_packet_batch_add(batch, pktcopy);
+ }
+
+ dp_packet_delete(packet);
+ }
+ }
+
+ return dp_packet_batch_size(batch);
+}
+
struct dpdk_qos_ingress_policer *
dpdk_qos_ingress_policer_construct(uint32_t rate, uint32_t burst)
{
@@ -27,8 +27,10 @@
#include <rte_ethdev.h>
#include <rte_mempool.h>
+#include "dp-packet.h"
#include "netdev-provider.h"
#include "openvswitch/compiler.h"
+#include "openvswitch/vlog.h"
#include "ovs-thread.h"
#include "packets.h"
@@ -327,4 +329,273 @@ int netdev_dpdk_common_queue_dump_next(const struct netdev_dpdk_common *common,
int netdev_dpdk_common_queue_dump_done(const struct netdev *netdev,
void *state_);
+void netdev_dpdk_mbuf_dump(const char *prefix, const char *message,
+ const struct rte_mbuf *mbuf);
+
+void netdev_dpdk_common_vlog_rl(enum vlog_level, const char *format, ...)
+ OVS_PRINTF_FORMAT(2, 3);
+
+uint32_t netdev_dpdk_extbuf_size(uint32_t data_len);
+void *netdev_dpdk_extbuf_allocate(uint32_t buf_len);
+void netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf,
+ uint32_t data_len);
+
+struct rte_mbuf *dpdk_pktmbuf_alloc(struct rte_mempool *mp, uint32_t data_len);
+struct dp_packet *dpdk_copy_dp_packet_to_mbuf(struct rte_mempool *mp,
+ struct dp_packet *pkt_orig);
+size_t dpdk_copy_batch_to_mbuf(struct netdev_dpdk_common *common,
+ struct dp_packet_batch *batch);
+
+/* Prepare the packet for HWOL.
+ * Return True if the packet is OK to continue. */
+static inline bool
+netdev_dpdk_prep_hwol_packet(struct netdev_dpdk_common *common,
+ struct rte_mbuf *mbuf)
+{
+ struct dp_packet *pkt = CONTAINER_OF(mbuf, struct dp_packet, mbuf);
+ uint64_t unexpected = mbuf->ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK;
+ struct netdev *netdev = &common->up;
+ const struct ip_header *ip;
+ bool is_sctp;
+ bool l3_csum;
+ bool l4_csum;
+ bool is_tcp;
+ bool is_udp;
+ void *l2;
+ void *l3;
+ void *l4;
+
+ if (OVS_UNLIKELY(unexpected)) {
+ netdev_dpdk_common_vlog_rl(VLL_WARN,
+ "%s: Unexpected Tx offload flags: %#"PRIx64,
+ netdev_get_name(netdev), unexpected);
+ netdev_dpdk_mbuf_dump(netdev_get_name(netdev),
+ "Packet with unexpected ol_flags", mbuf);
+ return false;
+ }
+
+ if (!dp_packet_ip_checksum_partial(pkt)
+ && !dp_packet_inner_ip_checksum_partial(pkt)
+ && !dp_packet_l4_checksum_partial(pkt)
+ && !dp_packet_inner_l4_checksum_partial(pkt)
+ && !mbuf->tso_segsz) {
+
+ return true;
+ }
+
+ if (dp_packet_tunnel(pkt)) {
+ mbuf->outer_l2_len = (char *) dp_packet_l3(pkt) -
+ (char *) dp_packet_eth(pkt);
+ mbuf->outer_l3_len = (char *) dp_packet_l4(pkt) -
+ (char *) dp_packet_l3(pkt);
+
+ if (dp_packet_tunnel_geneve(pkt)) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GENEVE;
+ } else if (dp_packet_tunnel_vxlan(pkt)) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_VXLAN;
+ } else {
+ ovs_assert(dp_packet_tunnel_gre(pkt));
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GRE;
+ }
+
+ if (dp_packet_ip_checksum_partial(pkt)) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM;
+ }
+
+ if (dp_packet_l4_checksum_partial(pkt)) {
+ ovs_assert(dp_packet_l4_proto_udp(pkt));
+ mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM;
+ }
+
+ ip = dp_packet_l3(pkt);
+ mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4
+ ? RTE_MBUF_F_TX_OUTER_IPV4
+ : RTE_MBUF_F_TX_OUTER_IPV6;
+
+ /* Inner L2 length must account for the tunnel header length. */
+ l2 = dp_packet_l4(pkt);
+ l3 = dp_packet_inner_l3(pkt);
+ l3_csum = dp_packet_inner_ip_checksum_partial(pkt);
+ l4 = dp_packet_inner_l4(pkt);
+ l4_csum = dp_packet_inner_l4_checksum_partial(pkt);
+ is_tcp = dp_packet_inner_l4_proto_tcp(pkt);
+ is_udp = dp_packet_inner_l4_proto_udp(pkt);
+ is_sctp = dp_packet_inner_l4_proto_sctp(pkt);
+ } else {
+ mbuf->outer_l2_len = 0;
+ mbuf->outer_l3_len = 0;
+
+ l2 = dp_packet_eth(pkt);
+ l3 = dp_packet_l3(pkt);
+ l3_csum = dp_packet_ip_checksum_partial(pkt);
+ l4 = dp_packet_l4(pkt);
+ l4_csum = dp_packet_l4_checksum_partial(pkt);
+ is_tcp = dp_packet_l4_proto_tcp(pkt);
+ is_udp = dp_packet_l4_proto_udp(pkt);
+ is_sctp = dp_packet_l4_proto_sctp(pkt);
+ }
+
+ ovs_assert(l4);
+
+ ip = l3;
+ mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4
+ ? RTE_MBUF_F_TX_IPV4 : RTE_MBUF_F_TX_IPV6;
+
+ if (l3_csum) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
+ }
+
+ if (l4_csum) {
+ if (is_tcp) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
+ } else if (is_udp) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
+ } else {
+ ovs_assert(is_sctp);
+ mbuf->ol_flags |= RTE_MBUF_F_TX_SCTP_CKSUM;
+ }
+ }
+
+ mbuf->l2_len = (char *) l3 - (char *) l2;
+ mbuf->l3_len = (char *) l4 - (char *) l3;
+
+ if (mbuf->tso_segsz) {
+ struct tcp_header *th = l4;
+ int hdr_len;
+
+ mbuf->l4_len = TCP_OFFSET(th->tcp_ctl) * 4;
+
+ hdr_len = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len;
+ if (dp_packet_tunnel(pkt)) {
+ hdr_len += mbuf->outer_l2_len + mbuf->outer_l3_len;
+ }
+
+ if (OVS_UNLIKELY((hdr_len + mbuf->tso_segsz)
+ > common->max_packet_len)) {
+ netdev_dpdk_common_vlog_rl(VLL_WARN,
+ "%s: Oversized TSO packet. hdr: %"PRIu32
+ ", gso: %"PRIu32", max len: %"PRIu32"",
+ netdev->name, hdr_len, mbuf->tso_segsz,
+ common->max_packet_len);
+ return false;
+ }
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_SEG;
+
+ /* DPDK API mandates IPv4 checksum when requesting TSO. */
+ if (IP_VER(ip->ip_ihl_ver) == 4) {
+ mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
+ }
+ }
+
+ return true;
+}
+
+/* Prepare a batch for HWOL.
+ * Return the number of good packets in the batch. */
+static inline int
+netdev_dpdk_prep_hwol_batch(struct netdev_dpdk_common *common,
+ struct rte_mbuf **pkts, int pkt_cnt)
+{
+ int i = 0;
+ int cnt = 0;
+ struct rte_mbuf *pkt;
+
+ /* Prepare and filter bad HWOL packets. */
+ for (i = 0; i < pkt_cnt; i++) {
+ pkt = pkts[i];
+ if (!netdev_dpdk_prep_hwol_packet(common, pkt)) {
+ rte_pktmbuf_free(pkt);
+ continue;
+ }
+
+ if (OVS_UNLIKELY(i != cnt)) {
+ pkts[cnt] = pkt;
+ }
+ cnt++;
+ }
+
+ return cnt;
+}
+
+static inline int
+netdev_dpdk_filter_packet_len(struct netdev_dpdk_common *common,
+ struct rte_mbuf **pkts, int pkt_cnt)
+{
+ int i = 0;
+ int cnt = 0;
+ struct rte_mbuf *pkt;
+
+ /* Filter oversized packets. The TSO packets are filtered out
+ * during the offloading preparation for performance reasons. */
+ for (i = 0; i < pkt_cnt; i++) {
+ pkt = pkts[i];
+ if (OVS_UNLIKELY((pkt->pkt_len > common->max_packet_len)
+ && !pkt->tso_segsz)) {
+ netdev_dpdk_common_vlog_rl(VLL_WARN,
+ "%s: Too big size %" PRIu32
+ " max_packet_len %d",
+ common->up.name, pkt->pkt_len,
+ common->max_packet_len);
+ rte_pktmbuf_free(pkt);
+ continue;
+ }
+
+ if (OVS_UNLIKELY(i != cnt)) {
+ pkts[cnt] = pkt;
+ }
+ cnt++;
+ }
+
+ return cnt;
+}
+
+static inline size_t
+netdev_dpdk_common_send(struct netdev_dpdk_common *common,
+ struct dp_packet_batch *batch,
+ struct netdev_dpdk_sw_stats *stats)
+{
+ struct rte_mbuf **pkts = (struct rte_mbuf **) batch->packets;
+ size_t cnt, pkt_cnt = dp_packet_batch_size(batch);
+ struct dpdk_qos_conf *qos_conf;
+ struct dp_packet *packet;
+ bool need_copy = false;
+
+ memset(stats, 0, sizeof *stats);
+
+ DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+ if (packet->source != DPBUF_DPDK) {
+ need_copy = true;
+ break;
+ }
+ }
+
+ /* Copy dp-packets to mbufs. */
+ if (OVS_UNLIKELY(need_copy)) {
+ cnt = dpdk_copy_batch_to_mbuf(common, batch);
+ stats->tx_failure_drops += pkt_cnt - cnt;
+ pkt_cnt = cnt;
+ }
+
+ /* Drop oversized packets. */
+ cnt = netdev_dpdk_filter_packet_len(common, pkts, pkt_cnt);
+ stats->tx_mtu_exceeded_drops += pkt_cnt - cnt;
+ pkt_cnt = cnt;
+
+ if (common->up.ol_flags) {
+ /* Prepare each mbuf for hardware offloading. */
+ cnt = netdev_dpdk_prep_hwol_batch(common, pkts, pkt_cnt);
+ stats->tx_invalid_hwol_drops += pkt_cnt - cnt;
+ pkt_cnt = cnt;
+ }
+
+ /* Apply Quality of Service policy. */
+ qos_conf = ovsrcu_get(struct dpdk_qos_conf *, &common->qos_conf);
+ if (qos_conf) {
+ cnt = dpdk_qos_run(qos_conf, pkts, pkt_cnt, true);
+ stats->tx_qos_drops += pkt_cnt - cnt;
+ }
+
+ return cnt;
+}
+
#endif /* NETDEV_DPDK_COMMON_H */
@@ -362,9 +362,6 @@ static void netdev_dpdk_clear_xstats(struct netdev_dpdk *dev);
int netdev_dpdk_get_vid(const struct netdev_dpdk *dev);
-static void netdev_dpdk_mbuf_dump(const char *prefix, const char *message,
- const struct rte_mbuf *);
-
static bool
is_eth_class(const struct netdev_class *class)
{
@@ -2043,204 +2040,6 @@ netdev_dpdk_batch_init_packet_fields(struct dp_packet_batch *batch)
}
}
-/* Prepare the packet for HWOL.
- * Return True if the packet is OK to continue. */
-static bool
-netdev_dpdk_prep_hwol_packet(struct netdev_dpdk_common *common,
- struct rte_mbuf *mbuf)
-{
- struct dp_packet *pkt = CONTAINER_OF(mbuf, struct dp_packet, mbuf);
- uint64_t unexpected = mbuf->ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK;
- struct netdev *netdev = &common->up;
- const struct ip_header *ip;
- bool is_sctp;
- bool l3_csum;
- bool l4_csum;
- bool is_tcp;
- bool is_udp;
- void *l2;
- void *l3;
- void *l4;
-
- if (OVS_UNLIKELY(unexpected)) {
- VLOG_WARN_RL(&rl, "%s: Unexpected Tx offload flags: %#"PRIx64,
- netdev_get_name(netdev), unexpected);
- netdev_dpdk_mbuf_dump(netdev_get_name(netdev),
- "Packet with unexpected ol_flags", mbuf);
- return false;
- }
-
- if (!dp_packet_ip_checksum_partial(pkt)
- && !dp_packet_inner_ip_checksum_partial(pkt)
- && !dp_packet_l4_checksum_partial(pkt)
- && !dp_packet_inner_l4_checksum_partial(pkt)
- && !mbuf->tso_segsz) {
-
- return true;
- }
-
- if (dp_packet_tunnel(pkt)) {
- mbuf->outer_l2_len = (char *) dp_packet_l3(pkt) -
- (char *) dp_packet_eth(pkt);
- mbuf->outer_l3_len = (char *) dp_packet_l4(pkt) -
- (char *) dp_packet_l3(pkt);
-
- if (dp_packet_tunnel_geneve(pkt)) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GENEVE;
- } else if (dp_packet_tunnel_vxlan(pkt)) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_VXLAN;
- } else {
- ovs_assert(dp_packet_tunnel_gre(pkt));
- mbuf->ol_flags |= RTE_MBUF_F_TX_TUNNEL_GRE;
- }
-
- if (dp_packet_ip_checksum_partial(pkt)) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM;
- }
-
- if (dp_packet_l4_checksum_partial(pkt)) {
- ovs_assert(dp_packet_l4_proto_udp(pkt));
- mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM;
- }
-
- ip = dp_packet_l3(pkt);
- mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4
- ? RTE_MBUF_F_TX_OUTER_IPV4
- : RTE_MBUF_F_TX_OUTER_IPV6;
-
- /* Inner L2 length must account for the tunnel header length. */
- l2 = dp_packet_l4(pkt);
- l3 = dp_packet_inner_l3(pkt);
- l3_csum = dp_packet_inner_ip_checksum_partial(pkt);
- l4 = dp_packet_inner_l4(pkt);
- l4_csum = dp_packet_inner_l4_checksum_partial(pkt);
- is_tcp = dp_packet_inner_l4_proto_tcp(pkt);
- is_udp = dp_packet_inner_l4_proto_udp(pkt);
- is_sctp = dp_packet_inner_l4_proto_sctp(pkt);
- } else {
- mbuf->outer_l2_len = 0;
- mbuf->outer_l3_len = 0;
-
- l2 = dp_packet_eth(pkt);
- l3 = dp_packet_l3(pkt);
- l3_csum = dp_packet_ip_checksum_partial(pkt);
- l4 = dp_packet_l4(pkt);
- l4_csum = dp_packet_l4_checksum_partial(pkt);
- is_tcp = dp_packet_l4_proto_tcp(pkt);
- is_udp = dp_packet_l4_proto_udp(pkt);
- is_sctp = dp_packet_l4_proto_sctp(pkt);
- }
-
- ovs_assert(l4);
-
- ip = l3;
- mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4
- ? RTE_MBUF_F_TX_IPV4 : RTE_MBUF_F_TX_IPV6;
-
- if (l3_csum) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
- }
-
- if (l4_csum) {
- if (is_tcp) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
- } else if (is_udp) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
- } else {
- ovs_assert(is_sctp);
- mbuf->ol_flags |= RTE_MBUF_F_TX_SCTP_CKSUM;
- }
- }
-
- mbuf->l2_len = (char *) l3 - (char *) l2;
- mbuf->l3_len = (char *) l4 - (char *) l3;
-
- if (mbuf->tso_segsz) {
- struct tcp_header *th = l4;
- int hdr_len;
-
- mbuf->l4_len = TCP_OFFSET(th->tcp_ctl) * 4;
-
- hdr_len = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len;
- if (dp_packet_tunnel(pkt)) {
- hdr_len += mbuf->outer_l2_len + mbuf->outer_l3_len;
- }
-
- if (OVS_UNLIKELY((hdr_len + mbuf->tso_segsz)
- > common->max_packet_len)) {
- VLOG_WARN_RL(&rl, "%s: Oversized TSO packet. hdr: %"PRIu32", "
- "gso: %"PRIu32", max len: %"PRIu32"",
- netdev->name, hdr_len, mbuf->tso_segsz,
- common->max_packet_len);
- return false;
- }
- mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_SEG;
-
- /* DPDK API mandates IPv4 checksum when requesting TSO. */
- if (IP_VER(ip->ip_ihl_ver) == 4) {
- mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
- }
- }
-
- return true;
-}
-
-/* Prepare a batch for HWOL.
- * Return the number of good packets in the batch. */
-static int
-netdev_dpdk_prep_hwol_batch(struct netdev_dpdk_common *common,
- struct rte_mbuf **pkts, int pkt_cnt)
-{
- int i = 0;
- int cnt = 0;
- struct rte_mbuf *pkt;
-
- /* Prepare and filter bad HWOL packets. */
- for (i = 0; i < pkt_cnt; i++) {
- pkt = pkts[i];
- if (!netdev_dpdk_prep_hwol_packet(common, pkt)) {
- rte_pktmbuf_free(pkt);
- continue;
- }
-
- if (OVS_UNLIKELY(i != cnt)) {
- pkts[cnt] = pkt;
- }
- cnt++;
- }
-
- return cnt;
-}
-
-static void
-netdev_dpdk_mbuf_dump(const char *prefix, const char *message,
- const struct rte_mbuf *mbuf)
-{
- static struct vlog_rate_limit dump_rl = VLOG_RATE_LIMIT_INIT(5, 5);
- char *response = NULL;
- FILE *stream;
- size_t size;
-
- if (VLOG_DROP_DBG(&dump_rl)) {
- return;
- }
-
- stream = open_memstream(&response, &size);
- if (!stream) {
- VLOG_ERR("Unable to open memstream for mbuf dump: %s.",
- ovs_strerror(errno));
- return;
- }
-
- rte_pktmbuf_dump(stream, mbuf, rte_pktmbuf_pkt_len(mbuf));
-
- fclose(stream);
-
- VLOG_DBG(prefix ? "%s: %s:\n%s" : "%s%s:\n%s",
- prefix ? prefix : "", message, response);
- free(response);
-}
-
/* Tries to transmit 'pkts' to txq 'qid' of device 'dev'. Takes ownership of
* 'pkts', even in case of failure.
*
@@ -2409,265 +2208,6 @@ netdev_dpdk_eth_rxq_recv(struct netdev_rxq *rxq, struct dp_packet_batch *batch,
return 0;
}
-static int
-netdev_dpdk_filter_packet_len(struct netdev_dpdk_common *common,
- struct rte_mbuf **pkts, int pkt_cnt)
-{
- int i = 0;
- int cnt = 0;
- struct rte_mbuf *pkt;
-
- /* Filter oversized packets. The TSO packets are filtered out
- * during the offloading preparation for performance reasons. */
- for (i = 0; i < pkt_cnt; i++) {
- pkt = pkts[i];
- if (OVS_UNLIKELY((pkt->pkt_len > common->max_packet_len)
- && !pkt->tso_segsz)) {
- VLOG_WARN_RL(&rl, "%s: Too big size %" PRIu32 " "
- "max_packet_len %d", common->up.name, pkt->pkt_len,
- common->max_packet_len);
- rte_pktmbuf_free(pkt);
- continue;
- }
-
- if (OVS_UNLIKELY(i != cnt)) {
- pkts[cnt] = pkt;
- }
- cnt++;
- }
-
- return cnt;
-}
-
-uint32_t
-netdev_dpdk_extbuf_size(uint32_t data_len)
-{
- uint32_t buf_len = data_len;
-
- buf_len += sizeof(struct rte_mbuf_ext_shared_info) + sizeof(uintptr_t);
- buf_len = RTE_ALIGN_CEIL(buf_len, sizeof(uintptr_t));
-
- return buf_len;
-}
-
-void *
-netdev_dpdk_extbuf_allocate(uint32_t buf_len)
-{
- return rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE);
-}
-
-static void
-netdev_dpdk_extbuf_free(void *addr OVS_UNUSED, void *opaque)
-{
- rte_free(opaque);
-}
-
-void
-netdev_dpdk_extbuf_replace(struct dp_packet *b, void *buf, uint32_t data_len)
-{
- struct rte_mbuf *pkt = (struct rte_mbuf *) b;
- struct rte_mbuf_ext_shared_info *shinfo;
- uint16_t buf_len = data_len;
-
- shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
- netdev_dpdk_extbuf_free,
- buf);
- ovs_assert(shinfo != NULL);
-
- if (RTE_MBUF_HAS_EXTBUF(pkt)) {
- rte_pktmbuf_detach_extbuf(pkt);
- }
- rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len,
- shinfo);
- /* OVS only supports mono segment.
- * Packet size did not change, restore the current segment length. */
- pkt->data_len = pkt->pkt_len;
-}
-
-static struct rte_mbuf *
-dpdk_pktmbuf_attach_extbuf(struct rte_mbuf *pkt, uint32_t data_len)
-{
- uint32_t total_len = RTE_PKTMBUF_HEADROOM + data_len;
- struct rte_mbuf_ext_shared_info *shinfo = NULL;
- uint16_t buf_len;
- void *buf;
-
- total_len = netdev_dpdk_extbuf_size(total_len);
- if (OVS_UNLIKELY(total_len > UINT16_MAX)) {
- VLOG_ERR("Can't copy packet: too big %u", total_len);
- return NULL;
- }
-
- buf_len = total_len;
- buf = netdev_dpdk_extbuf_allocate(buf_len);
- if (OVS_UNLIKELY(buf == NULL)) {
- VLOG_ERR("Failed to allocate memory using rte_malloc: %u", buf_len);
- return NULL;
- }
-
- /* Initialize shinfo. */
- shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
- netdev_dpdk_extbuf_free,
- buf);
- if (OVS_UNLIKELY(shinfo == NULL)) {
- netdev_dpdk_extbuf_free(NULL, buf);
- VLOG_ERR("Failed to initialize shared info for mbuf while "
- "attempting to attach an external buffer.");
- return NULL;
- }
-
- rte_pktmbuf_attach_extbuf(pkt, buf, rte_malloc_virt2iova(buf), buf_len,
- shinfo);
- rte_pktmbuf_reset_headroom(pkt);
-
- return pkt;
-}
-
-static struct rte_mbuf *
-dpdk_pktmbuf_alloc(struct rte_mempool *mp, uint32_t data_len)
-{
- struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
-
- if (OVS_UNLIKELY(!pkt)) {
- return NULL;
- }
-
- if (rte_pktmbuf_tailroom(pkt) >= data_len) {
- return pkt;
- }
-
- if (dpdk_pktmbuf_attach_extbuf(pkt, data_len)) {
- return pkt;
- }
-
- rte_pktmbuf_free(pkt);
-
- return NULL;
-}
-
-static struct dp_packet *
-dpdk_copy_dp_packet_to_mbuf(struct rte_mempool *mp, struct dp_packet *pkt_orig)
-{
- struct rte_mbuf *mbuf_dest;
- struct dp_packet *pkt_dest;
- uint32_t pkt_len;
-
- pkt_len = dp_packet_size(pkt_orig);
- mbuf_dest = dpdk_pktmbuf_alloc(mp, pkt_len);
- if (OVS_UNLIKELY(mbuf_dest == NULL)) {
- return NULL;
- }
-
- pkt_dest = CONTAINER_OF(mbuf_dest, struct dp_packet, mbuf);
- memcpy(dp_packet_data(pkt_dest), dp_packet_data(pkt_orig), pkt_len);
- dp_packet_set_size(pkt_dest, pkt_len);
-
- mbuf_dest->tx_offload = pkt_orig->mbuf.tx_offload;
- mbuf_dest->packet_type = pkt_orig->mbuf.packet_type;
- mbuf_dest->ol_flags |= (pkt_orig->mbuf.ol_flags &
- ~(RTE_MBUF_F_EXTERNAL | RTE_MBUF_F_INDIRECT));
- mbuf_dest->tso_segsz = pkt_orig->mbuf.tso_segsz;
-
- memcpy(&pkt_dest->l2_pad_size, &pkt_orig->l2_pad_size,
- sizeof(struct dp_packet) - offsetof(struct dp_packet, l2_pad_size));
-
- if (dp_packet_l3(pkt_dest)) {
- if (dp_packet_eth(pkt_dest)) {
- mbuf_dest->l2_len = (char *) dp_packet_l3(pkt_dest)
- - (char *) dp_packet_eth(pkt_dest);
- } else {
- mbuf_dest->l2_len = 0;
- }
- if (dp_packet_l4(pkt_dest)) {
- mbuf_dest->l3_len = (char *) dp_packet_l4(pkt_dest)
- - (char *) dp_packet_l3(pkt_dest);
- } else {
- mbuf_dest->l3_len = 0;
- }
- }
-
- return pkt_dest;
-}
-
-/* Replace packets in a 'batch' with their corresponding copies using
- * DPDK memory.
- *
- * Returns the number of good packets in the batch. */
-static size_t
-dpdk_copy_batch_to_mbuf(struct netdev *netdev, struct dp_packet_batch *batch)
-{
- struct netdev_dpdk_common *common = netdev_dpdk_common_cast(netdev);
- size_t i, size = dp_packet_batch_size(batch);
- struct dp_packet *packet;
-
- DP_PACKET_BATCH_REFILL_FOR_EACH (i, size, packet, batch) {
- if (OVS_UNLIKELY(packet->source == DPBUF_DPDK)) {
- dp_packet_batch_add(batch, packet);
- } else {
- struct dp_packet *pktcopy;
-
- pktcopy = dpdk_copy_dp_packet_to_mbuf(common->dpdk_mp->mp, packet);
- if (pktcopy) {
- dp_packet_batch_add(batch, pktcopy);
- }
-
- dp_packet_delete(packet);
- }
- }
-
- return dp_packet_batch_size(batch);
-}
-
-static size_t
-netdev_dpdk_common_send(struct netdev_dpdk_common *common,
- struct dp_packet_batch *batch,
- struct netdev_dpdk_sw_stats *stats)
-{
- struct rte_mbuf **pkts = (struct rte_mbuf **) batch->packets;
- size_t cnt, pkt_cnt = dp_packet_batch_size(batch);
- struct netdev *netdev = &common->up;
- struct dpdk_qos_conf *qos_conf;
- struct dp_packet *packet;
- bool need_copy = false;
-
- memset(stats, 0, sizeof *stats);
-
- DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
- if (packet->source != DPBUF_DPDK) {
- need_copy = true;
- break;
- }
- }
-
- /* Copy dp-packets to mbufs. */
- if (OVS_UNLIKELY(need_copy)) {
- cnt = dpdk_copy_batch_to_mbuf(netdev, batch);
- stats->tx_failure_drops += pkt_cnt - cnt;
- pkt_cnt = cnt;
- }
-
- /* Drop oversized packets. */
- cnt = netdev_dpdk_filter_packet_len(common, pkts, pkt_cnt);
- stats->tx_mtu_exceeded_drops += pkt_cnt - cnt;
- pkt_cnt = cnt;
-
- if (netdev->ol_flags) {
- /* Prepare each mbuf for hardware offloading. */
- cnt = netdev_dpdk_prep_hwol_batch(common, pkts, pkt_cnt);
- stats->tx_invalid_hwol_drops += pkt_cnt - cnt;
- pkt_cnt = cnt;
- }
-
- /* Apply Quality of Service policy. */
- qos_conf = ovsrcu_get(struct dpdk_qos_conf *, &common->qos_conf);
- if (qos_conf) {
- cnt = dpdk_qos_run(qos_conf, pkts, pkt_cnt, true);
- stats->tx_qos_drops += pkt_cnt - cnt;
- }
-
- return cnt;
-}
-
static int
netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
struct dp_packet_batch *batch,
Move all preparation, packet copy and dump and transmission code in the common file. One consequence is that all debug logs are now under netdev_dpdk_common vlog. Signed-off-by: David Marchand <david.marchand@redhat.com> --- lib/netdev-dpdk-common.c | 228 +++++++++++++++++++ lib/netdev-dpdk-common.h | 271 +++++++++++++++++++++++ lib/netdev-dpdk.c | 460 --------------------------------------- 3 files changed, 499 insertions(+), 460 deletions(-)