@@ -258,7 +258,7 @@ dp_packet_copy_common_members(struct dp_packet *new_b,
static inline void *
dp_packet_at(const struct dp_packet *b, size_t offset, size_t size)
{
- if (offset + size > dp_packet_size(b)) {
+ if (OVS_UNLIKELY(offset + size > dp_packet_size(b))) {
return NULL;
}
@@ -266,7 +266,7 @@ dp_packet_at(const struct dp_packet *b, size_t offset, size_t size)
if (b->source == DPBUF_DPDK) {
const struct rte_mbuf *mbuf = dp_packet_mbuf_from_offset(b, &offset);
- if (!mbuf || offset + size > mbuf->data_len) {
+ if (OVS_UNLIKELY(!mbuf || offset + size > mbuf->data_len)) {
return NULL;
}
@@ -290,7 +290,7 @@ static inline void *
dp_packet_tail(const struct dp_packet *b)
{
#ifdef DPDK_NETDEV
- if (b->source == DPBUF_DPDK) {
+ if (OVS_UNLIKELY(b->source == DPBUF_DPDK && !dp_packet_is_linear(b))) {
struct rte_mbuf *buf = CONST_CAST(struct rte_mbuf *, &b->mbuf);
/* Find last segment where data ends, meaning the tail of the chained
* mbufs must be there */
@@ -308,7 +308,7 @@ static inline void *
dp_packet_end(const struct dp_packet *b)
{
#ifdef DPDK_NETDEV
- if (b->source == DPBUF_DPDK) {
+ if (OVS_UNLIKELY(b->source == DPBUF_DPDK && !dp_packet_is_linear(b))) {
struct rte_mbuf *buf = CONST_CAST(struct rte_mbuf *, &(b->mbuf));
buf = rte_pktmbuf_lastseg(buf);
@@ -342,7 +342,7 @@ static inline void
dp_packet_clear(struct dp_packet *b)
{
#ifdef DPDK_NETDEV
- if (b->source == DPBUF_DPDK) {
+ if (OVS_UNLIKELY(b->source == DPBUF_DPDK && !dp_packet_is_linear(b))) {
/* sets pkt_len and data_len to zero and frees unused mbufs */
dp_packet_set_size(b, 0);
rte_pktmbuf_reset(&b->mbuf);
@@ -383,7 +383,7 @@ dp_packet_may_pull(const struct dp_packet *b, uint16_t offset, size_t size)
}
#ifdef DPDK_NETDEV
/* Offset needs to be within the first mbuf */
- if (offset + size > b->mbuf.data_len) {
+ if (OVS_UNLIKELY(offset + size > b->mbuf.data_len)) {
return false;
}
#endif
@@ -693,7 +693,7 @@ dp_packet_size(const struct dp_packet *b)
static inline void
dp_packet_set_size(struct dp_packet *b, uint32_t v)
{
- if (b->source == DPBUF_DPDK) {
+ if (OVS_UNLIKELY(b->source == DPBUF_DPDK && !dp_packet_is_linear(b))) {
struct rte_mbuf *mbuf = &b->mbuf;
uint16_t new_len = v;
uint16_t data_len;
@@ -906,8 +906,8 @@ dp_packet_copy_from_offset(const struct dp_packet *b, size_t offset,
static inline bool
dp_packet_is_linear(const struct dp_packet *b)
{
- if (b->source == DPBUF_DPDK) {
- return rte_pktmbuf_is_contiguous(&b->mbuf);
+ if (OVS_UNLIKELY(b->mbuf.nb_segs > 1)) {
+ return false;
}
return true;
@@ -711,6 +711,25 @@ ipv6_sanity_check(const struct ovs_16aligned_ip6_hdr *nh, size_t size)
return true;
}
+#define MAX_IPV4_LEN \
+ (ETH_HEADER_LEN + IP_HEADER_LEN + \
+ MAX(TCP_HEADER_LEN, \
+ MAX(UDP_HEADER_LEN, \
+ MAX(SCTP_HEADER_LEN, MAX(ICMP_HEADER_LEN, \
+ IGMP_HEADER_LEN)))))
+
+#define MAX_IPV6_LEN \
+ (ETH_HEADER_LEN + IPV6_HEADER_LEN + \
+ MAX(TCP_HEADER_LEN, \
+ MAX(UDP_HEADER_LEN, \
+ MAX(SCTP_HEADER_LEN, \
+ MAX(sizeof(struct tcp_header), IGMP_HEADER_LEN)))))
+
+#define MAX_ARP_NSH_LEN (ETH_HEADER_LEN + MAX(ARP_ETH_HEADER_LEN, \
+ NSH_BASE_HDR_LEN))
+
+#define MAX_HEADER_LEN MAX(MAX_IPV4_LEN, MAX(MAX_IPV6_LEN, MAX_ARP_NSH_LEN))
+
/* Initializes 'dst' from 'packet' and 'md', taking the packet type into
* account. 'dst' must have enough space for FLOW_U64S * 8 bytes.
*
@@ -759,6 +778,13 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
uint8_t *ct_nw_proto_p = NULL;
ovs_be16 ct_tp_src = 0, ct_tp_dst = 0;
+ /* Check if header is in first mbuf, otherwise return error */
+ if (OVS_UNLIKELY(!dp_packet_is_linear(packet))) {
+ if (!dp_packet_may_pull(packet, 0, MAX_HEADER_LEN)) {
+ return -EINVAL;
+ }
+ }
+
/* Metadata. */
if (flow_tnl_dst_is_set(&md->tunnel)) {
miniflow_push_words(mf, tunnel, &md->tunnel,
@@ -862,13 +888,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
int ip_len;
uint16_t tot_len;
- /* Check if header is in first mbuf, otherwise return error */
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l3_ofs, sizeof *nh)) {
- return -EINVAL;
- }
- }
-
if (OVS_UNLIKELY(!ipv4_sanity_check(nh, size, &ip_len, &tot_len))) {
goto out;
}
@@ -899,12 +918,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
ovs_be32 tc_flow;
uint16_t plen;
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l3_ofs, sizeof *nh)) {
- return -EINVAL;
- }
- }
-
if (OVS_UNLIKELY(!ipv6_sanity_check(nh, size))) {
goto out;
}
@@ -951,13 +964,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
dl_type == htons(ETH_TYPE_RARP)) {
struct eth_addr arp_buf[2];
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l3_ofs,
- ARP_ETH_HEADER_LEN)) {
- return -EINVAL;
- }
- }
-
const struct arp_eth_header *arp = (const struct arp_eth_header *)
data_try_pull(&data, &size, ARP_ETH_HEADER_LEN);
@@ -1005,13 +1011,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
if (OVS_LIKELY(size >= TCP_HEADER_LEN)) {
const struct tcp_header *tcp = data;
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l4_ofs,
- TCP_HEADER_LEN)) {
- return -EINVAL;
- }
- }
-
miniflow_push_be32(mf, arp_tha.ea[2], 0);
miniflow_push_be32(mf, tcp_flags,
TCP_FLAGS_BE32(tcp->tcp_ctl));
@@ -1024,13 +1023,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
if (OVS_LIKELY(size >= UDP_HEADER_LEN)) {
const struct udp_header *udp = data;
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l4_ofs,
- UDP_HEADER_LEN)) {
- return -EINVAL;
- }
- }
-
miniflow_push_be16(mf, tp_src, udp->udp_src);
miniflow_push_be16(mf, tp_dst, udp->udp_dst);
miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
@@ -1040,13 +1032,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
if (OVS_LIKELY(size >= SCTP_HEADER_LEN)) {
const struct sctp_header *sctp = data;
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l4_ofs,
- SCTP_HEADER_LEN)) {
- return -EINVAL;
- }
- }
-
miniflow_push_be16(mf, tp_src, sctp->sctp_src);
miniflow_push_be16(mf, tp_dst, sctp->sctp_dst);
miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
@@ -1056,13 +1041,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
if (OVS_LIKELY(size >= ICMP_HEADER_LEN)) {
const struct icmp_header *icmp = data;
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l4_ofs,
- ICMP_HEADER_LEN)) {
- return -EINVAL;
- }
- }
-
miniflow_push_be16(mf, tp_src, htons(icmp->icmp_type));
miniflow_push_be16(mf, tp_dst, htons(icmp->icmp_code));
miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
@@ -1072,13 +1050,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
if (OVS_LIKELY(size >= IGMP_HEADER_LEN)) {
const struct igmp_header *igmp = data;
- if (!dp_packet_is_linear(packet)) {
- if (!dp_packet_may_pull(packet, packet->l4_ofs,
- IGMP_HEADER_LEN)) {
- return -EINVAL;
- }
- }
-
miniflow_push_be16(mf, tp_src, htons(igmp->igmp_type));
miniflow_push_be16(mf, tp_dst, htons(igmp->igmp_code));
miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
@@ -559,17 +559,17 @@ void
free_dpdk_buf(struct dp_packet *p)
{
/* If non-pmd we need to lock on nonpmd_mp_mutex mutex */
- if (!dpdk_thread_is_pmd()) {
+ if (dpdk_thread_is_pmd()) {
+ rte_pktmbuf_free(&p->mbuf);
+
+ return;
+ } else {
ovs_mutex_lock(&nonpmd_mp_mutex);
rte_pktmbuf_free(&p->mbuf);
ovs_mutex_unlock(&nonpmd_mp_mutex);
-
- return;
}
-
- rte_pktmbuf_free(&p->mbuf);
}
static void
@@ -2159,9 +2159,6 @@ netdev_dpdk_prep_tso_packet(struct rte_mbuf *mbuf, int mtu)
mbuf->outer_l2_len = 0;
mbuf->outer_l3_len = 0;
- /* Reset packet RX RSS flag to reuse in egress. */
- dp_packet_mbuf_rss_flag_reset(pkt);
-
if (!(mbuf->ol_flags & PKT_TX_TCP_SEG)) {
return;
}
@@ -363,7 +363,7 @@ parse_gre_header(struct dp_packet *packet,
}
hlen = ulen + gre_header_len(greh->flags);
- if (hlen > dp_packet_size(packet)) {
+ if (!dp_packet_may_pull(packet, packet->l3_ofs, hlen)) {
return -EINVAL;
}
@@ -534,7 +534,7 @@ netdev_erspan_pop_header(struct dp_packet *packet)
IPV6_HEADER_LEN : IP_HEADER_LEN;
pkt_metadata_init_tnl(md);
- if (hlen > dp_packet_size(packet)) {
+ if (!dp_packet_may_pull(packet, packet->l3_ofs, hlen)) {
goto err;
}
@@ -719,7 +719,7 @@ netdev_vxlan_pop_header(struct dp_packet *packet)
ovs_assert(packet->l4_ofs > 0);
pkt_metadata_init_tnl(md);
- if (VXLAN_HLEN > dp_packet_l4_size(packet)) {
+ if (!dp_packet_may_pull(packet, packet->l4_ofs, VXLAN_HLEN)) {
goto err;
}
@@ -841,7 +841,7 @@ netdev_geneve_pop_header(struct dp_packet *packet)
unsigned int hlen, opts_len, ulen;
pkt_metadata_init_tnl(md);
- if (GENEVE_BASE_HLEN > dp_packet_l4_size(packet)) {
+ if (!dp_packet_may_pull(packet, packet->l4_ofs, GENEVE_BASE_HLEN)) {
VLOG_WARN_RL(&err_rl, "geneve packet too small: min header=%u packet size=%"PRIuSIZE"\n",
(unsigned int)GENEVE_BASE_HLEN, dp_packet_l4_size(packet));
goto err;
@@ -854,7 +854,7 @@ netdev_geneve_pop_header(struct dp_packet *packet)
opts_len = gnh->opt_len * 4;
hlen = ulen + GENEVE_BASE_HLEN + opts_len;
- if (hlen > dp_packet_size(packet)) {
+ if (!dp_packet_may_pull(packet, packet->l4_ofs, hlen)) {
VLOG_WARN_RL(&err_rl, "geneve packet too small: header len=%u packet size=%u\n",
hlen, dp_packet_size(packet));
goto err;