@@ -257,7 +257,19 @@ needs to be affinitized accordingly.
The rx queues are assigned to pmd threads on the same NUMA node in a
round-robin fashion.
-### 4.4 Exact Match Cache
+### 4.4 DPDK Physical Port Queue Sizes
+ `ovs-vsctl set Open_vSwitch . other_config:dpdk-rxq-size=<n-rxq-desc>`
+ `ovs-vsctl set Open_vSwitch . other_config:dpdk-txq-size=<n-txq-desc>`
+
+ The command above sets the number of rx/tx descriptors that the NICs
+ associated with 'dpdk' ports will be initialised with.
+
+ Different 'dpdk-rxq-size' and 'dpdk-txq-size' configurations yield different
+ benefits in terms of throughput and latency for different scenarios.
+ Generally, smaller queue sizes can have a positive impact for latency at the
+ expense of throughput. The opposite is often true for larger queue sizes.
+
+### 4.5 Exact Match Cache
Each pmd thread contains one EMC. After initial flow setup in the
datapath, the EMC contains a single table and provides the lowest level
@@ -274,7 +286,7 @@ needs to be affinitized accordingly.
avoiding datapath classifier lookups is to have multiple pmd threads
running. This can be done as described in section 4.2.
-### 4.5 Rx Mergeable buffers
+### 4.6 Rx Mergeable buffers
Rx Mergeable buffers is a virtio feature that allows chaining of multiple
virtio descriptors to handle large packet sizes. As such, large packets
@@ -90,6 +90,9 @@ v2.6.0 - xx xxx xxxx
* Jumbo frame support
* Remove dpdkvhostcuse port type.
* OVS client mode for vHost and vHost reconnect (Requires QEMU 2.7)
+ * New 'other_config:dpdk-rxq-size' and 'other_config:dpdk-txq-size' fields
+ that specify the number of rxq and txq descriptors to initialise DPDK
+ NICs with.
- Increase number of registers to 16.
- ovs-benchmark: This utility has been removed due to lack of use and
bitrot.
@@ -132,8 +132,9 @@ BUILD_ASSERT_DECL((MAX_NB_MBUF / ROUND_DOWN_POW2(MAX_NB_MBUF/MIN_NB_MBUF))
#define SOCKET0 0
-#define NIC_PORT_RX_Q_SIZE 2048 /* Size of Physical NIC RX Queue, Max (n+32<=4096)*/
-#define NIC_PORT_TX_Q_SIZE 2048 /* Size of Physical NIC TX Queue, Max (n+32<=4096)*/
+#define NIC_PORT_DEFAULT_RXQ_SIZE 2048 /* Default size of Physical NIC RXQ */
+#define NIC_PORT_DEFAULT_TXQ_SIZE 2048 /* Default size of Physical NIC TXQ */
+#define NIC_PORT_MAX_Q_SIZE 4096 /* Maximum size of Physical NIC Queue */
#define OVS_VHOST_MAX_QUEUE_NUM 1024 /* Maximum number of vHost TX queues. */
#define OVS_VHOST_QUEUE_MAP_UNKNOWN (-1) /* Mapping not initialized. */
@@ -142,6 +143,9 @@ BUILD_ASSERT_DECL((MAX_NB_MBUF / ROUND_DOWN_POW2(MAX_NB_MBUF/MIN_NB_MBUF))
static char *vhost_sock_dir = NULL; /* Location of vhost-user sockets */
+static int dpdk_rxq_size = 0; /* Configured size of Physical NIC RX Queue */
+static int dpdk_txq_size = 0; /* Configured size of Physical NIC TX Queue */
+
#define VHOST_ENQ_RETRY_NUM 8
#define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
@@ -642,7 +646,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int n_rxq, int n_txq)
}
for (i = 0; i < n_txq; i++) {
- diag = rte_eth_tx_queue_setup(dev->port_id, i, NIC_PORT_TX_Q_SIZE,
+ diag = rte_eth_tx_queue_setup(dev->port_id, i, dpdk_txq_size,
dev->socket_id, NULL);
if (diag) {
VLOG_INFO("Interface %s txq(%d) setup error: %s",
@@ -658,7 +662,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int n_rxq, int n_txq)
}
for (i = 0; i < n_rxq; i++) {
- diag = rte_eth_rx_queue_setup(dev->port_id, i, NIC_PORT_RX_Q_SIZE,
+ diag = rte_eth_rx_queue_setup(dev->port_id, i, dpdk_rxq_size,
dev->socket_id, NULL,
dev->dpdk_mp->mp);
if (diag) {
@@ -3122,6 +3126,23 @@ process_vhost_flags(char *flag, char *default_val, int size,
return changed;
}
+static void
+process_queue_size_flags(const struct smap *ovs_other_config, char *flag,
+ int default_size, int *new_size)
+{
+ int queue_size;
+
+ queue_size = smap_get_int(ovs_other_config, flag, 0);
+ if (queue_size > 0 && queue_size <= NIC_PORT_MAX_Q_SIZE) {
+ *new_size = queue_size;
+ } else {
+ *new_size = default_size;
+ }
+
+ VLOG_INFO("'dpdk' ports will be configured with a %s of %i",
+ flag, *new_size);
+}
+
static char **
grow_argv(char ***argv, size_t cur_siz, size_t grow_by)
{
@@ -3364,6 +3385,12 @@ dpdk_init__(const struct smap *ovs_other_config)
vhost_sock_dir = sock_dir_subcomponent;
}
+ /* Determine the queue sizes to specify when initializing 'dpdk' ports */
+ process_queue_size_flags(ovs_other_config, "dpdk-rxq-size",
+ NIC_PORT_DEFAULT_RXQ_SIZE, &dpdk_rxq_size);
+ process_queue_size_flags(ovs_other_config, "dpdk-txq-size",
+ NIC_PORT_DEFAULT_TXQ_SIZE, &dpdk_txq_size);
+
argv = grow_argv(&argv, 0, 1);
argc = 1;
argv[0] = xstrdup(ovs_get_program_name());
@@ -299,6 +299,32 @@
</p>
</column>
+ <column name="other_config" key="dpdk-rxq-size"
+ type='{"type": "integer", "minInteger": 1, "maxInteger": 4096}'>
+ <p>
+ Specifies the queue size (number rx descriptors) of dpdk ports.
+ Ensure that your NIC(s) can support the particular value before
+ modifying 'dpdk-rxq-size'.
+ </p>
+ <p>
+ Defaults to 2048. Maximum value is 4096. Changing this value requires
+ restarting the daemon.
+ </p>
+ </column>
+
+ <column name="other_config" key="dpdk-txq-size"
+ type='{"type": "integer", "minInteger": 1, "maxInteger": 4096}'>
+ <p>
+ Specifies the queue size (number tx descriptors) of dpdk ports.
+ Ensure that your NIC(s) can support the particular value before
+ modifying 'dpdk-txq-size'.
+ </p>
+ <p>
+ Defaults to 2048. Maximum value is 4096. Changing this value requires
+ restarting the daemon.
+ </p>
+ </column>
+
<column name="other_config" key="n-handler-threads"
type='{"type": "integer", "minInteger": 1}'>
<p>
The other_config:dpdk-rxq-size and dpdk-txq-size fields allow for an integer between 1 and 4096 that reflects the number of rx/tx descriptors to initialise 'dpdk' devices with. If no value is specified, they default to 2048. 'dpdk-*xq-size' fields must be set before launching the daemon as changing the queue size requires the NIC to restart. Signed-off-by: Ciara Loftus <ciara.loftus@intel.com> --- INSTALL.DPDK-ADVANCED.md | 16 ++++++++++++++-- NEWS | 3 +++ lib/netdev-dpdk.c | 35 +++++++++++++++++++++++++++++++---- vswitchd/vswitch.xml | 26 ++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 6 deletions(-)