@@ -208,8 +208,9 @@ needs to be affinitized accordingly.
pmd thread is CPU bound, and needs to be affinitized to isolated
cores for optimum performance.
- By setting a bit in the mask, a pmd thread is created and pinned
- to the corresponding CPU core. e.g. to run a pmd thread on core 2
+ By setting a bit in the mask, a pmd thread is created, pinned
+ to the corresponding CPU core and the nice value set to '-20'.
+ e.g. to pin a pmd thread on core 2
`ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=4`
@@ -249,6 +250,10 @@ needs to be affinitized accordingly.
NIC port0 <-> OVS <-> VM <-> OVS <-> NIC port 1
+ Note: It is recommended that the OVS control thread and pmd thread
+ shouldn't be pinned to same core i.e 'dpdk-lcore-mask' and 'pmd-cpu-mask'
+ cpu mask settings should be non-overlapping.
+
### 4.3 DPDK physical port Rx Queues
`ovs-vsctl set Interface <DPDK interface> options:n_rxq=<integer>`
@@ -3100,6 +3100,10 @@ pmd_thread_main(void *f_)
ovs_numa_thread_setaffinity_core(pmd->core_id);
dpdk_set_lcore_id(pmd->core_id);
poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list);
+
+ /* Set pmd thread's nice value to -20 */
+#define MIN_NICE -20
+ ovs_numa_thread_setpriority(MIN_NICE);
reload:
emc_cache_init(&pmd->flow_cache);
@@ -24,6 +24,7 @@
#include <stddef.h>
#include <string.h>
#include <sys/types.h>
+#include <sys/resource.h>
#include <unistd.h>
#endif /* __linux__ */
@@ -613,3 +614,21 @@ int ovs_numa_thread_setaffinity_core(unsigned core_id OVS_UNUSED)
return EOPNOTSUPP;
#endif /* __linux__ */
}
+
+void
+ovs_numa_thread_setpriority(int nice OVS_UNUSED)
+{
+ if (dummy_numa) {
+ return;
+ }
+
+#ifndef _WIN32
+ int err;
+ err = setpriority(PRIO_PROCESS, 0, nice);
+ if (err) {
+ VLOG_ERR("Thread priority error %s",ovs_strerror(err));
+ }
+#else
+ return EOPNOTSUPP;
+#endif
+}
@@ -56,6 +56,7 @@ void ovs_numa_unpin_core(unsigned core_id);
struct ovs_numa_dump *ovs_numa_dump_cores_on_numa(int numa_id);
void ovs_numa_dump_destroy(struct ovs_numa_dump *);
int ovs_numa_thread_setaffinity_core(unsigned core_id);
+void ovs_numa_thread_setpriority(int nice);
#define FOR_EACH_CORE_ON_NUMA(ITER, DUMP) \
LIST_FOR_EACH((ITER), list_node, &(DUMP)->dump)
Increase the DPDK pmd thread scheduling priority by lowering the nice value. This will advise the kernel scheduler to prioritize pmd thread over other processes. Signed-off-by: Bhanuprakash Bodireddy <bhanuprakash.bodireddy@intel.com> --- v6->v7: * Remove realtime scheduling policy logic. * Increase pmd thread scheduling priority by lowering nice value to -20. * Update doc accordingly. v5->v6: * Prohibit spawning pmd thread on the lowest core in dpdk-lcore-mask if lcore-mask and pmd-mask affinity are identical. * Updated Note section in INSTALL.DPDK-ADVANCED doc. * Tested below cases to verify system stability with pmd priority patch dpdk-lcore-mask | pmd-cpu-mask | Comment 1. Not set | Not set | control threads affinity: 0-27 pmd thread: core 0 2. 1 | 1 | pmd thread isn't spawned and warning logged in logfile. 3. 1 | c | 4. F0 | F0 | control threads pinned to core 4. 3 pmd threads created on core 5,6,7 but 4. v4->v5: * Reword Note section in DPDK-ADVANCED.md v3->v4: * Document update * Use ovs_strerror for reporting errors in lib-numa.c v2->v3: * Move set_priority() function to lib/ovs-numa.c * Apply realtime scheduling policy and priority to pmd thread only if pmd-cpu-mask is passed. * Update INSTALL.DPDK-ADVANCED. v1->v2: * Removed #ifdef and introduced dummy function "pmd_thread_setpriority" in netdev-dpdk.h * Rebase INSTALL.DPDK-ADVANCED.md | 9 +++++++-- lib/dpif-netdev.c | 4 ++++ lib/ovs-numa.c | 19 +++++++++++++++++++ lib/ovs-numa.h | 1 + 4 files changed, 31 insertions(+), 2 deletions(-)