| Message ID | SY4PR01MB8438A3F17ABF417EC30E6663CD07A@SY4PR01MB8438.ausprd01.prod.outlook.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | netdev-dpdk: Add support for userspace port-based packet-per-second policing. | expand |
| Context | Check | Description |
|---|---|---|
| ovsrobot/apply-robot | warning | apply and check: warning |
| ovsrobot/github-robot-_Build_and_Test | fail | github build: failed |
miterv@outlook.com writes: > From: Lin Huang <linhuang@ruijie.com.cn> > > OvS has supported packet-per-second policer which can be set at ingress > and egress side in kernel datapath. But the userspace datapath doesn't > support for ingress and egress packet-per-second policing now. > > So, this patch add support for userspace egress pps policing by using > native ovs token bucket library. Token bucket is accumulated by 'rate' > tokens per millisecond and store maximum tokens at 'burst' bucket size. > One token in the bucket means one packet (1 kpkts * millisecond) which > will drop or pass by policer. > > This patch add a new egress Qos type called 'kpkts-policer'. > the policer police the kilo-packet per second at which the token bucket > be updated by 'kpkts_rate'. and the policer's burst size is defined by > 'kpkts_burst'. > > Examples: > $ovs-vsctl set port vhost-user0 qos=@newqos -- > --id=@newqos create qos type=kpkts-policer \ > other-config:kpkts_rate=123 other-config:kpkts_burst=123 > > Add some unit tests for egress packet-per-second policing. > > Signed-off-by: Lin Huang <linhuang@ruijie.com.cn> > --- > Documentation/topics/dpdk/qos.rst | 21 +++ > NEWS | 1 + > lib/netdev-dpdk.c | 159 +++++++++++++++++++ > tests/system-dpdk.at | 255 ++++++++++++++++++++++++++++++ > vswitchd/vswitch.xml | 32 ++++ > 5 files changed, 468 insertions(+) > > diff --git a/Documentation/topics/dpdk/qos.rst b/Documentation/topics/dpdk/qos.rst > index a98ec672f..6a4408127 100644 > --- a/Documentation/topics/dpdk/qos.rst > +++ b/Documentation/topics/dpdk/qos.rst > @@ -36,6 +36,9 @@ QoS (Egress Policing) > Single Queue Policer > ~~~~~~~~~~~~~~~~~~~~ > > +Bytes Per Second Policer > ++++++++++++++++++++ > + > Assuming you have a :doc:`vhost-user port <vhost-user>` transmitting traffic > consisting of packets of size 64 bytes, the following command would limit the > egress transmission rate of the port to ~1,000,000 packets per second:: > @@ -52,6 +55,24 @@ To clear the QoS configuration from the port and ovsdb, run:: > > $ ovs-vsctl destroy QoS vhost-user0 -- clear Port vhost-user0 qos > > +Packets Per Second Policer > ++++++++++++++++++++ > + > +Assuming you have a :doc:`vhost-user port <vhost-user>` transmitting traffic, > +the following command would limit the egress transmission rate of the port to > +~1,000,000 packets per second:: > + > + ovs-vsctl set port vhost-user0 qos=@newqos -- \ > + --id=@newqos create qos type=kpkts-policer \ > + other-config:kpkts_rate=1000 other-config:kpkts_burst=1000 > + > +To examine the QoS configuration of the port, run:: > + > + $ ovs-appctl -t ovs-vswitchd qos/show vhost-user0 > + > +To clear the QoS configuration from the port and ovsdb, run:: > + > + $ ovs-vsctl destroy QoS vhost-user0 -- clear Port vhost-user0 qos > > Multi Queue Policer > ~~~~~~~~~~~~~~~~~~~ > diff --git a/NEWS b/NEWS > index 7a852427e..3d1ab282e 100644 > --- a/NEWS > +++ b/NEWS > @@ -63,6 +63,7 @@ v3.2.0 - xx xxx xxxx > * 'ovs-appctl dpif-netdev/pmd-sleep-show' command was added to get the > max sleep configuration of PMD thread cores. > * Removed experimental tag from PMD load based sleeping. > + * Added new Qos type 'pkts-policer' to support kilo packet-per-second policing. > - Linux TC offload: > * Add support for offloading VXLAN tunnels with the GBP extensions. > - Python > diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c > index 7e3020a7d..e6b8922aa 100644 > --- a/lib/netdev-dpdk.c > +++ b/lib/netdev-dpdk.c > @@ -19,6 +19,7 @@ > > #include <errno.h> > #include <signal.h> > +#include <stdint.h> > #include <stdlib.h> > #include <string.h> > #include <unistd.h> > @@ -59,6 +60,7 @@ > #include "openvswitch/ofp-parse.h" > #include "openvswitch/ofp-print.h" > #include "openvswitch/shash.h" > +#include "openvswitch/token-bucket.h" > #include "openvswitch/vlog.h" > #include "ovs-numa.h" > #include "ovs-rcu.h" > @@ -91,6 +93,8 @@ static bool per_port_memory = false; /* Status of per port memory support */ > #define OVS_CACHE_LINE_SIZE CACHE_LINE_SIZE > #define OVS_VPORT_DPDK "ovs_dpdk" > > +#define MAX_KPKTS_PARAMETER 4294967U /* UINT32_MAX / 1000 */ > + > /* > * need to reserve tons of extra space in the mbufs so we can align the > * DMA addresses to 4KB. > @@ -346,6 +350,7 @@ struct dpdk_qos_ops { > /* dpdk_qos_ops for each type of user space QoS implementation. */ > static const struct dpdk_qos_ops egress_policer_ops; > static const struct dpdk_qos_ops trtcm_policer_ops; > +static const struct dpdk_qos_ops kpkts_policer_ops; > > /* > * Array of dpdk_qos_ops, contains pointer to all supported QoS > @@ -354,6 +359,7 @@ static const struct dpdk_qos_ops trtcm_policer_ops; > static const struct dpdk_qos_ops *const qos_confs[] = { > &egress_policer_ops, > &trtcm_policer_ops, > + &kpkts_policer_ops, > NULL > }; > > @@ -5572,6 +5578,159 @@ static const struct dpdk_qos_ops trtcm_policer_ops = { > .qos_queue_dump_state_init = trtcm_policer_qos_queue_dump_state_init > }; > > +/* kpkts-policer details */ > +struct kpkts_policer { > + struct qos_conf qos_conf; > + struct token_bucket tb; > + uint32_t kpkts_rate; > + uint32_t kpkts_burst; > +}; > + > +static int > +kpkts_policer_run_single_packet(struct token_bucket *tb, struct rte_mbuf **pkts, > + int pkt_cnt, bool should_steal) > +{ > + struct rte_mbuf *batch[NETDEV_MAX_BURST] = {0}; > + long long int now = time_msec(); > + struct rte_mbuf *pkt = NULL; > + int i = 0, n = 0; > + int cnt = 0; > + > + for (i = 0; i < pkt_cnt; i++) { > + pkt = pkts[i]; > + /* Handle current packet. */ > + if (token_bucket_withdraw(tb, 1, now)) { > + /* Count passed packets. */ > + if (cnt != i) { > + pkts[cnt] = pkt; > + } > + cnt++; > + } else { > + /* Count dropped packets. */ > + batch[n++] = pkt; > + } > + } > + > + if (should_steal && n) { > + rte_pktmbuf_free_bulk(batch, n); > + } > + > + return cnt; > +} > + > +static int > +kpkts_policer_profile_config(struct token_bucket *tb, > + uint32_t kpkts_rate, uint32_t kpkts_burst) > +{ > + if (kpkts_rate > MAX_KPKTS_PARAMETER || > + kpkts_burst > MAX_KPKTS_PARAMETER) { > + return EINVAL; > + } > + > + /* Rate in kilo-packets/second, bucket 1000 packets. > + * msec * kilo-packets/sec = 1 packets. */ > + if (kpkts_rate) { > + /* Parameters between (1 ~ MAX_KPKTS_PARAMETER). */ > + token_bucket_init(tb, kpkts_rate, kpkts_burst * 1000); > + } else { > + /* Zero means not to police the traffic. */ > + return EINVAL; > + } > + > + return 0; > +} > + > +static int > +kpkts_policer_qos_construct(const struct smap *details, struct qos_conf **conf) > +{ > + uint32_t kpkts_rate, kpkts_burst; > + struct kpkts_policer *policer; > + int err; > + > + policer = xmalloc(sizeof *policer); > + kpkts_rate = smap_get_uint(details, "kpkts_rate", 0); > + kpkts_burst = smap_get_uint(details, "kpkts_burst", 0); > + > + /* > + * Force to 0 if no rate specified, > + * default to rate if burst is 0, > + * else stick with user-specified value. > + */ > + kpkts_burst = (!kpkts_rate ? 0 : !kpkts_burst ? kpkts_rate : kpkts_burst); > + > + qos_conf_init(&policer->qos_conf, &kpkts_policer_ops); > + err = kpkts_policer_profile_config(&policer->tb, kpkts_rate, kpkts_burst); > + if (err) { > + VLOG_DBG("Could not create token bucket for egress policer"); I didn't do an in-depth look at this yet, just noticed this needs free here for policer object. > + return err; > + } > + > + policer->kpkts_rate = kpkts_rate; > + policer->kpkts_burst = kpkts_burst; > + > + *conf = &policer->qos_conf; > + > + return err; > +} > + > +static void > +kpkts_policer_qos_destruct(struct qos_conf *conf) > +{ > + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, > + qos_conf); > + > + free(policer); > +} > + > +static int > +kpkts_policer_qos_get(const struct qos_conf *conf, struct smap *details) > +{ > + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, > + qos_conf); > + > + smap_add_format(details, "kpkts_rate", "%"PRIu32, policer->kpkts_rate); > + smap_add_format(details, "kpkts_burst", "%"PRIu32, policer->kpkts_burst); > + > + return 0; > +} > + > +static bool > +kpkts_pkts_policer_qos_is_equal(const struct qos_conf *conf, > + const struct smap *details) > +{ > + uint32_t rate, burst; > + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, > + qos_conf); > + > + rate = smap_get_uint(details, "pkts_rate", 0); > + burst = smap_get_uint(details, "pkts_burst", 0); > + > + return (policer->tb.rate == rate && policer->tb.burst == burst); > +} > + > +static int > +kpkts_policer_run(struct qos_conf *conf, struct rte_mbuf **pkts, int pkt_cnt, > + bool should_steal) > +{ > + int cnt; > + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, > + qos_conf); > + > + cnt = kpkts_policer_run_single_packet(&policer->tb, pkts, pkt_cnt, > + should_steal); > + > + return cnt; > +} > + > +static const struct dpdk_qos_ops kpkts_policer_ops = { > + .qos_name = "kpkts-policer", > + .qos_construct = kpkts_policer_qos_construct, > + .qos_destruct = kpkts_policer_qos_destruct, > + .qos_get = kpkts_policer_qos_get, > + .qos_is_equal = kpkts_pkts_policer_qos_is_equal, > + .qos_run = kpkts_policer_run > +}; > + > static int > dpdk_rx_steer_add_flow(struct netdev_dpdk *dev, > const struct rte_flow_item items[], > diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at > index 0f58e8574..8b80a31e6 100644 > --- a/tests/system-dpdk.at > +++ b/tests/system-dpdk.at > @@ -570,6 +570,261 @@ dnl -------------------------------------------------------------------------- > > > > +dnl -------------------------------------------------------------------------- > +dnl QoS (kpkts) create delete vport port > +AT_SETUP([OVS-DPDK - QoS (kpkts) create delete vport port]) > +AT_KEYWORDS([dpdk]) > + > +OVS_DPDK_PRE_CHECK() > +OVS_DPDK_START() > + > +dnl Add userspace bridge and attach it to OVS and add egress policer > +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) > +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) > +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=123 other-config:kpkts_burst=456]) > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +sleep 2 > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) > + > +dnl Remove egress policer > +AT_CHECK([ovs-vsctl destroy QoS dpdkvhostuserclient0 -- clear Port dpdkvhostuserclient0 qos]) > + > +dnl Check egress policer was removed correctly > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) > + > +dnl Clean up > +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) > +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ > +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d > +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d > +])") > +AT_CLEANUP > +dnl -------------------------------------------------------------------------- > + > + > + > +dnl -------------------------------------------------------------------------- > +dnl QoS (kpkts) no rate > +AT_SETUP([OVS-DPDK - QoS (kpkts) no rate]) > +AT_KEYWORDS([dpdk]) > + > +OVS_DPDK_PRE_CHECK() > +OVS_DPDK_START() > + > +dnl Add userspace bridge and attach it to OVS and add egress policer > +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) > +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) > +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_burst=123]) > +sleep 2 > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) > + > +dnl Check egress policer was not created > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) > + > +dnl Clean up > +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) > +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ > +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d > +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d > +])") > +AT_CLEANUP > +dnl -------------------------------------------------------------------------- > + > + > + > +dnl -------------------------------------------------------------------------- > +dnl QoS (kpkts) no burst > +AT_SETUP([OVS-DPDK - QoS (kpkts) no burst]) > +AT_KEYWORDS([dpdk]) > + > +OVS_DPDK_PRE_CHECK() > +OVS_DPDK_START() > + > +dnl Add userspace bridge and attach it to OVS and add egress policer > +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) > +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) > +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=123]) > +sleep 2 > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) > + > +dnl Check egress policer was created > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +AT_CHECK([grep -E 'kpkts_rate: 123' stdout], [], [stdout]) > + > +dnl Check egress policer was deleted > +QOS_UUID=`ovs-vsctl get port dpdkvhostuserclient0 qos` > +AT_CHECK([ovs-vsctl set qos $QOS_UUID other_config:kpkts_rate=0]) > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) > + > +dnl Clean up > +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) > +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ > +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d > +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d > +])") > +AT_CLEANUP > +dnl -------------------------------------------------------------------------- > + > + > + > +dnl -------------------------------------------------------------------------- > +dnl QoS (kpkts) max rate > +AT_SETUP([OVS-DPDK - QoS (kpkts) max rate]) > +AT_KEYWORDS([dpdk]) > + > +OVS_DPDK_PRE_CHECK() > +OVS_DPDK_START() > + > +dnl Add userspace bridge and attach it to OVS and add egress policer > +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) > +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) > +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=42949671]) > +sleep 2 > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) > + > +dnl Check egress policer was not created > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) > + > +dnl Clean up > +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) > +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ > +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d > +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d > +])") > +AT_CLEANUP > +dnl -------------------------------------------------------------------------- > + > + > +dnl -------------------------------------------------------------------------- > +dnl QoS (kpkts) max burst > +AT_SETUP([OVS-DPDK - QoS (kpkts) max burst]) > +AT_KEYWORDS([dpdk]) > + > +OVS_DPDK_PRE_CHECK() > +OVS_DPDK_START() > + > +dnl Add userspace bridge and attach it to OVS and add egress policer > +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) > +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) > +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_burst=42949671]) > +sleep 2 > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) > + > +dnl Check egress policer was not created > +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) > +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) > + > +dnl Clean up > +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) > +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ > +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d > +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d > +])") > +AT_CLEANUP > +dnl -------------------------------------------------------------------------- > + > + > +dnl -------------------------------------------------------------------------- > +dnl QoS (kpkts) testpmd flowgen test > +AT_SETUP([OVS-DPDK - QoS (kpkts) police]) > +AT_KEYWORDS([dpdk]) > + > +OVS_DPDK_PRE_CHECK() > +AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) > +OVS_DPDK_START([--no-pci]) > + > +dnl Find number of sockets > +AT_CHECK([lscpu], [], [stdout]) > +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) > + > +dnl Add userspace bridge and attach it to OVS > +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) > + > +dnl Parse log file > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser], [], [stdout], [stderr]) > +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser], [], [stdout], [stderr]) > +AT_CHECK([ovs-vsctl show], [], [stdout]) > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser0) vhost-user server: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "Socket $OVS_RUNDIR/dpdkvhostuser0 created for vhost-user port dpdkvhostuser0" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser0) binding succeeded" ovs-vswitchd.log], [], [stdout]) > + > +dnl Parse log file > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser1) vhost-user server: socket created" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "Socket $OVS_RUNDIR/dpdkvhostuser1 created for vhost-user port dpdkvhostuser1" ovs-vswitchd.log], [], [stdout]) > +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser1) binding succeeded" ovs-vswitchd.log], [], [stdout]) > + > +dnl Configure the same QoS for both ports. > +AT_CHECK([ovs-vsctl set port dpdkvhostuser0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=1 other-config:kpkts_burst=1], [0], [ignore]) > + > +dnl add flows, only send packets from dpdkvhostuser1 to dpdkvhostuser0. > +AT_DATA([flows.txt], [dnl > +priority=100,in_port=dpdkvhostuser1,ip,actions=dpdkvhostuser0 > +]) > + > +AT_CHECK([ovs-ofctl del-flows br10]) > +AT_CHECK([ovs-ofctl add-flows br10 flows.txt]) > + > +dnl Execute testpmd in background > +on_exit "pkill -f -x -9 'tail -f /dev/null'" > +tail -f /dev/null | dpdk-testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ > + --vdev="net_virtio_user0,path=$OVS_RUNDIR/dpdkvhostuser0" \ > + --vdev="net_virtio_user1,path=$OVS_RUNDIR/dpdkvhostuser1" \ > + --single-file-segments -- --forward-mode=flowgen -a > $OVS_RUNDIR/testpmd-dpdkvhostuser.log 2>&1 & > + > +dnl sent packet 10 second. > +AT_CHECK([sleep 10]) > + > +dnl Clean up the testpmd now > +pkill -f -x -9 'tail -f /dev/null' > + > +dnl ---------------------- Forward statistics for port 0 ---------------------- > +dnl RX-packets: 9911 RX-dropped: 0 RX-total: 9911 > +dnl TX-packets: 15937632 TX-dropped: 226661984 TX-total: 242599616 > +dnl ---------------------------------------------------------------------------- > +port0_rx_packets=`cat testpmd-dpdkvhostuser.log | grep "Forward statistics for port 0" -A 1 | grep "RX-packets:" | awk '{print $2}'` > +echo "port0_rx_packets=$port0_rx_packets" > + > +AT_CHECK([test $port0_rx_packets -lt 10500]) > +AT_CHECK([test $port0_rx_packets -gt 9500]) > + > +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ > +\@dpdkvhostuser ports are considered deprecated; please migrate to dpdkvhostuserclient ports.@d > +])") > +AT_CLEANUP > +dnl -------------------------------------------------------------------------- > + > + > > dnl -------------------------------------------------------------------------- > dnl MTU increase phy port > diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml > index cfcde34ff..19fc13873 100644 > --- a/vswitchd/vswitch.xml > +++ b/vswitchd/vswitch.xml > @@ -4874,6 +4874,19 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \ > created with the same <code>other_config</code> values as the > physical port. > </dd> > + <dt><code>kpkts-policer</code></dt> > + <dd> > + A DPDK egress packet per second policer algorithm using the ovs > + token bucket library. The token bucket library provides an > + implementation which allows the policing of packets traffic. The > + implementation in OVS essentially creates a single token bucket used > + to police traffic. It should be noted that when the token bucket is > + configured as part of QoS there will be a performance overhead as the > + token bucket itself will consume CPU cycles in order to police > + traffic. These CPU cycles ordinarily are used for packet proccessing. > + As such the drop in performance will be noticed in terms of overall > + aggregate traffic throughput. > + </dd> > </dl> > </column> > > @@ -4966,6 +4979,25 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \ > </column> > </group> > > + <group title="Configuration for kpkts-policer QoS"> > + <p> > + <ref table="QoS"/> <ref table="QoS" column="type"/> > + <code>kpkts-policer</code> provides egress pkts policing for userspace > + port types with DPDK. > + > + It has the following key-value pairs defined. > + </p> > + > + <column name="other_config" key="kpkts_rate" type='{"type": "integer"}'> > + The Kilo Packets Per Second (kpps) represents the packet per second > + rate at which the token bucket will be updated. > + </column> > + <column name="other_config" key="kpkts_burst" type='{"type": "integer"}'> > + The Packets Per Second Burst Size is measured in count and represents a > + token bucket. > + </column> > + </group> > + > <group title="Configuration for linux-sfq"> > <p> > The <code>linux-sfq</code> QoS supports the following key-value pairs:
Hi Aaron, Thanks for reviewing my code. I will update a new patch later. On 8/1/2023 4:10 AM, Aaron Conole wrote: > miterv@outlook.com writes: > >> From: Lin Huang <linhuang@ruijie.com.cn> >> >> OvS has supported packet-per-second policer which can be set at ingress >> and egress side in kernel datapath. But the userspace datapath doesn't >> support for ingress and egress packet-per-second policing now. >> >> So, this patch add support for userspace egress pps policing by using >> native ovs token bucket library. Token bucket is accumulated by 'rate' >> tokens per millisecond and store maximum tokens at 'burst' bucket size. >> One token in the bucket means one packet (1 kpkts * millisecond) which >> will drop or pass by policer. >> >> This patch add a new egress Qos type called 'kpkts-policer'. >> the policer police the kilo-packet per second at which the token bucket >> be updated by 'kpkts_rate'. and the policer's burst size is defined by >> 'kpkts_burst'. >> >> Examples: >> $ovs-vsctl set port vhost-user0 qos=@newqos -- >> --id=@newqos create qos type=kpkts-policer \ >> other-config:kpkts_rate=123 other-config:kpkts_burst=123 >> >> Add some unit tests for egress packet-per-second policing. >> >> Signed-off-by: Lin Huang <linhuang@ruijie.com.cn> >> --- >> Documentation/topics/dpdk/qos.rst | 21 +++ >> NEWS | 1 + >> lib/netdev-dpdk.c | 159 +++++++++++++++++++ >> tests/system-dpdk.at | 255 ++++++++++++++++++++++++++++++ >> vswitchd/vswitch.xml | 32 ++++ >> 5 files changed, 468 insertions(+) >> >> diff --git a/Documentation/topics/dpdk/qos.rst b/Documentation/topics/dpdk/qos.rst >> index a98ec672f..6a4408127 100644 >> --- a/Documentation/topics/dpdk/qos.rst >> +++ b/Documentation/topics/dpdk/qos.rst >> @@ -36,6 +36,9 @@ QoS (Egress Policing) >> Single Queue Policer >> ~~~~~~~~~~~~~~~~~~~~ >> >> +Bytes Per Second Policer >> ++++++++++++++++++++ >> + >> Assuming you have a :doc:`vhost-user port <vhost-user>` transmitting traffic >> consisting of packets of size 64 bytes, the following command would limit the >> egress transmission rate of the port to ~1,000,000 packets per second:: >> @@ -52,6 +55,24 @@ To clear the QoS configuration from the port and ovsdb, run:: >> >> $ ovs-vsctl destroy QoS vhost-user0 -- clear Port vhost-user0 qos >> >> +Packets Per Second Policer >> ++++++++++++++++++++ >> + >> +Assuming you have a :doc:`vhost-user port <vhost-user>` transmitting traffic, >> +the following command would limit the egress transmission rate of the port to >> +~1,000,000 packets per second:: >> + >> + ovs-vsctl set port vhost-user0 qos=@newqos -- \ >> + --id=@newqos create qos type=kpkts-policer \ >> + other-config:kpkts_rate=1000 other-config:kpkts_burst=1000 >> + >> +To examine the QoS configuration of the port, run:: >> + >> + $ ovs-appctl -t ovs-vswitchd qos/show vhost-user0 >> + >> +To clear the QoS configuration from the port and ovsdb, run:: >> + >> + $ ovs-vsctl destroy QoS vhost-user0 -- clear Port vhost-user0 qos >> >> Multi Queue Policer >> ~~~~~~~~~~~~~~~~~~~ >> diff --git a/NEWS b/NEWS >> index 7a852427e..3d1ab282e 100644 >> --- a/NEWS >> +++ b/NEWS >> @@ -63,6 +63,7 @@ v3.2.0 - xx xxx xxxx >> * 'ovs-appctl dpif-netdev/pmd-sleep-show' command was added to get the >> max sleep configuration of PMD thread cores. >> * Removed experimental tag from PMD load based sleeping. >> + * Added new Qos type 'pkts-policer' to support kilo packet-per-second policing. >> - Linux TC offload: >> * Add support for offloading VXLAN tunnels with the GBP extensions. >> - Python >> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c >> index 7e3020a7d..e6b8922aa 100644 >> --- a/lib/netdev-dpdk.c >> +++ b/lib/netdev-dpdk.c >> @@ -19,6 +19,7 @@ >> >> #include <errno.h> >> #include <signal.h> >> +#include <stdint.h> >> #include <stdlib.h> >> #include <string.h> >> #include <unistd.h> >> @@ -59,6 +60,7 @@ >> #include "openvswitch/ofp-parse.h" >> #include "openvswitch/ofp-print.h" >> #include "openvswitch/shash.h" >> +#include "openvswitch/token-bucket.h" >> #include "openvswitch/vlog.h" >> #include "ovs-numa.h" >> #include "ovs-rcu.h" >> @@ -91,6 +93,8 @@ static bool per_port_memory = false; /* Status of per port memory support */ >> #define OVS_CACHE_LINE_SIZE CACHE_LINE_SIZE >> #define OVS_VPORT_DPDK "ovs_dpdk" >> >> +#define MAX_KPKTS_PARAMETER 4294967U /* UINT32_MAX / 1000 */ >> + >> /* >> * need to reserve tons of extra space in the mbufs so we can align the >> * DMA addresses to 4KB. >> @@ -346,6 +350,7 @@ struct dpdk_qos_ops { >> /* dpdk_qos_ops for each type of user space QoS implementation. */ >> static const struct dpdk_qos_ops egress_policer_ops; >> static const struct dpdk_qos_ops trtcm_policer_ops; >> +static const struct dpdk_qos_ops kpkts_policer_ops; >> >> /* >> * Array of dpdk_qos_ops, contains pointer to all supported QoS >> @@ -354,6 +359,7 @@ static const struct dpdk_qos_ops trtcm_policer_ops; >> static const struct dpdk_qos_ops *const qos_confs[] = { >> &egress_policer_ops, >> &trtcm_policer_ops, >> + &kpkts_policer_ops, >> NULL >> }; >> >> @@ -5572,6 +5578,159 @@ static const struct dpdk_qos_ops trtcm_policer_ops = { >> .qos_queue_dump_state_init = trtcm_policer_qos_queue_dump_state_init >> }; >> >> +/* kpkts-policer details */ >> +struct kpkts_policer { >> + struct qos_conf qos_conf; >> + struct token_bucket tb; >> + uint32_t kpkts_rate; >> + uint32_t kpkts_burst; >> +}; >> + >> +static int >> +kpkts_policer_run_single_packet(struct token_bucket *tb, struct rte_mbuf **pkts, >> + int pkt_cnt, bool should_steal) >> +{ >> + struct rte_mbuf *batch[NETDEV_MAX_BURST] = {0}; >> + long long int now = time_msec(); >> + struct rte_mbuf *pkt = NULL; >> + int i = 0, n = 0; >> + int cnt = 0; >> + >> + for (i = 0; i < pkt_cnt; i++) { >> + pkt = pkts[i]; >> + /* Handle current packet. */ >> + if (token_bucket_withdraw(tb, 1, now)) { >> + /* Count passed packets. */ >> + if (cnt != i) { >> + pkts[cnt] = pkt; >> + } >> + cnt++; >> + } else { >> + /* Count dropped packets. */ >> + batch[n++] = pkt; >> + } >> + } >> + >> + if (should_steal && n) { >> + rte_pktmbuf_free_bulk(batch, n); >> + } >> + >> + return cnt; >> +} >> + >> +static int >> +kpkts_policer_profile_config(struct token_bucket *tb, >> + uint32_t kpkts_rate, uint32_t kpkts_burst) >> +{ >> + if (kpkts_rate > MAX_KPKTS_PARAMETER || >> + kpkts_burst > MAX_KPKTS_PARAMETER) { >> + return EINVAL; >> + } >> + >> + /* Rate in kilo-packets/second, bucket 1000 packets. >> + * msec * kilo-packets/sec = 1 packets. */ >> + if (kpkts_rate) { >> + /* Parameters between (1 ~ MAX_KPKTS_PARAMETER). */ >> + token_bucket_init(tb, kpkts_rate, kpkts_burst * 1000); >> + } else { >> + /* Zero means not to police the traffic. */ >> + return EINVAL; >> + } >> + >> + return 0; >> +} >> + >> +static int >> +kpkts_policer_qos_construct(const struct smap *details, struct qos_conf **conf) >> +{ >> + uint32_t kpkts_rate, kpkts_burst; >> + struct kpkts_policer *policer; >> + int err; >> + >> + policer = xmalloc(sizeof *policer); >> + kpkts_rate = smap_get_uint(details, "kpkts_rate", 0); >> + kpkts_burst = smap_get_uint(details, "kpkts_burst", 0); >> + >> + /* >> + * Force to 0 if no rate specified, >> + * default to rate if burst is 0, >> + * else stick with user-specified value. >> + */ >> + kpkts_burst = (!kpkts_rate ? 0 : !kpkts_burst ? kpkts_rate : kpkts_burst); >> + >> + qos_conf_init(&policer->qos_conf, &kpkts_policer_ops); >> + err = kpkts_policer_profile_config(&policer->tb, kpkts_rate, kpkts_burst); >> + if (err) { >> + VLOG_DBG("Could not create token bucket for egress policer"); > I didn't do an in-depth look at this yet, just noticed this needs free > here for policer object. > >> + return err; >> + } >> + >> + policer->kpkts_rate = kpkts_rate; >> + policer->kpkts_burst = kpkts_burst; >> + >> + *conf = &policer->qos_conf; >> + >> + return err; >> +} >> + >> +static void >> +kpkts_policer_qos_destruct(struct qos_conf *conf) >> +{ >> + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, >> + qos_conf); >> + >> + free(policer); >> +} >> + >> +static int >> +kpkts_policer_qos_get(const struct qos_conf *conf, struct smap *details) >> +{ >> + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, >> + qos_conf); >> + >> + smap_add_format(details, "kpkts_rate", "%"PRIu32, policer->kpkts_rate); >> + smap_add_format(details, "kpkts_burst", "%"PRIu32, policer->kpkts_burst); >> + >> + return 0; >> +} >> + >> +static bool >> +kpkts_pkts_policer_qos_is_equal(const struct qos_conf *conf, >> + const struct smap *details) >> +{ >> + uint32_t rate, burst; >> + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, >> + qos_conf); >> + >> + rate = smap_get_uint(details, "pkts_rate", 0); >> + burst = smap_get_uint(details, "pkts_burst", 0); >> + >> + return (policer->tb.rate == rate && policer->tb.burst == burst); >> +} >> + >> +static int >> +kpkts_policer_run(struct qos_conf *conf, struct rte_mbuf **pkts, int pkt_cnt, >> + bool should_steal) >> +{ >> + int cnt; >> + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, >> + qos_conf); >> + >> + cnt = kpkts_policer_run_single_packet(&policer->tb, pkts, pkt_cnt, >> + should_steal); >> + >> + return cnt; >> +} >> + >> +static const struct dpdk_qos_ops kpkts_policer_ops = { >> + .qos_name = "kpkts-policer", >> + .qos_construct = kpkts_policer_qos_construct, >> + .qos_destruct = kpkts_policer_qos_destruct, >> + .qos_get = kpkts_policer_qos_get, >> + .qos_is_equal = kpkts_pkts_policer_qos_is_equal, >> + .qos_run = kpkts_policer_run >> +}; >> + >> static int >> dpdk_rx_steer_add_flow(struct netdev_dpdk *dev, >> const struct rte_flow_item items[], >> diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at >> index 0f58e8574..8b80a31e6 100644 >> --- a/tests/system-dpdk.at >> +++ b/tests/system-dpdk.at >> @@ -570,6 +570,261 @@ dnl -------------------------------------------------------------------------- >> >> >> >> +dnl -------------------------------------------------------------------------- >> +dnl QoS (kpkts) create delete vport port >> +AT_SETUP([OVS-DPDK - QoS (kpkts) create delete vport port]) >> +AT_KEYWORDS([dpdk]) >> + >> +OVS_DPDK_PRE_CHECK() >> +OVS_DPDK_START() >> + >> +dnl Add userspace bridge and attach it to OVS and add egress policer >> +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) >> +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) >> +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=123 other-config:kpkts_burst=456]) >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +sleep 2 >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Remove egress policer >> +AT_CHECK([ovs-vsctl destroy QoS dpdkvhostuserclient0 -- clear Port dpdkvhostuserclient0 qos]) >> + >> +dnl Check egress policer was removed correctly >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) >> + >> +dnl Clean up >> +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) >> +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ >> +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d >> +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d >> +])") >> +AT_CLEANUP >> +dnl -------------------------------------------------------------------------- >> + >> + >> + >> +dnl -------------------------------------------------------------------------- >> +dnl QoS (kpkts) no rate >> +AT_SETUP([OVS-DPDK - QoS (kpkts) no rate]) >> +AT_KEYWORDS([dpdk]) >> + >> +OVS_DPDK_PRE_CHECK() >> +OVS_DPDK_START() >> + >> +dnl Add userspace bridge and attach it to OVS and add egress policer >> +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) >> +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) >> +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_burst=123]) >> +sleep 2 >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Check egress policer was not created >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) >> + >> +dnl Clean up >> +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) >> +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ >> +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d >> +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d >> +])") >> +AT_CLEANUP >> +dnl -------------------------------------------------------------------------- >> + >> + >> + >> +dnl -------------------------------------------------------------------------- >> +dnl QoS (kpkts) no burst >> +AT_SETUP([OVS-DPDK - QoS (kpkts) no burst]) >> +AT_KEYWORDS([dpdk]) >> + >> +OVS_DPDK_PRE_CHECK() >> +OVS_DPDK_START() >> + >> +dnl Add userspace bridge and attach it to OVS and add egress policer >> +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) >> +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) >> +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=123]) >> +sleep 2 >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Check egress policer was created >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +AT_CHECK([grep -E 'kpkts_rate: 123' stdout], [], [stdout]) >> + >> +dnl Check egress policer was deleted >> +QOS_UUID=`ovs-vsctl get port dpdkvhostuserclient0 qos` >> +AT_CHECK([ovs-vsctl set qos $QOS_UUID other_config:kpkts_rate=0]) >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) >> + >> +dnl Clean up >> +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) >> +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ >> +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d >> +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d >> +])") >> +AT_CLEANUP >> +dnl -------------------------------------------------------------------------- >> + >> + >> + >> +dnl -------------------------------------------------------------------------- >> +dnl QoS (kpkts) max rate >> +AT_SETUP([OVS-DPDK - QoS (kpkts) max rate]) >> +AT_KEYWORDS([dpdk]) >> + >> +OVS_DPDK_PRE_CHECK() >> +OVS_DPDK_START() >> + >> +dnl Add userspace bridge and attach it to OVS and add egress policer >> +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) >> +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) >> +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=42949671]) >> +sleep 2 >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Check egress policer was not created >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) >> + >> +dnl Clean up >> +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) >> +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ >> +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d >> +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d >> +])") >> +AT_CLEANUP >> +dnl -------------------------------------------------------------------------- >> + >> + >> +dnl -------------------------------------------------------------------------- >> +dnl QoS (kpkts) max burst >> +AT_SETUP([OVS-DPDK - QoS (kpkts) max burst]) >> +AT_KEYWORDS([dpdk]) >> + >> +OVS_DPDK_PRE_CHECK() >> +OVS_DPDK_START() >> + >> +dnl Add userspace bridge and attach it to OVS and add egress policer >> +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) >> +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) >> +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_burst=42949671]) >> +sleep 2 >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Check egress policer was not created >> +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) >> +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) >> + >> +dnl Clean up >> +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) >> +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ >> +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d >> +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d >> +])") >> +AT_CLEANUP >> +dnl -------------------------------------------------------------------------- >> + >> + >> +dnl -------------------------------------------------------------------------- >> +dnl QoS (kpkts) testpmd flowgen test >> +AT_SETUP([OVS-DPDK - QoS (kpkts) police]) >> +AT_KEYWORDS([dpdk]) >> + >> +OVS_DPDK_PRE_CHECK() >> +AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) >> +OVS_DPDK_START([--no-pci]) >> + >> +dnl Find number of sockets >> +AT_CHECK([lscpu], [], [stdout]) >> +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) >> + >> +dnl Add userspace bridge and attach it to OVS >> +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) >> + >> +dnl Parse log file >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser], [], [stdout], [stderr]) >> +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser], [], [stdout], [stderr]) >> +AT_CHECK([ovs-vsctl show], [], [stdout]) >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser0) vhost-user server: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "Socket $OVS_RUNDIR/dpdkvhostuser0 created for vhost-user port dpdkvhostuser0" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser0) binding succeeded" ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Parse log file >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser1) vhost-user server: socket created" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "Socket $OVS_RUNDIR/dpdkvhostuser1 created for vhost-user port dpdkvhostuser1" ovs-vswitchd.log], [], [stdout]) >> +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser1) binding succeeded" ovs-vswitchd.log], [], [stdout]) >> + >> +dnl Configure the same QoS for both ports. >> +AT_CHECK([ovs-vsctl set port dpdkvhostuser0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=1 other-config:kpkts_burst=1], [0], [ignore]) >> + >> +dnl add flows, only send packets from dpdkvhostuser1 to dpdkvhostuser0. >> +AT_DATA([flows.txt], [dnl >> +priority=100,in_port=dpdkvhostuser1,ip,actions=dpdkvhostuser0 >> +]) >> + >> +AT_CHECK([ovs-ofctl del-flows br10]) >> +AT_CHECK([ovs-ofctl add-flows br10 flows.txt]) >> + >> +dnl Execute testpmd in background >> +on_exit "pkill -f -x -9 'tail -f /dev/null'" >> +tail -f /dev/null | dpdk-testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ >> + --vdev="net_virtio_user0,path=$OVS_RUNDIR/dpdkvhostuser0" \ >> + --vdev="net_virtio_user1,path=$OVS_RUNDIR/dpdkvhostuser1" \ >> + --single-file-segments -- --forward-mode=flowgen -a > $OVS_RUNDIR/testpmd-dpdkvhostuser.log 2>&1 & >> + >> +dnl sent packet 10 second. >> +AT_CHECK([sleep 10]) >> + >> +dnl Clean up the testpmd now >> +pkill -f -x -9 'tail -f /dev/null' >> + >> +dnl ---------------------- Forward statistics for port 0 ---------------------- >> +dnl RX-packets: 9911 RX-dropped: 0 RX-total: 9911 >> +dnl TX-packets: 15937632 TX-dropped: 226661984 TX-total: 242599616 >> +dnl ---------------------------------------------------------------------------- >> +port0_rx_packets=`cat testpmd-dpdkvhostuser.log | grep "Forward statistics for port 0" -A 1 | grep "RX-packets:" | awk '{print $2}'` >> +echo "port0_rx_packets=$port0_rx_packets" >> + >> +AT_CHECK([test $port0_rx_packets -lt 10500]) >> +AT_CHECK([test $port0_rx_packets -gt 9500]) >> + >> +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ >> +\@dpdkvhostuser ports are considered deprecated; please migrate to dpdkvhostuserclient ports.@d >> +])") >> +AT_CLEANUP >> +dnl -------------------------------------------------------------------------- >> + >> + >> >> dnl -------------------------------------------------------------------------- >> dnl MTU increase phy port >> diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml >> index cfcde34ff..19fc13873 100644 >> --- a/vswitchd/vswitch.xml >> +++ b/vswitchd/vswitch.xml >> @@ -4874,6 +4874,19 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \ >> created with the same <code>other_config</code> values as the >> physical port. >> </dd> >> + <dt><code>kpkts-policer</code></dt> >> + <dd> >> + A DPDK egress packet per second policer algorithm using the ovs >> + token bucket library. The token bucket library provides an >> + implementation which allows the policing of packets traffic. The >> + implementation in OVS essentially creates a single token bucket used >> + to police traffic. It should be noted that when the token bucket is >> + configured as part of QoS there will be a performance overhead as the >> + token bucket itself will consume CPU cycles in order to police >> + traffic. These CPU cycles ordinarily are used for packet proccessing. >> + As such the drop in performance will be noticed in terms of overall >> + aggregate traffic throughput. >> + </dd> >> </dl> >> </column> >> >> @@ -4966,6 +4979,25 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \ >> </column> >> </group> >> >> + <group title="Configuration for kpkts-policer QoS"> >> + <p> >> + <ref table="QoS"/> <ref table="QoS" column="type"/> >> + <code>kpkts-policer</code> provides egress pkts policing for userspace >> + port types with DPDK. >> + >> + It has the following key-value pairs defined. >> + </p> >> + >> + <column name="other_config" key="kpkts_rate" type='{"type": "integer"}'> >> + The Kilo Packets Per Second (kpps) represents the packet per second >> + rate at which the token bucket will be updated. >> + </column> >> + <column name="other_config" key="kpkts_burst" type='{"type": "integer"}'> >> + The Packets Per Second Burst Size is measured in count and represents a >> + token bucket. >> + </column> >> + </group> >> + >> <group title="Configuration for linux-sfq"> >> <p> >> The <code>linux-sfq</code> QoS supports the following key-value pairs:
diff --git a/Documentation/topics/dpdk/qos.rst b/Documentation/topics/dpdk/qos.rst index a98ec672f..6a4408127 100644 --- a/Documentation/topics/dpdk/qos.rst +++ b/Documentation/topics/dpdk/qos.rst @@ -36,6 +36,9 @@ QoS (Egress Policing) Single Queue Policer ~~~~~~~~~~~~~~~~~~~~ +Bytes Per Second Policer ++++++++++++++++++++ + Assuming you have a :doc:`vhost-user port <vhost-user>` transmitting traffic consisting of packets of size 64 bytes, the following command would limit the egress transmission rate of the port to ~1,000,000 packets per second:: @@ -52,6 +55,24 @@ To clear the QoS configuration from the port and ovsdb, run:: $ ovs-vsctl destroy QoS vhost-user0 -- clear Port vhost-user0 qos +Packets Per Second Policer ++++++++++++++++++++ + +Assuming you have a :doc:`vhost-user port <vhost-user>` transmitting traffic, +the following command would limit the egress transmission rate of the port to +~1,000,000 packets per second:: + + ovs-vsctl set port vhost-user0 qos=@newqos -- \ + --id=@newqos create qos type=kpkts-policer \ + other-config:kpkts_rate=1000 other-config:kpkts_burst=1000 + +To examine the QoS configuration of the port, run:: + + $ ovs-appctl -t ovs-vswitchd qos/show vhost-user0 + +To clear the QoS configuration from the port and ovsdb, run:: + + $ ovs-vsctl destroy QoS vhost-user0 -- clear Port vhost-user0 qos Multi Queue Policer ~~~~~~~~~~~~~~~~~~~ diff --git a/NEWS b/NEWS index 7a852427e..3d1ab282e 100644 --- a/NEWS +++ b/NEWS @@ -63,6 +63,7 @@ v3.2.0 - xx xxx xxxx * 'ovs-appctl dpif-netdev/pmd-sleep-show' command was added to get the max sleep configuration of PMD thread cores. * Removed experimental tag from PMD load based sleeping. + * Added new Qos type 'pkts-policer' to support kilo packet-per-second policing. - Linux TC offload: * Add support for offloading VXLAN tunnels with the GBP extensions. - Python diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index 7e3020a7d..e6b8922aa 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -19,6 +19,7 @@ #include <errno.h> #include <signal.h> +#include <stdint.h> #include <stdlib.h> #include <string.h> #include <unistd.h> @@ -59,6 +60,7 @@ #include "openvswitch/ofp-parse.h" #include "openvswitch/ofp-print.h" #include "openvswitch/shash.h" +#include "openvswitch/token-bucket.h" #include "openvswitch/vlog.h" #include "ovs-numa.h" #include "ovs-rcu.h" @@ -91,6 +93,8 @@ static bool per_port_memory = false; /* Status of per port memory support */ #define OVS_CACHE_LINE_SIZE CACHE_LINE_SIZE #define OVS_VPORT_DPDK "ovs_dpdk" +#define MAX_KPKTS_PARAMETER 4294967U /* UINT32_MAX / 1000 */ + /* * need to reserve tons of extra space in the mbufs so we can align the * DMA addresses to 4KB. @@ -346,6 +350,7 @@ struct dpdk_qos_ops { /* dpdk_qos_ops for each type of user space QoS implementation. */ static const struct dpdk_qos_ops egress_policer_ops; static const struct dpdk_qos_ops trtcm_policer_ops; +static const struct dpdk_qos_ops kpkts_policer_ops; /* * Array of dpdk_qos_ops, contains pointer to all supported QoS @@ -354,6 +359,7 @@ static const struct dpdk_qos_ops trtcm_policer_ops; static const struct dpdk_qos_ops *const qos_confs[] = { &egress_policer_ops, &trtcm_policer_ops, + &kpkts_policer_ops, NULL }; @@ -5572,6 +5578,159 @@ static const struct dpdk_qos_ops trtcm_policer_ops = { .qos_queue_dump_state_init = trtcm_policer_qos_queue_dump_state_init }; +/* kpkts-policer details */ +struct kpkts_policer { + struct qos_conf qos_conf; + struct token_bucket tb; + uint32_t kpkts_rate; + uint32_t kpkts_burst; +}; + +static int +kpkts_policer_run_single_packet(struct token_bucket *tb, struct rte_mbuf **pkts, + int pkt_cnt, bool should_steal) +{ + struct rte_mbuf *batch[NETDEV_MAX_BURST] = {0}; + long long int now = time_msec(); + struct rte_mbuf *pkt = NULL; + int i = 0, n = 0; + int cnt = 0; + + for (i = 0; i < pkt_cnt; i++) { + pkt = pkts[i]; + /* Handle current packet. */ + if (token_bucket_withdraw(tb, 1, now)) { + /* Count passed packets. */ + if (cnt != i) { + pkts[cnt] = pkt; + } + cnt++; + } else { + /* Count dropped packets. */ + batch[n++] = pkt; + } + } + + if (should_steal && n) { + rte_pktmbuf_free_bulk(batch, n); + } + + return cnt; +} + +static int +kpkts_policer_profile_config(struct token_bucket *tb, + uint32_t kpkts_rate, uint32_t kpkts_burst) +{ + if (kpkts_rate > MAX_KPKTS_PARAMETER || + kpkts_burst > MAX_KPKTS_PARAMETER) { + return EINVAL; + } + + /* Rate in kilo-packets/second, bucket 1000 packets. + * msec * kilo-packets/sec = 1 packets. */ + if (kpkts_rate) { + /* Parameters between (1 ~ MAX_KPKTS_PARAMETER). */ + token_bucket_init(tb, kpkts_rate, kpkts_burst * 1000); + } else { + /* Zero means not to police the traffic. */ + return EINVAL; + } + + return 0; +} + +static int +kpkts_policer_qos_construct(const struct smap *details, struct qos_conf **conf) +{ + uint32_t kpkts_rate, kpkts_burst; + struct kpkts_policer *policer; + int err; + + policer = xmalloc(sizeof *policer); + kpkts_rate = smap_get_uint(details, "kpkts_rate", 0); + kpkts_burst = smap_get_uint(details, "kpkts_burst", 0); + + /* + * Force to 0 if no rate specified, + * default to rate if burst is 0, + * else stick with user-specified value. + */ + kpkts_burst = (!kpkts_rate ? 0 : !kpkts_burst ? kpkts_rate : kpkts_burst); + + qos_conf_init(&policer->qos_conf, &kpkts_policer_ops); + err = kpkts_policer_profile_config(&policer->tb, kpkts_rate, kpkts_burst); + if (err) { + VLOG_DBG("Could not create token bucket for egress policer"); + return err; + } + + policer->kpkts_rate = kpkts_rate; + policer->kpkts_burst = kpkts_burst; + + *conf = &policer->qos_conf; + + return err; +} + +static void +kpkts_policer_qos_destruct(struct qos_conf *conf) +{ + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, + qos_conf); + + free(policer); +} + +static int +kpkts_policer_qos_get(const struct qos_conf *conf, struct smap *details) +{ + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, + qos_conf); + + smap_add_format(details, "kpkts_rate", "%"PRIu32, policer->kpkts_rate); + smap_add_format(details, "kpkts_burst", "%"PRIu32, policer->kpkts_burst); + + return 0; +} + +static bool +kpkts_pkts_policer_qos_is_equal(const struct qos_conf *conf, + const struct smap *details) +{ + uint32_t rate, burst; + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, + qos_conf); + + rate = smap_get_uint(details, "pkts_rate", 0); + burst = smap_get_uint(details, "pkts_burst", 0); + + return (policer->tb.rate == rate && policer->tb.burst == burst); +} + +static int +kpkts_policer_run(struct qos_conf *conf, struct rte_mbuf **pkts, int pkt_cnt, + bool should_steal) +{ + int cnt; + struct kpkts_policer *policer = CONTAINER_OF(conf, struct kpkts_policer, + qos_conf); + + cnt = kpkts_policer_run_single_packet(&policer->tb, pkts, pkt_cnt, + should_steal); + + return cnt; +} + +static const struct dpdk_qos_ops kpkts_policer_ops = { + .qos_name = "kpkts-policer", + .qos_construct = kpkts_policer_qos_construct, + .qos_destruct = kpkts_policer_qos_destruct, + .qos_get = kpkts_policer_qos_get, + .qos_is_equal = kpkts_pkts_policer_qos_is_equal, + .qos_run = kpkts_policer_run +}; + static int dpdk_rx_steer_add_flow(struct netdev_dpdk *dev, const struct rte_flow_item items[], diff --git a/tests/system-dpdk.at b/tests/system-dpdk.at index 0f58e8574..8b80a31e6 100644 --- a/tests/system-dpdk.at +++ b/tests/system-dpdk.at @@ -570,6 +570,261 @@ dnl -------------------------------------------------------------------------- +dnl -------------------------------------------------------------------------- +dnl QoS (kpkts) create delete vport port +AT_SETUP([OVS-DPDK - QoS (kpkts) create delete vport port]) +AT_KEYWORDS([dpdk]) + +OVS_DPDK_PRE_CHECK() +OVS_DPDK_START() + +dnl Add userspace bridge and attach it to OVS and add egress policer +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=123 other-config:kpkts_burst=456]) +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +sleep 2 + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) + +dnl Remove egress policer +AT_CHECK([ovs-vsctl destroy QoS dpdkvhostuserclient0 -- clear Port dpdkvhostuserclient0 qos]) + +dnl Check egress policer was removed correctly +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) + +dnl Clean up +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d +])") +AT_CLEANUP +dnl -------------------------------------------------------------------------- + + + +dnl -------------------------------------------------------------------------- +dnl QoS (kpkts) no rate +AT_SETUP([OVS-DPDK - QoS (kpkts) no rate]) +AT_KEYWORDS([dpdk]) + +OVS_DPDK_PRE_CHECK() +OVS_DPDK_START() + +dnl Add userspace bridge and attach it to OVS and add egress policer +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_burst=123]) +sleep 2 + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) + +dnl Check egress policer was not created +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) + +dnl Clean up +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d +])") +AT_CLEANUP +dnl -------------------------------------------------------------------------- + + + +dnl -------------------------------------------------------------------------- +dnl QoS (kpkts) no burst +AT_SETUP([OVS-DPDK - QoS (kpkts) no burst]) +AT_KEYWORDS([dpdk]) + +OVS_DPDK_PRE_CHECK() +OVS_DPDK_START() + +dnl Add userspace bridge and attach it to OVS and add egress policer +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=123]) +sleep 2 + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) + +dnl Check egress policer was created +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +AT_CHECK([grep -E 'kpkts_rate: 123' stdout], [], [stdout]) + +dnl Check egress policer was deleted +QOS_UUID=`ovs-vsctl get port dpdkvhostuserclient0 qos` +AT_CHECK([ovs-vsctl set qos $QOS_UUID other_config:kpkts_rate=0]) +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) + +dnl Clean up +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d +])") +AT_CLEANUP +dnl -------------------------------------------------------------------------- + + + +dnl -------------------------------------------------------------------------- +dnl QoS (kpkts) max rate +AT_SETUP([OVS-DPDK - QoS (kpkts) max rate]) +AT_KEYWORDS([dpdk]) + +OVS_DPDK_PRE_CHECK() +OVS_DPDK_START() + +dnl Add userspace bridge and attach it to OVS and add egress policer +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=42949671]) +sleep 2 + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) + +dnl Check egress policer was not created +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) + +dnl Clean up +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d +])") +AT_CLEANUP +dnl -------------------------------------------------------------------------- + + +dnl -------------------------------------------------------------------------- +dnl QoS (kpkts) max burst +AT_SETUP([OVS-DPDK - QoS (kpkts) max burst]) +AT_KEYWORDS([dpdk]) + +OVS_DPDK_PRE_CHECK() +OVS_DPDK_START() + +dnl Add userspace bridge and attach it to OVS and add egress policer +AT_CHECK([ovs-appctl vlog/set netdev_dpdk:dbg]) +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuserclient0 -- set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient options:vhost-server-path=$OVS_RUNDIR/dpdkvhostclient0], [], [stdout], [stderr]) +OVS_WAIT_UNTIL([ovs-vsctl set port dpdkvhostuserclient0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_burst=42949671]) +sleep 2 + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) vhost-user client: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "vHost User device 'dpdkvhostuserclient0' created in 'client' mode, using client socket" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) reconnecting..." ovs-vswitchd.log], [], [stdout]) + +dnl Check egress policer was not created +AT_CHECK([ovs-appctl -t ovs-vswitchd qos/show dpdkvhostuserclient0], [], [stdout]) +AT_CHECK([grep -E 'QoS not configured on dpdkvhostuserclient0' stdout], [], [stdout]) + +dnl Clean up +AT_CHECK([ovs-vsctl del-port br10 dpdkvhostuserclient0], [], [stdout], [stderr]) +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ +\@VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostclient0) failed to connect: No such file or directory@d +\@Failed to set QoS type kpkts-policer on port dpdkvhostuserclient0: Invalid argument@d +])") +AT_CLEANUP +dnl -------------------------------------------------------------------------- + + +dnl -------------------------------------------------------------------------- +dnl QoS (kpkts) testpmd flowgen test +AT_SETUP([OVS-DPDK - QoS (kpkts) police]) +AT_KEYWORDS([dpdk]) + +OVS_DPDK_PRE_CHECK() +AT_SKIP_IF([! which dpdk-testpmd >/dev/null 2>/dev/null]) +OVS_DPDK_START([--no-pci]) + +dnl Find number of sockets +AT_CHECK([lscpu], [], [stdout]) +AT_CHECK([cat stdout | grep "NUMA node(s)" | awk '{c=1; while (c++<$(3)) {printf "512,"}; print "512"}' > NUMA_NODE]) + +dnl Add userspace bridge and attach it to OVS +AT_CHECK([ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev]) + +dnl Parse log file +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser], [], [stdout], [stderr]) +AT_CHECK([ovs-vsctl add-port br10 dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser], [], [stdout], [stderr]) +AT_CHECK([ovs-vsctl show], [], [stdout]) + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser0) vhost-user server: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "Socket $OVS_RUNDIR/dpdkvhostuser0 created for vhost-user port dpdkvhostuser0" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser0) binding succeeded" ovs-vswitchd.log], [], [stdout]) + +dnl Parse log file +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser1) vhost-user server: socket created" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "Socket $OVS_RUNDIR/dpdkvhostuser1 created for vhost-user port dpdkvhostuser1" ovs-vswitchd.log], [], [stdout]) +AT_CHECK([grep "VHOST_CONFIG: ($OVS_RUNDIR/dpdkvhostuser1) binding succeeded" ovs-vswitchd.log], [], [stdout]) + +dnl Configure the same QoS for both ports. +AT_CHECK([ovs-vsctl set port dpdkvhostuser0 qos=@newqos -- --id=@newqos create qos type=kpkts-policer other-config:kpkts_rate=1 other-config:kpkts_burst=1], [0], [ignore]) + +dnl add flows, only send packets from dpdkvhostuser1 to dpdkvhostuser0. +AT_DATA([flows.txt], [dnl +priority=100,in_port=dpdkvhostuser1,ip,actions=dpdkvhostuser0 +]) + +AT_CHECK([ovs-ofctl del-flows br10]) +AT_CHECK([ovs-ofctl add-flows br10 flows.txt]) + +dnl Execute testpmd in background +on_exit "pkill -f -x -9 'tail -f /dev/null'" +tail -f /dev/null | dpdk-testpmd --socket-mem="$(cat NUMA_NODE)" --no-pci\ + --vdev="net_virtio_user0,path=$OVS_RUNDIR/dpdkvhostuser0" \ + --vdev="net_virtio_user1,path=$OVS_RUNDIR/dpdkvhostuser1" \ + --single-file-segments -- --forward-mode=flowgen -a > $OVS_RUNDIR/testpmd-dpdkvhostuser.log 2>&1 & + +dnl sent packet 10 second. +AT_CHECK([sleep 10]) + +dnl Clean up the testpmd now +pkill -f -x -9 'tail -f /dev/null' + +dnl ---------------------- Forward statistics for port 0 ---------------------- +dnl RX-packets: 9911 RX-dropped: 0 RX-total: 9911 +dnl TX-packets: 15937632 TX-dropped: 226661984 TX-total: 242599616 +dnl ---------------------------------------------------------------------------- +port0_rx_packets=`cat testpmd-dpdkvhostuser.log | grep "Forward statistics for port 0" -A 1 | grep "RX-packets:" | awk '{print $2}'` +echo "port0_rx_packets=$port0_rx_packets" + +AT_CHECK([test $port0_rx_packets -lt 10500]) +AT_CHECK([test $port0_rx_packets -gt 9500]) + +OVS_VSWITCHD_STOP("m4_join([], [SYSTEM_DPDK_ALLOWED_LOGS], [ +\@dpdkvhostuser ports are considered deprecated; please migrate to dpdkvhostuserclient ports.@d +])") +AT_CLEANUP +dnl -------------------------------------------------------------------------- + + dnl -------------------------------------------------------------------------- dnl MTU increase phy port diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml index cfcde34ff..19fc13873 100644 --- a/vswitchd/vswitch.xml +++ b/vswitchd/vswitch.xml @@ -4874,6 +4874,19 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \ created with the same <code>other_config</code> values as the physical port. </dd> + <dt><code>kpkts-policer</code></dt> + <dd> + A DPDK egress packet per second policer algorithm using the ovs + token bucket library. The token bucket library provides an + implementation which allows the policing of packets traffic. The + implementation in OVS essentially creates a single token bucket used + to police traffic. It should be noted that when the token bucket is + configured as part of QoS there will be a performance overhead as the + token bucket itself will consume CPU cycles in order to police + traffic. These CPU cycles ordinarily are used for packet proccessing. + As such the drop in performance will be noticed in terms of overall + aggregate traffic throughput. + </dd> </dl> </column> @@ -4966,6 +4979,25 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \ </column> </group> + <group title="Configuration for kpkts-policer QoS"> + <p> + <ref table="QoS"/> <ref table="QoS" column="type"/> + <code>kpkts-policer</code> provides egress pkts policing for userspace + port types with DPDK. + + It has the following key-value pairs defined. + </p> + + <column name="other_config" key="kpkts_rate" type='{"type": "integer"}'> + The Kilo Packets Per Second (kpps) represents the packet per second + rate at which the token bucket will be updated. + </column> + <column name="other_config" key="kpkts_burst" type='{"type": "integer"}'> + The Packets Per Second Burst Size is measured in count and represents a + token bucket. + </column> + </group> + <group title="Configuration for linux-sfq"> <p> The <code>linux-sfq</code> QoS supports the following key-value pairs: