@@ -1279,26 +1279,6 @@ common_construct(struct netdev *netdev, dpdk_port_t port_no,
return 0;
}
-/* Get the number of OVS interfaces which have the same DPDK
- * rte device (e.g. same pci bus address).
- * FIXME: avoid direct access to DPDK internal array rte_eth_devices.
- */
-static int
-netdev_dpdk_get_num_ports(struct rte_device *device)
- OVS_REQUIRES(dpdk_mutex)
-{
- struct netdev_dpdk *dev;
- int count = 0;
-
- LIST_FOR_EACH (dev, list_node, &dpdk_list) {
- if (rte_eth_devices[dev->port_id].device == device
- && rte_eth_devices[dev->port_id].state != RTE_ETH_DEV_UNUSED) {
- count++;
- }
- }
- return count;
-}
-
static int
vhost_common_construct(struct netdev *netdev)
OVS_REQUIRES(dpdk_mutex)
@@ -1452,8 +1432,6 @@ static void
netdev_dpdk_destruct(struct netdev *netdev)
{
struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
- struct rte_device *rte_dev;
- struct rte_eth_dev *eth_dev;
ovs_mutex_lock(&dpdk_mutex);
@@ -1461,20 +1439,46 @@ netdev_dpdk_destruct(struct netdev *netdev)
dev->started = false;
if (dev->attached) {
- /* Retrieve eth device data before closing it.
- * FIXME: avoid direct access to DPDK internal array rte_eth_devices.
- */
- eth_dev = &rte_eth_devices[dev->port_id];
- rte_dev = eth_dev->device;
+ bool dpdk_resources_still_used = false;
+ struct rte_eth_dev_info dev_info;
+ dpdk_port_t sibling_port_id;
+
+ /* Check if this netdev has siblings (i.e. shares DPDK resources) among
+ * other OVS netdevs. */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+ RTE_ETH_FOREACH_DEV_SIBLING (sibling_port_id, dev->port_id) {
+#pragma GCC diagnostic pop
+ struct netdev_dpdk *sibling;
+
+ /* RTE_ETH_FOREACH_DEV_SIBLING lists dev->port_id as part of the
+ * loop. */
+ if (sibling_port_id == dev->port_id) {
+ continue;
+ }
+ LIST_FOR_EACH (sibling, list_node, &dpdk_list) {
+ if (sibling->port_id != sibling_port_id) {
+ continue;
+ }
+ dpdk_resources_still_used = true;
+ break;
+ }
+ if (dpdk_resources_still_used) {
+ break;
+ }
+ }
+
+ /* Retrieve eth device data before closing it. */
+ rte_eth_dev_info_get(dev->port_id, &dev_info);
/* Remove the eth device. */
rte_eth_dev_close(dev->port_id);
- /* Remove this rte device and all its eth devices if all the eth
- * devices belonging to the rte device are closed.
- */
- if (!netdev_dpdk_get_num_ports(rte_dev)) {
- int ret = rte_dev_remove(rte_dev);
+ /* Remove the rte device if no associated eth device is used by OVS.
+ * Note: any remaining eth devices associated to this rte device are
+ * closed by DPDK ethdev layer. */
+ if (!dpdk_resources_still_used) {
+ int ret = rte_dev_remove(dev_info.device);
if (ret < 0) {
VLOG_ERR("Device '%s' can not be detached: %s.",
@@ -3793,12 +3797,12 @@ static void
netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
const char *argv[], void *aux OVS_UNUSED)
{
- char *response;
- dpdk_port_t port_id;
- struct netdev_dpdk *dev;
- struct rte_device *rte_dev;
struct ds used_interfaces = DS_EMPTY_INITIALIZER;
+ struct rte_eth_dev_info dev_info;
+ dpdk_port_t sibling_port_id;
+ dpdk_port_t port_id;
bool used = false;
+ char *response;
ovs_mutex_lock(&dpdk_mutex);
@@ -3808,18 +3812,24 @@ netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
goto error;
}
- rte_dev = rte_eth_devices[port_id].device;
ds_put_format(&used_interfaces,
"Device '%s' is being used by the following interfaces:",
argv[1]);
- LIST_FOR_EACH (dev, list_node, &dpdk_list) {
- /* FIXME: avoid direct access to DPDK array rte_eth_devices. */
- if (rte_eth_devices[dev->port_id].device == rte_dev
- && rte_eth_devices[dev->port_id].state != RTE_ETH_DEV_UNUSED) {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+ RTE_ETH_FOREACH_DEV_SIBLING (sibling_port_id, port_id) {
+#pragma GCC diagnostic pop
+ struct netdev_dpdk *dev;
+
+ LIST_FOR_EACH (dev, list_node, &dpdk_list) {
+ if (dev->port_id != sibling_port_id) {
+ continue;
+ }
used = true;
ds_put_format(&used_interfaces, " %s",
netdev_get_name(&dev->up));
+ break;
}
}
@@ -3831,8 +3841,9 @@ netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
}
ds_destroy(&used_interfaces);
+ rte_eth_dev_info_get(port_id, &dev_info);
rte_eth_dev_close(port_id);
- if (rte_dev_remove(rte_dev) < 0) {
+ if (rte_dev_remove(dev_info.device) < 0) {
response = xasprintf("Device '%s' can not be detached", argv[1]);
goto error;
}
Instead of dereferencing DPDK ethdev internals, we can list "sibling" ports [1]: such ports share the underlying rte_device object. This API was experimental so far, but it will go to stable in 21.11 (see [2]) as it underwent no change since introduction in v19.05. For the time being, waive the warning for this experimental API. 1: https://git.dpdk.org/dpdk/commit/?id=7f98942886bc 2: https://patchwork.dpdk.org/project/dpdk/list/?series=18705 Signed-off-by: David Marchand <david.marchand@redhat.com> --- Note: I sent this against the master branch for discussion, but I am fine with re-sending this against dpdk-latest without the #pragma trick. --- lib/netdev-dpdk.c | 95 ++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 42 deletions(-)