| Message ID | 20191111140205.115245.76451.stgit@netdev64 |
|---|---|
| State | Accepted |
| Delegated to: | Ilya Maximets |
| Headers | show |
| Series | [ovs-dev,v6] netdev-dpdk: add support for the RTE_ETH_EVENT_INTR_RESET event | expand |
Any comments on this patch? Cheers, Eelco On 11 Nov 2019, at 15:02, Eelco Chaudron wrote: > Currently, OVS does not register and therefore not handle the > interface reset event from the DPDK framework. This would cause a > problem in cases where a VF is used as an interface, and its > configuration changes. > > As an example in the following scenario the MAC change is not > detected/acted upon until OVS is restarted without the patch applied: > > $ echo 1 > /sys/bus/pci/devices/0000:05:00.1/sriov_numvfs > $ ovs-vsctl add-port ovs_pvp_br0 dpdk0 -- \ > set Interface dpdk0 type=dpdk -- \ > set Interface dpdk0 options:dpdk-devargs=0000:05:0a.0 > > $ ip link set p5p2 vf 0 mac 52:54:00:92:d3:33 > > Requires patch "bridge: Allow manual notifications about interfaces' > updates." > > Signed-off-by: Eelco Chaudron <echaudro@redhat.com> > --- > v5 -> v6: > Moved the if_notifier_manual_report() function till after the > actual reset of the interface to make it more clear that it's > supposed to happen AFTER the reset. > > Removed empty line in the code > > v4 -> v5: > Using new if_notifier_manual_report() API > > v3 -> v4: > Add support for if-notification to make sure set_config() > gets called > > v2 -> v3: > v1 -> v2: > Fixed Ilya's comments > > lib/netdev-dpdk.c | 54 > +++++++++++++++++++++++++++++++++++++++++++++++++++-- > 1 file changed, 52 insertions(+), 2 deletions(-) > > diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c > index 4805783..cb57b96 100644 > --- a/lib/netdev-dpdk.c > +++ b/lib/netdev-dpdk.c > @@ -46,6 +46,7 @@ > #include "dpdk.h" > #include "dpif-netdev.h" > #include "fatal-signal.h" > +#include "if-notifier.h" > #include "netdev-provider.h" > #include "netdev-vport.h" > #include "odp-util.h" > @@ -396,6 +397,8 @@ struct netdev_dpdk { > bool attached; > /* If true, rte_eth_dev_start() was successfully called */ > bool started; > + bool reset_needed; > + /* 1 pad byte here. */ > struct eth_addr hwaddr; > int mtu; > int socket_id; > @@ -1173,6 +1176,8 @@ common_construct(struct netdev *netdev, > dpdk_port_t port_no, > ovsrcu_index_init(&dev->vid, -1); > dev->vhost_reconfigured = false; > dev->attached = false; > + dev->started = false; > + dev->reset_needed = false; > > ovsrcu_init(&dev->qos_conf, NULL); > > @@ -1769,6 +1774,34 @@ netdev_dpdk_process_devargs(struct netdev_dpdk > *dev, > return new_port_id; > } > > +static int > +dpdk_eth_event_callback(dpdk_port_t port_id, enum rte_eth_event_type > type, > + void *param OVS_UNUSED, void *ret_param > OVS_UNUSED) > +{ > + struct netdev_dpdk *dev; > + > + switch ((int) type) { > + case RTE_ETH_EVENT_INTR_RESET: > + ovs_mutex_lock(&dpdk_mutex); > + dev = netdev_dpdk_lookup_by_port_id(port_id); > + if (dev) { > + ovs_mutex_lock(&dev->mutex); > + dev->reset_needed = true; > + netdev_request_reconfigure(&dev->up); > + VLOG_DBG_RL(&rl, "%s: Device reset requested.", > + netdev_get_name(&dev->up)); > + ovs_mutex_unlock(&dev->mutex); > + } > + ovs_mutex_unlock(&dpdk_mutex); > + break; > + > + default: > + /* Ignore all other types. */ > + break; > + } > + return 0; > +} > + > static void > dpdk_set_rxq_config(struct netdev_dpdk *dev, const struct smap *args) > OVS_REQUIRES(dev->mutex) > @@ -3807,6 +3840,8 @@ netdev_dpdk_class_init(void) > /* This function can be called for different classes. The > initialization > * needs to be done only once */ > if (ovsthread_once_start(&once)) { > + int ret; > + > ovs_thread_create("dpdk_watchdog", dpdk_watchdog, NULL); > unixctl_command_register("netdev-dpdk/set-admin-state", > "[netdev] up|down", 1, 2, > @@ -3820,6 +3855,14 @@ netdev_dpdk_class_init(void) > "[netdev]", 0, 1, > netdev_dpdk_get_mempool_info, NULL); > > + ret = rte_eth_dev_callback_register(RTE_ETH_ALL, > + RTE_ETH_EVENT_INTR_RESET, > + dpdk_eth_event_callback, > NULL); > + if (ret != 0) { > + VLOG_ERR("Ethernet device callback register error: %s", > + rte_strerror(-ret)); > + } > + > ovsthread_once_done(&once); > } > > @@ -4180,13 +4223,20 @@ netdev_dpdk_reconfigure(struct netdev *netdev) > && dev->rxq_size == dev->requested_rxq_size > && dev->txq_size == dev->requested_txq_size > && dev->socket_id == dev->requested_socket_id > - && dev->started) { > + && dev->started && !dev->reset_needed) { > /* Reconfiguration is unnecessary */ > > goto out; > } > > - rte_eth_dev_stop(dev->port_id); > + if (dev->reset_needed) { > + rte_eth_dev_reset(dev->port_id); > + if_notifier_manual_report(); > + dev->reset_needed = false; > + } else { > + rte_eth_dev_stop(dev->port_id); > + } > + > dev->started = false; > > err = netdev_dpdk_mempool_configure(dev); > > _______________________________________________ > dev mailing list > dev@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Any comments on this patch? Guess it might be waiting for “[PATCH] bridge: Allow manual notifications about interfaces”. Would like to get this ready for 2.13… Cheers, Eelco > On 11 Nov 2019, at 15:02, Eelco Chaudron wrote: > >> Currently, OVS does not register and therefore not handle the >> interface reset event from the DPDK framework. This would cause a >> problem in cases where a VF is used as an interface, and its >> configuration changes. >> >> As an example in the following scenario the MAC change is not >> detected/acted upon until OVS is restarted without the patch applied: >> >> $ echo 1 > /sys/bus/pci/devices/0000:05:00.1/sriov_numvfs >> $ ovs-vsctl add-port ovs_pvp_br0 dpdk0 -- \ >> set Interface dpdk0 type=dpdk -- \ >> set Interface dpdk0 options:dpdk-devargs=0000:05:0a.0 >> >> $ ip link set p5p2 vf 0 mac 52:54:00:92:d3:33 >> >> Requires patch "bridge: Allow manual notifications about interfaces' >> updates." >> >> Signed-off-by: Eelco Chaudron <echaudro@redhat.com> >> --- >> v5 -> v6: >> Moved the if_notifier_manual_report() function till after the >> actual reset of the interface to make it more clear that it's >> supposed to happen AFTER the reset. >> >> Removed empty line in the code >> >> v4 -> v5: >> Using new if_notifier_manual_report() API >> >> v3 -> v4: >> Add support for if-notification to make sure set_config() >> gets called >> >> v2 -> v3: >> v1 -> v2: >> Fixed Ilya's comments >> >> lib/netdev-dpdk.c | 54 >> +++++++++++++++++++++++++++++++++++++++++++++++++++-- >> 1 file changed, 52 insertions(+), 2 deletions(-) >> >> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c >> index 4805783..cb57b96 100644 >> --- a/lib/netdev-dpdk.c >> +++ b/lib/netdev-dpdk.c >> @@ -46,6 +46,7 @@ >> #include "dpdk.h" >> #include "dpif-netdev.h" >> #include "fatal-signal.h" >> +#include "if-notifier.h" >> #include "netdev-provider.h" >> #include "netdev-vport.h" >> #include "odp-util.h" >> @@ -396,6 +397,8 @@ struct netdev_dpdk { >> bool attached; >> /* If true, rte_eth_dev_start() was successfully called */ >> bool started; >> + bool reset_needed; >> + /* 1 pad byte here. */ >> struct eth_addr hwaddr; >> int mtu; >> int socket_id; >> @@ -1173,6 +1176,8 @@ common_construct(struct netdev *netdev, >> dpdk_port_t port_no, >> ovsrcu_index_init(&dev->vid, -1); >> dev->vhost_reconfigured = false; >> dev->attached = false; >> + dev->started = false; >> + dev->reset_needed = false; >> >> ovsrcu_init(&dev->qos_conf, NULL); >> >> @@ -1769,6 +1774,34 @@ netdev_dpdk_process_devargs(struct netdev_dpdk >> *dev, >> return new_port_id; >> } >> >> +static int >> +dpdk_eth_event_callback(dpdk_port_t port_id, enum rte_eth_event_type >> type, >> + void *param OVS_UNUSED, void *ret_param >> OVS_UNUSED) >> +{ >> + struct netdev_dpdk *dev; >> + >> + switch ((int) type) { >> + case RTE_ETH_EVENT_INTR_RESET: >> + ovs_mutex_lock(&dpdk_mutex); >> + dev = netdev_dpdk_lookup_by_port_id(port_id); >> + if (dev) { >> + ovs_mutex_lock(&dev->mutex); >> + dev->reset_needed = true; >> + netdev_request_reconfigure(&dev->up); >> + VLOG_DBG_RL(&rl, "%s: Device reset requested.", >> + netdev_get_name(&dev->up)); >> + ovs_mutex_unlock(&dev->mutex); >> + } >> + ovs_mutex_unlock(&dpdk_mutex); >> + break; >> + >> + default: >> + /* Ignore all other types. */ >> + break; >> + } >> + return 0; >> +} >> + >> static void >> dpdk_set_rxq_config(struct netdev_dpdk *dev, const struct smap >> *args) >> OVS_REQUIRES(dev->mutex) >> @@ -3807,6 +3840,8 @@ netdev_dpdk_class_init(void) >> /* This function can be called for different classes. The >> initialization >> * needs to be done only once */ >> if (ovsthread_once_start(&once)) { >> + int ret; >> + >> ovs_thread_create("dpdk_watchdog", dpdk_watchdog, NULL); >> unixctl_command_register("netdev-dpdk/set-admin-state", >> "[netdev] up|down", 1, 2, >> @@ -3820,6 +3855,14 @@ netdev_dpdk_class_init(void) >> "[netdev]", 0, 1, >> netdev_dpdk_get_mempool_info, >> NULL); >> >> + ret = rte_eth_dev_callback_register(RTE_ETH_ALL, >> + >> RTE_ETH_EVENT_INTR_RESET, >> + dpdk_eth_event_callback, >> NULL); >> + if (ret != 0) { >> + VLOG_ERR("Ethernet device callback register error: %s", >> + rte_strerror(-ret)); >> + } >> + >> ovsthread_once_done(&once); >> } >> >> @@ -4180,13 +4223,20 @@ netdev_dpdk_reconfigure(struct netdev >> *netdev) >> && dev->rxq_size == dev->requested_rxq_size >> && dev->txq_size == dev->requested_txq_size >> && dev->socket_id == dev->requested_socket_id >> - && dev->started) { >> + && dev->started && !dev->reset_needed) { >> /* Reconfiguration is unnecessary */ >> >> goto out; >> } >> >> - rte_eth_dev_stop(dev->port_id); >> + if (dev->reset_needed) { >> + rte_eth_dev_reset(dev->port_id); >> + if_notifier_manual_report(); >> + dev->reset_needed = false; >> + } else { >> + rte_eth_dev_stop(dev->port_id); >> + } >> + >> dev->started = false; >> >> err = netdev_dpdk_mempool_configure(dev); >> >> _______________________________________________ >> dev mailing list >> dev@openvswitch.org >> https://mail.openvswitch.org/mailman/listinfo/ovs-dev > > _______________________________________________ > dev mailing list > dev@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev
On 11.11.2019 15:02, Eelco Chaudron wrote: > Currently, OVS does not register and therefore not handle the > interface reset event from the DPDK framework. This would cause a > problem in cases where a VF is used as an interface, and its > configuration changes. > > As an example in the following scenario the MAC change is not > detected/acted upon until OVS is restarted without the patch applied: > > $ echo 1 > /sys/bus/pci/devices/0000:05:00.1/sriov_numvfs > $ ovs-vsctl add-port ovs_pvp_br0 dpdk0 -- \ > set Interface dpdk0 type=dpdk -- \ > set Interface dpdk0 options:dpdk-devargs=0000:05:0a.0 > > $ ip link set p5p2 vf 0 mac 52:54:00:92:d3:33 > > Requires patch "bridge: Allow manual notifications about interfaces' updates." > > Signed-off-by: Eelco Chaudron <echaudro@redhat.com> Thanks! Applied to master. Best regards, Ilya Maximets.
On 18 Dec 2019, at 1:32, Ilya Maximets wrote: > On 11.11.2019 15:02, Eelco Chaudron wrote: >> Currently, OVS does not register and therefore not handle the >> interface reset event from the DPDK framework. This would cause a >> problem in cases where a VF is used as an interface, and its >> configuration changes. >> >> As an example in the following scenario the MAC change is not >> detected/acted upon until OVS is restarted without the patch applied: >> >> $ echo 1 > /sys/bus/pci/devices/0000:05:00.1/sriov_numvfs >> $ ovs-vsctl add-port ovs_pvp_br0 dpdk0 -- \ >> set Interface dpdk0 type=dpdk -- \ >> set Interface dpdk0 options:dpdk-devargs=0000:05:0a.0 >> >> $ ip link set p5p2 vf 0 mac 52:54:00:92:d3:33 >> >> Requires patch "bridge: Allow manual notifications about interfaces' >> updates." >> >> Signed-off-by: Eelco Chaudron <echaudro@redhat.com> > > Thanks! Applied to master. > > Best regards, Ilya Maximets. Thanks you!
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index 4805783..cb57b96 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -46,6 +46,7 @@ #include "dpdk.h" #include "dpif-netdev.h" #include "fatal-signal.h" +#include "if-notifier.h" #include "netdev-provider.h" #include "netdev-vport.h" #include "odp-util.h" @@ -396,6 +397,8 @@ struct netdev_dpdk { bool attached; /* If true, rte_eth_dev_start() was successfully called */ bool started; + bool reset_needed; + /* 1 pad byte here. */ struct eth_addr hwaddr; int mtu; int socket_id; @@ -1173,6 +1176,8 @@ common_construct(struct netdev *netdev, dpdk_port_t port_no, ovsrcu_index_init(&dev->vid, -1); dev->vhost_reconfigured = false; dev->attached = false; + dev->started = false; + dev->reset_needed = false; ovsrcu_init(&dev->qos_conf, NULL); @@ -1769,6 +1774,34 @@ netdev_dpdk_process_devargs(struct netdev_dpdk *dev, return new_port_id; } +static int +dpdk_eth_event_callback(dpdk_port_t port_id, enum rte_eth_event_type type, + void *param OVS_UNUSED, void *ret_param OVS_UNUSED) +{ + struct netdev_dpdk *dev; + + switch ((int) type) { + case RTE_ETH_EVENT_INTR_RESET: + ovs_mutex_lock(&dpdk_mutex); + dev = netdev_dpdk_lookup_by_port_id(port_id); + if (dev) { + ovs_mutex_lock(&dev->mutex); + dev->reset_needed = true; + netdev_request_reconfigure(&dev->up); + VLOG_DBG_RL(&rl, "%s: Device reset requested.", + netdev_get_name(&dev->up)); + ovs_mutex_unlock(&dev->mutex); + } + ovs_mutex_unlock(&dpdk_mutex); + break; + + default: + /* Ignore all other types. */ + break; + } + return 0; +} + static void dpdk_set_rxq_config(struct netdev_dpdk *dev, const struct smap *args) OVS_REQUIRES(dev->mutex) @@ -3807,6 +3840,8 @@ netdev_dpdk_class_init(void) /* This function can be called for different classes. The initialization * needs to be done only once */ if (ovsthread_once_start(&once)) { + int ret; + ovs_thread_create("dpdk_watchdog", dpdk_watchdog, NULL); unixctl_command_register("netdev-dpdk/set-admin-state", "[netdev] up|down", 1, 2, @@ -3820,6 +3855,14 @@ netdev_dpdk_class_init(void) "[netdev]", 0, 1, netdev_dpdk_get_mempool_info, NULL); + ret = rte_eth_dev_callback_register(RTE_ETH_ALL, + RTE_ETH_EVENT_INTR_RESET, + dpdk_eth_event_callback, NULL); + if (ret != 0) { + VLOG_ERR("Ethernet device callback register error: %s", + rte_strerror(-ret)); + } + ovsthread_once_done(&once); } @@ -4180,13 +4223,20 @@ netdev_dpdk_reconfigure(struct netdev *netdev) && dev->rxq_size == dev->requested_rxq_size && dev->txq_size == dev->requested_txq_size && dev->socket_id == dev->requested_socket_id - && dev->started) { + && dev->started && !dev->reset_needed) { /* Reconfiguration is unnecessary */ goto out; } - rte_eth_dev_stop(dev->port_id); + if (dev->reset_needed) { + rte_eth_dev_reset(dev->port_id); + if_notifier_manual_report(); + dev->reset_needed = false; + } else { + rte_eth_dev_stop(dev->port_id); + } + dev->started = false; err = netdev_dpdk_mempool_configure(dev);
Currently, OVS does not register and therefore not handle the interface reset event from the DPDK framework. This would cause a problem in cases where a VF is used as an interface, and its configuration changes. As an example in the following scenario the MAC change is not detected/acted upon until OVS is restarted without the patch applied: $ echo 1 > /sys/bus/pci/devices/0000:05:00.1/sriov_numvfs $ ovs-vsctl add-port ovs_pvp_br0 dpdk0 -- \ set Interface dpdk0 type=dpdk -- \ set Interface dpdk0 options:dpdk-devargs=0000:05:0a.0 $ ip link set p5p2 vf 0 mac 52:54:00:92:d3:33 Requires patch "bridge: Allow manual notifications about interfaces' updates." Signed-off-by: Eelco Chaudron <echaudro@redhat.com> --- v5 -> v6: Moved the if_notifier_manual_report() function till after the actual reset of the interface to make it more clear that it's supposed to happen AFTER the reset. Removed empty line in the code v4 -> v5: Using new if_notifier_manual_report() API v3 -> v4: Add support for if-notification to make sure set_config() gets called v2 -> v3: v1 -> v2: Fixed Ilya's comments lib/netdev-dpdk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-)