| Message ID | 1356505614-16683-1-git-send-email-gaowanlong@cn.fujitsu.com |
|---|---|
| State | RFC, archived |
| Delegated to: | David Miller |
| Headers | show |
On 12/26/2012 03:06 PM, Wanlong Gao wrote: > Add a cpu notifier to virtio-net, so that we can reset the > virtqueue affinity if the cpu hotplug happens. It improve > the performance through enabling or disabling the virtqueue > affinity after doing cpu hotplug. Hi Wanlong: Thanks for looking at this. > Cc: Rusty Russell <rusty@rustcorp.com.au> > Cc: "Michael S. Tsirkin" <mst@redhat.com> > Cc: Jason Wang <jasowang@redhat.com> > Cc: virtualization@lists.linux-foundation.org > Cc: netdev@vger.kernel.org > Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> > --- > drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- > 1 file changed, 38 insertions(+), 1 deletion(-) > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index a6fcf15..9710cf4 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -26,6 +26,7 @@ > #include <linux/scatterlist.h> > #include <linux/if_vlan.h> > #include <linux/slab.h> > +#include <linux/cpu.h> > > static int napi_weight = 128; > module_param(napi_weight, int, 0444); > @@ -34,6 +35,8 @@ static bool csum = true, gso = true; > module_param(csum, bool, 0444); > module_param(gso, bool, 0444); > > +static bool cpu_hotplug = false; > + > /* FIXME: MTU in config. */ > #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > #define GOOD_COPY_LEN 128 > @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) > vi->affinity_hint_set = false; > } > > +static int virtnet_cpu_callback(struct notifier_block *nfb, > + unsigned long action, void *hcpu) > +{ > + switch(action) { > + case CPU_ONLINE: > + case CPU_ONLINE_FROZEN: > + case CPU_DEAD: > + case CPU_DEAD_FROZEN: > + cpu_hotplug = true; > + break; > + default: > + break; > + } > + return NOTIFY_OK; > +} > + > +static struct notifier_block virtnet_cpu_notifier = { > + .notifier_call = virtnet_cpu_callback, > +}; > + > static void virtnet_get_ringparam(struct net_device *dev, > struct ethtool_ringparam *ring) > { > @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) > */ > static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) > { > - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > + int txq; > + > + if (unlikely(cpu_hotplug == true)) { > + virtnet_set_affinity(netdev_priv(dev), true); > + cpu_hotplug = false; > + } > + Why don't you just do this in callback? btw. Does qemu/kvm support cpu-hotplug now? > + txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > smp_processor_id(); > > while (unlikely(txq >= dev->real_num_tx_queues)) > @@ -1248,6 +1278,8 @@ static void virtnet_del_vqs(struct virtnet_info *vi) > { > struct virtio_device *vdev = vi->vdev; > > + unregister_hotcpu_notifier(&virtnet_cpu_notifier); > + > virtnet_set_affinity(vi, false); > > vdev->config->del_vqs(vdev); > @@ -1372,6 +1404,11 @@ static int init_vqs(struct virtnet_info *vi) > goto err_free; > > virtnet_set_affinity(vi, true); > + > + ret = register_hotcpu_notifier(&virtnet_cpu_notifier); > + if (ret) > + goto err_free; > + > return 0; > > err_free: -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 12/26/2012 06:06 PM, Jason Wang wrote: > On 12/26/2012 03:06 PM, Wanlong Gao wrote: >> Add a cpu notifier to virtio-net, so that we can reset the >> virtqueue affinity if the cpu hotplug happens. It improve >> the performance through enabling or disabling the virtqueue >> affinity after doing cpu hotplug. > > Hi Wanlong: > > Thanks for looking at this. >> Cc: Rusty Russell <rusty@rustcorp.com.au> >> Cc: "Michael S. Tsirkin" <mst@redhat.com> >> Cc: Jason Wang <jasowang@redhat.com> >> Cc: virtualization@lists.linux-foundation.org >> Cc: netdev@vger.kernel.org >> Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> >> --- >> drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- >> 1 file changed, 38 insertions(+), 1 deletion(-) >> >> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c >> index a6fcf15..9710cf4 100644 >> --- a/drivers/net/virtio_net.c >> +++ b/drivers/net/virtio_net.c >> @@ -26,6 +26,7 @@ >> #include <linux/scatterlist.h> >> #include <linux/if_vlan.h> >> #include <linux/slab.h> >> +#include <linux/cpu.h> >> >> static int napi_weight = 128; >> module_param(napi_weight, int, 0444); >> @@ -34,6 +35,8 @@ static bool csum = true, gso = true; >> module_param(csum, bool, 0444); >> module_param(gso, bool, 0444); >> >> +static bool cpu_hotplug = false; >> + >> /* FIXME: MTU in config. */ >> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) >> #define GOOD_COPY_LEN 128 >> @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) >> vi->affinity_hint_set = false; >> } >> >> +static int virtnet_cpu_callback(struct notifier_block *nfb, >> + unsigned long action, void *hcpu) >> +{ >> + switch(action) { >> + case CPU_ONLINE: >> + case CPU_ONLINE_FROZEN: >> + case CPU_DEAD: >> + case CPU_DEAD_FROZEN: >> + cpu_hotplug = true; >> + break; >> + default: >> + break; >> + } >> + return NOTIFY_OK; >> +} >> + >> +static struct notifier_block virtnet_cpu_notifier = { >> + .notifier_call = virtnet_cpu_callback, >> +}; >> + >> static void virtnet_get_ringparam(struct net_device *dev, >> struct ethtool_ringparam *ring) >> { >> @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) >> */ >> static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) >> { >> - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : >> + int txq; >> + >> + if (unlikely(cpu_hotplug == true)) { >> + virtnet_set_affinity(netdev_priv(dev), true); >> + cpu_hotplug = false; >> + } >> + > > Why don't you just do this in callback? Callback can just give us a "hcpu", can't get the virtnet_info from callback. Am I missing something? > > btw. Does qemu/kvm support cpu-hotplug now? From http://www.linux-kvm.org/page/CPUHotPlug, I saw that qemu-kvm can support hotplug but failed to merge to qemu.git, right? Thanks, Wanlong Gao -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Wed, Dec 26, 2012 at 03:06:54PM +0800, Wanlong Gao wrote: > Add a cpu notifier to virtio-net, so that we can reset the > virtqueue affinity if the cpu hotplug happens. It improve > the performance through enabling or disabling the virtqueue > affinity after doing cpu hotplug. > > Cc: Rusty Russell <rusty@rustcorp.com.au> > Cc: "Michael S. Tsirkin" <mst@redhat.com> > Cc: Jason Wang <jasowang@redhat.com> > Cc: virtualization@lists.linux-foundation.org > Cc: netdev@vger.kernel.org > Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> Thanks for looking into this. Some comments: 1. Looks like the logic in virtnet_set_affinity (and in virtnet_select_queue) will not work very well when CPU IDs are not consequitive. This can happen with hot unplug. Maybe we should add a VQ allocator, and defining a per-cpu variable specifying the VQ instead of using CPU ID. 2. The below code seems racy e.g. when CPU is added during device init. 3. using a global cpu_hotplug seems inelegant. In any case we should document what is the meaning of this variable. > --- > drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- > 1 file changed, 38 insertions(+), 1 deletion(-) > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index a6fcf15..9710cf4 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -26,6 +26,7 @@ > #include <linux/scatterlist.h> > #include <linux/if_vlan.h> > #include <linux/slab.h> > +#include <linux/cpu.h> > > static int napi_weight = 128; > module_param(napi_weight, int, 0444); > @@ -34,6 +35,8 @@ static bool csum = true, gso = true; > module_param(csum, bool, 0444); > module_param(gso, bool, 0444); > > +static bool cpu_hotplug = false; > + > /* FIXME: MTU in config. */ > #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > #define GOOD_COPY_LEN 128 > @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) > vi->affinity_hint_set = false; > } > > +static int virtnet_cpu_callback(struct notifier_block *nfb, > + unsigned long action, void *hcpu) > +{ > + switch(action) { > + case CPU_ONLINE: > + case CPU_ONLINE_FROZEN: > + case CPU_DEAD: > + case CPU_DEAD_FROZEN: > + cpu_hotplug = true; > + break; > + default: > + break; > + } > + return NOTIFY_OK; > +} > + > +static struct notifier_block virtnet_cpu_notifier = { > + .notifier_call = virtnet_cpu_callback, > +}; > + > static void virtnet_get_ringparam(struct net_device *dev, > struct ethtool_ringparam *ring) > { > @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) > */ > static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) > { > - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > + int txq; > + > + if (unlikely(cpu_hotplug == true)) { > + virtnet_set_affinity(netdev_priv(dev), true); > + cpu_hotplug = false; > + } > + > + txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > smp_processor_id(); > > while (unlikely(txq >= dev->real_num_tx_queues)) > @@ -1248,6 +1278,8 @@ static void virtnet_del_vqs(struct virtnet_info *vi) > { > struct virtio_device *vdev = vi->vdev; > > + unregister_hotcpu_notifier(&virtnet_cpu_notifier); > + > virtnet_set_affinity(vi, false); > > vdev->config->del_vqs(vdev); > @@ -1372,6 +1404,11 @@ static int init_vqs(struct virtnet_info *vi) > goto err_free; > > virtnet_set_affinity(vi, true); > + > + ret = register_hotcpu_notifier(&virtnet_cpu_notifier); > + if (ret) > + goto err_free; > + > return 0; > > err_free: > -- > 1.8.0 -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Wed, 2012-12-26 at 15:06 +0800, Wanlong Gao wrote: > Add a cpu notifier to virtio-net, so that we can reset the > virtqueue affinity if the cpu hotplug happens. It improve > the performance through enabling or disabling the virtqueue > affinity after doing cpu hotplug. > > Cc: Rusty Russell <rusty@rustcorp.com.au> > Cc: "Michael S. Tsirkin" <mst@redhat.com> > Cc: Jason Wang <jasowang@redhat.com> > Cc: virtualization@lists.linux-foundation.org > Cc: netdev@vger.kernel.org > Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> > --- > drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- > 1 file changed, 38 insertions(+), 1 deletion(-) > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index a6fcf15..9710cf4 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -26,6 +26,7 @@ > #include <linux/scatterlist.h> > #include <linux/if_vlan.h> > #include <linux/slab.h> > +#include <linux/cpu.h> > > static int napi_weight = 128; > module_param(napi_weight, int, 0444); > @@ -34,6 +35,8 @@ static bool csum = true, gso = true; > module_param(csum, bool, 0444); > module_param(gso, bool, 0444); > > +static bool cpu_hotplug = false; > + > /* FIXME: MTU in config. */ > #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > #define GOOD_COPY_LEN 128 > @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) > vi->affinity_hint_set = false; > } > > +static int virtnet_cpu_callback(struct notifier_block *nfb, > + unsigned long action, void *hcpu) > +{ > + switch(action) { > + case CPU_ONLINE: > + case CPU_ONLINE_FROZEN: > + case CPU_DEAD: > + case CPU_DEAD_FROZEN: > + cpu_hotplug = true; > + break; > + default: > + break; > + } > + return NOTIFY_OK; > +} > + > +static struct notifier_block virtnet_cpu_notifier = { > + .notifier_call = virtnet_cpu_callback, > +}; > + > static void virtnet_get_ringparam(struct net_device *dev, > struct ethtool_ringparam *ring) > { > @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) > */ > static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) > { > - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > + int txq; > + > + if (unlikely(cpu_hotplug == true)) { > + virtnet_set_affinity(netdev_priv(dev), true); > + cpu_hotplug = false; > + } > + > + txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > smp_processor_id(); > > while (unlikely(txq >= dev->real_num_tx_queues)) > @@ -1248,6 +1278,8 @@ static void virtnet_del_vqs(struct virtnet_info *vi) > { > struct virtio_device *vdev = vi->vdev; > > + unregister_hotcpu_notifier(&virtnet_cpu_notifier); > + > virtnet_set_affinity(vi, false); > > vdev->config->del_vqs(vdev); > @@ -1372,6 +1404,11 @@ static int init_vqs(struct virtnet_info *vi) > goto err_free; > > virtnet_set_affinity(vi, true); > + > + ret = register_hotcpu_notifier(&virtnet_cpu_notifier); > + if (ret) > + goto err_free; > + > return 0; > > err_free: It looks like this patch assumes virtio_net supports a single instance. Try your patch with two instances, I am pretty sure it wont do very well. It seems to me you need something else than a single boolean. A sequence number for example should be better... -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 12/26/2012 06:19 PM, Wanlong Gao wrote: > On 12/26/2012 06:06 PM, Jason Wang wrote: >> On 12/26/2012 03:06 PM, Wanlong Gao wrote: >>> Add a cpu notifier to virtio-net, so that we can reset the >>> virtqueue affinity if the cpu hotplug happens. It improve >>> the performance through enabling or disabling the virtqueue >>> affinity after doing cpu hotplug. >> Hi Wanlong: >> >> Thanks for looking at this. >>> Cc: Rusty Russell <rusty@rustcorp.com.au> >>> Cc: "Michael S. Tsirkin" <mst@redhat.com> >>> Cc: Jason Wang <jasowang@redhat.com> >>> Cc: virtualization@lists.linux-foundation.org >>> Cc: netdev@vger.kernel.org >>> Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> >>> --- >>> drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- >>> 1 file changed, 38 insertions(+), 1 deletion(-) >>> >>> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c >>> index a6fcf15..9710cf4 100644 >>> --- a/drivers/net/virtio_net.c >>> +++ b/drivers/net/virtio_net.c >>> @@ -26,6 +26,7 @@ >>> #include <linux/scatterlist.h> >>> #include <linux/if_vlan.h> >>> #include <linux/slab.h> >>> +#include <linux/cpu.h> >>> >>> static int napi_weight = 128; >>> module_param(napi_weight, int, 0444); >>> @@ -34,6 +35,8 @@ static bool csum = true, gso = true; >>> module_param(csum, bool, 0444); >>> module_param(gso, bool, 0444); >>> >>> +static bool cpu_hotplug = false; >>> + >>> /* FIXME: MTU in config. */ >>> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) >>> #define GOOD_COPY_LEN 128 >>> @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) >>> vi->affinity_hint_set = false; >>> } >>> >>> +static int virtnet_cpu_callback(struct notifier_block *nfb, >>> + unsigned long action, void *hcpu) >>> +{ >>> + switch(action) { >>> + case CPU_ONLINE: >>> + case CPU_ONLINE_FROZEN: >>> + case CPU_DEAD: >>> + case CPU_DEAD_FROZEN: >>> + cpu_hotplug = true; >>> + break; >>> + default: >>> + break; >>> + } >>> + return NOTIFY_OK; >>> +} >>> + >>> +static struct notifier_block virtnet_cpu_notifier = { >>> + .notifier_call = virtnet_cpu_callback, >>> +}; >>> + >>> static void virtnet_get_ringparam(struct net_device *dev, >>> struct ethtool_ringparam *ring) >>> { >>> @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) >>> */ >>> static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) >>> { >>> - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : >>> + int txq; >>> + >>> + if (unlikely(cpu_hotplug == true)) { >>> + virtnet_set_affinity(netdev_priv(dev), true); >>> + cpu_hotplug = false; >>> + } >>> + >> Why don't you just do this in callback? > Callback can just give us a "hcpu", can't get the virtnet_info from callback. Am I missing something? Well, I think you can just embed the notifier block into virtnet_info, then use something like container_of in the callback to make the notifier per device. This also solve the concern of Eric. >> btw. Does qemu/kvm support cpu-hotplug now? > From http://www.linux-kvm.org/page/CPUHotPlug, I saw that qemu-kvm can support hotplug > but failed to merge to qemu.git, right? Not sure, I just try latest qemu, it even does not have a cpu_set command. Thanks > > Thanks, > Wanlong Gao > -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 12/26/2012 06:46 PM, Michael S. Tsirkin wrote: > On Wed, Dec 26, 2012 at 03:06:54PM +0800, Wanlong Gao wrote: >> Add a cpu notifier to virtio-net, so that we can reset the >> virtqueue affinity if the cpu hotplug happens. It improve >> the performance through enabling or disabling the virtqueue >> affinity after doing cpu hotplug. >> >> Cc: Rusty Russell <rusty@rustcorp.com.au> >> Cc: "Michael S. Tsirkin" <mst@redhat.com> >> Cc: Jason Wang <jasowang@redhat.com> >> Cc: virtualization@lists.linux-foundation.org >> Cc: netdev@vger.kernel.org >> Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> > Thanks for looking into this. > Some comments: > > 1. Looks like the logic in > virtnet_set_affinity (and in virtnet_select_queue) > will not work very well when CPU IDs are not > consequitive. This can happen with hot unplug. > > Maybe we should add a VQ allocator, and defining > a per-cpu variable specifying the VQ instead > of using CPU ID. Yes, and generate the affinity hint based on the mapping. Btw, what does VQ allocator means here? > > > 2. The below code seems racy e.g. when CPU is added > during device init. > > 3. using a global cpu_hotplug seems inelegant. > In any case we should document what is the > meaning of this variable. > >> --- >> drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- >> 1 file changed, 38 insertions(+), 1 deletion(-) >> >> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c >> index a6fcf15..9710cf4 100644 >> --- a/drivers/net/virtio_net.c >> +++ b/drivers/net/virtio_net.c >> @@ -26,6 +26,7 @@ >> #include <linux/scatterlist.h> >> #include <linux/if_vlan.h> >> #include <linux/slab.h> >> +#include <linux/cpu.h> >> >> static int napi_weight = 128; >> module_param(napi_weight, int, 0444); >> @@ -34,6 +35,8 @@ static bool csum = true, gso = true; >> module_param(csum, bool, 0444); >> module_param(gso, bool, 0444); >> >> +static bool cpu_hotplug = false; >> + >> /* FIXME: MTU in config. */ >> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) >> #define GOOD_COPY_LEN 128 >> @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) >> vi->affinity_hint_set = false; >> } >> >> +static int virtnet_cpu_callback(struct notifier_block *nfb, >> + unsigned long action, void *hcpu) >> +{ >> + switch(action) { >> + case CPU_ONLINE: >> + case CPU_ONLINE_FROZEN: >> + case CPU_DEAD: >> + case CPU_DEAD_FROZEN: >> + cpu_hotplug = true; >> + break; >> + default: >> + break; >> + } >> + return NOTIFY_OK; >> +} >> + >> +static struct notifier_block virtnet_cpu_notifier = { >> + .notifier_call = virtnet_cpu_callback, >> +}; >> + >> static void virtnet_get_ringparam(struct net_device *dev, >> struct ethtool_ringparam *ring) >> { >> @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) >> */ >> static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) >> { >> - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : >> + int txq; >> + >> + if (unlikely(cpu_hotplug == true)) { >> + virtnet_set_affinity(netdev_priv(dev), true); >> + cpu_hotplug = false; >> + } >> + >> + txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : >> smp_processor_id(); >> >> while (unlikely(txq >= dev->real_num_tx_queues)) >> @@ -1248,6 +1278,8 @@ static void virtnet_del_vqs(struct virtnet_info *vi) >> { >> struct virtio_device *vdev = vi->vdev; >> >> + unregister_hotcpu_notifier(&virtnet_cpu_notifier); >> + >> virtnet_set_affinity(vi, false); >> >> vdev->config->del_vqs(vdev); >> @@ -1372,6 +1404,11 @@ static int init_vqs(struct virtnet_info *vi) >> goto err_free; >> >> virtnet_set_affinity(vi, true); >> + >> + ret = register_hotcpu_notifier(&virtnet_cpu_notifier); >> + if (ret) >> + goto err_free; >> + >> return 0; >> >> err_free: >> -- >> 1.8.0 -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 12/27/2012 11:28 AM, Jason Wang wrote: > On 12/26/2012 06:19 PM, Wanlong Gao wrote: >> On 12/26/2012 06:06 PM, Jason Wang wrote: >>> On 12/26/2012 03:06 PM, Wanlong Gao wrote: >>>> Add a cpu notifier to virtio-net, so that we can reset the >>>> virtqueue affinity if the cpu hotplug happens. It improve >>>> the performance through enabling or disabling the virtqueue >>>> affinity after doing cpu hotplug. >>> Hi Wanlong: >>> >>> Thanks for looking at this. >>>> Cc: Rusty Russell <rusty@rustcorp.com.au> >>>> Cc: "Michael S. Tsirkin" <mst@redhat.com> >>>> Cc: Jason Wang <jasowang@redhat.com> >>>> Cc: virtualization@lists.linux-foundation.org >>>> Cc: netdev@vger.kernel.org >>>> Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> >>>> --- >>>> drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- >>>> 1 file changed, 38 insertions(+), 1 deletion(-) >>>> >>>> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c >>>> index a6fcf15..9710cf4 100644 >>>> --- a/drivers/net/virtio_net.c >>>> +++ b/drivers/net/virtio_net.c >>>> @@ -26,6 +26,7 @@ >>>> #include <linux/scatterlist.h> >>>> #include <linux/if_vlan.h> >>>> #include <linux/slab.h> >>>> +#include <linux/cpu.h> >>>> >>>> static int napi_weight = 128; >>>> module_param(napi_weight, int, 0444); >>>> @@ -34,6 +35,8 @@ static bool csum = true, gso = true; >>>> module_param(csum, bool, 0444); >>>> module_param(gso, bool, 0444); >>>> >>>> +static bool cpu_hotplug = false; >>>> + >>>> /* FIXME: MTU in config. */ >>>> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) >>>> #define GOOD_COPY_LEN 128 >>>> @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) >>>> vi->affinity_hint_set = false; >>>> } >>>> >>>> +static int virtnet_cpu_callback(struct notifier_block *nfb, >>>> + unsigned long action, void *hcpu) >>>> +{ >>>> + switch(action) { >>>> + case CPU_ONLINE: >>>> + case CPU_ONLINE_FROZEN: >>>> + case CPU_DEAD: >>>> + case CPU_DEAD_FROZEN: >>>> + cpu_hotplug = true; >>>> + break; >>>> + default: >>>> + break; >>>> + } >>>> + return NOTIFY_OK; >>>> +} >>>> + >>>> +static struct notifier_block virtnet_cpu_notifier = { >>>> + .notifier_call = virtnet_cpu_callback, >>>> +}; >>>> + >>>> static void virtnet_get_ringparam(struct net_device *dev, >>>> struct ethtool_ringparam *ring) >>>> { >>>> @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) >>>> */ >>>> static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) >>>> { >>>> - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : >>>> + int txq; >>>> + >>>> + if (unlikely(cpu_hotplug == true)) { >>>> + virtnet_set_affinity(netdev_priv(dev), true); >>>> + cpu_hotplug = false; >>>> + } >>>> + >>> Why don't you just do this in callback? >> Callback can just give us a "hcpu", can't get the virtnet_info from callback. Am I missing something? > > Well, I think you can just embed the notifier block into virtnet_info, > then use something like container_of in the callback to make the > notifier per device. This also solve the concern of Eric. Yeah, thank you very much for your suggestion. I'll try it. >>> btw. Does qemu/kvm support cpu-hotplug now? >> From http://www.linux-kvm.org/page/CPUHotPlug, I saw that qemu-kvm can support hotplug >> but failed to merge to qemu.git, right? > > Not sure, I just try latest qemu, it even does not have a cpu_set command. Adding Igor to CC, As I know, hotplug support is cleaned from qemu, and Igor want to rework it but not been completed? I'm not sure about that, Igor, could you send out your tech-preview-patches? Thanks, Wanlong Gao > > Thanks >> >> Thanks, >> Wanlong Gao >> > > -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Thu, Dec 27, 2012 at 11:34:16AM +0800, Jason Wang wrote: > On 12/26/2012 06:46 PM, Michael S. Tsirkin wrote: > > On Wed, Dec 26, 2012 at 03:06:54PM +0800, Wanlong Gao wrote: > >> Add a cpu notifier to virtio-net, so that we can reset the > >> virtqueue affinity if the cpu hotplug happens. It improve > >> the performance through enabling or disabling the virtqueue > >> affinity after doing cpu hotplug. > >> > >> Cc: Rusty Russell <rusty@rustcorp.com.au> > >> Cc: "Michael S. Tsirkin" <mst@redhat.com> > >> Cc: Jason Wang <jasowang@redhat.com> > >> Cc: virtualization@lists.linux-foundation.org > >> Cc: netdev@vger.kernel.org > >> Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> > > Thanks for looking into this. > > Some comments: > > > > 1. Looks like the logic in > > virtnet_set_affinity (and in virtnet_select_queue) > > will not work very well when CPU IDs are not > > consequitive. This can happen with hot unplug. > > > > Maybe we should add a VQ allocator, and defining > > a per-cpu variable specifying the VQ instead > > of using CPU ID. > > Yes, and generate the affinity hint based on the mapping. Btw, what does > VQ allocator means here? Some logic to generate CPU to VQ mapping. > > > > > > 2. The below code seems racy e.g. when CPU is added > > during device init. > > > > 3. using a global cpu_hotplug seems inelegant. > > In any case we should document what is the > > meaning of this variable. > > > >> --- > >> drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- > >> 1 file changed, 38 insertions(+), 1 deletion(-) > >> > >> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > >> index a6fcf15..9710cf4 100644 > >> --- a/drivers/net/virtio_net.c > >> +++ b/drivers/net/virtio_net.c > >> @@ -26,6 +26,7 @@ > >> #include <linux/scatterlist.h> > >> #include <linux/if_vlan.h> > >> #include <linux/slab.h> > >> +#include <linux/cpu.h> > >> > >> static int napi_weight = 128; > >> module_param(napi_weight, int, 0444); > >> @@ -34,6 +35,8 @@ static bool csum = true, gso = true; > >> module_param(csum, bool, 0444); > >> module_param(gso, bool, 0444); > >> > >> +static bool cpu_hotplug = false; > >> + > >> /* FIXME: MTU in config. */ > >> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > >> #define GOOD_COPY_LEN 128 > >> @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) > >> vi->affinity_hint_set = false; > >> } > >> > >> +static int virtnet_cpu_callback(struct notifier_block *nfb, > >> + unsigned long action, void *hcpu) > >> +{ > >> + switch(action) { > >> + case CPU_ONLINE: > >> + case CPU_ONLINE_FROZEN: > >> + case CPU_DEAD: > >> + case CPU_DEAD_FROZEN: > >> + cpu_hotplug = true; > >> + break; > >> + default: > >> + break; > >> + } > >> + return NOTIFY_OK; > >> +} > >> + > >> +static struct notifier_block virtnet_cpu_notifier = { > >> + .notifier_call = virtnet_cpu_callback, > >> +}; > >> + > >> static void virtnet_get_ringparam(struct net_device *dev, > >> struct ethtool_ringparam *ring) > >> { > >> @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) > >> */ > >> static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) > >> { > >> - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > >> + int txq; > >> + > >> + if (unlikely(cpu_hotplug == true)) { > >> + virtnet_set_affinity(netdev_priv(dev), true); > >> + cpu_hotplug = false; > >> + } > >> + > >> + txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > >> smp_processor_id(); > >> > >> while (unlikely(txq >= dev->real_num_tx_queues)) > >> @@ -1248,6 +1278,8 @@ static void virtnet_del_vqs(struct virtnet_info *vi) > >> { > >> struct virtio_device *vdev = vi->vdev; > >> > >> + unregister_hotcpu_notifier(&virtnet_cpu_notifier); > >> + > >> virtnet_set_affinity(vi, false); > >> > >> vdev->config->del_vqs(vdev); > >> @@ -1372,6 +1404,11 @@ static int init_vqs(struct virtnet_info *vi) > >> goto err_free; > >> > >> virtnet_set_affinity(vi, true); > >> + > >> + ret = register_hotcpu_notifier(&virtnet_cpu_notifier); > >> + if (ret) > >> + goto err_free; > >> + > >> return 0; > >> > >> err_free: > >> -- > >> 1.8.0 -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Thu, 27 Dec 2012 11:43:30 +0800 Wanlong Gao <gaowanlong@cn.fujitsu.com> wrote: > On 12/27/2012 11:28 AM, Jason Wang wrote: > > On 12/26/2012 06:19 PM, Wanlong Gao wrote: > >> On 12/26/2012 06:06 PM, Jason Wang wrote: > >>> On 12/26/2012 03:06 PM, Wanlong Gao wrote: > >>>> Add a cpu notifier to virtio-net, so that we can reset the > >>>> virtqueue affinity if the cpu hotplug happens. It improve > >>>> the performance through enabling or disabling the virtqueue > >>>> affinity after doing cpu hotplug. > >>> Hi Wanlong: > >>> > >>> Thanks for looking at this. > >>>> Cc: Rusty Russell <rusty@rustcorp.com.au> > >>>> Cc: "Michael S. Tsirkin" <mst@redhat.com> > >>>> Cc: Jason Wang <jasowang@redhat.com> > >>>> Cc: virtualization@lists.linux-foundation.org > >>>> Cc: netdev@vger.kernel.org > >>>> Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> > >>>> --- > >>>> drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- > >>>> 1 file changed, 38 insertions(+), 1 deletion(-) > >>>> > >>>> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > >>>> index a6fcf15..9710cf4 100644 > >>>> --- a/drivers/net/virtio_net.c > >>>> +++ b/drivers/net/virtio_net.c > >>>> @@ -26,6 +26,7 @@ > >>>> #include <linux/scatterlist.h> > >>>> #include <linux/if_vlan.h> > >>>> #include <linux/slab.h> > >>>> +#include <linux/cpu.h> > >>>> > >>>> static int napi_weight = 128; > >>>> module_param(napi_weight, int, 0444); > >>>> @@ -34,6 +35,8 @@ static bool csum = true, gso = true; > >>>> module_param(csum, bool, 0444); > >>>> module_param(gso, bool, 0444); > >>>> > >>>> +static bool cpu_hotplug = false; > >>>> + > >>>> /* FIXME: MTU in config. */ > >>>> #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) > >>>> #define GOOD_COPY_LEN 128 > >>>> @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) > >>>> vi->affinity_hint_set = false; > >>>> } > >>>> > >>>> +static int virtnet_cpu_callback(struct notifier_block *nfb, > >>>> + unsigned long action, void *hcpu) > >>>> +{ > >>>> + switch(action) { > >>>> + case CPU_ONLINE: > >>>> + case CPU_ONLINE_FROZEN: > >>>> + case CPU_DEAD: > >>>> + case CPU_DEAD_FROZEN: > >>>> + cpu_hotplug = true; > >>>> + break; > >>>> + default: > >>>> + break; > >>>> + } > >>>> + return NOTIFY_OK; > >>>> +} > >>>> + > >>>> +static struct notifier_block virtnet_cpu_notifier = { > >>>> + .notifier_call = virtnet_cpu_callback, > >>>> +}; > >>>> + > >>>> static void virtnet_get_ringparam(struct net_device *dev, > >>>> struct ethtool_ringparam *ring) > >>>> { > >>>> @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) > >>>> */ > >>>> static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) > >>>> { > >>>> - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : > >>>> + int txq; > >>>> + > >>>> + if (unlikely(cpu_hotplug == true)) { > >>>> + virtnet_set_affinity(netdev_priv(dev), true); > >>>> + cpu_hotplug = false; > >>>> + } > >>>> + > >>> Why don't you just do this in callback? > >> Callback can just give us a "hcpu", can't get the virtnet_info from callback. Am I missing something? > > > > Well, I think you can just embed the notifier block into virtnet_info, > > then use something like container_of in the callback to make the > > notifier per device. This also solve the concern of Eric. > > Yeah, thank you very much for your suggestion. I'll try it. > > >>> btw. Does qemu/kvm support cpu-hotplug now? > >> From http://www.linux-kvm.org/page/CPUHotPlug, I saw that qemu-kvm can support hotplug > >> but failed to merge to qemu.git, right? > > > > Not sure, I just try latest qemu, it even does not have a cpu_set command. > > Adding Igor to CC, > > As I know, hotplug support is cleaned from qemu, and Igor want to rework it but not been completed? > I'm not sure about that, Igor, could you send out your tech-preview-patches? CPU hot-add is supported by upstream now, hot-remove is not supported yet, besides of qemu work it would require quite a work on kvm side as well. > > Thanks, > Wanlong Gao > > > > > Thanks > >> > >> Thanks, > >> Wanlong Gao > >> > > > > > -- To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index a6fcf15..9710cf4 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -26,6 +26,7 @@ #include <linux/scatterlist.h> #include <linux/if_vlan.h> #include <linux/slab.h> +#include <linux/cpu.h> static int napi_weight = 128; module_param(napi_weight, int, 0444); @@ -34,6 +35,8 @@ static bool csum = true, gso = true; module_param(csum, bool, 0444); module_param(gso, bool, 0444); +static bool cpu_hotplug = false; + /* FIXME: MTU in config. */ #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) #define GOOD_COPY_LEN 128 @@ -1041,6 +1044,26 @@ static void virtnet_set_affinity(struct virtnet_info *vi, bool set) vi->affinity_hint_set = false; } +static int virtnet_cpu_callback(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + switch(action) { + case CPU_ONLINE: + case CPU_ONLINE_FROZEN: + case CPU_DEAD: + case CPU_DEAD_FROZEN: + cpu_hotplug = true; + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block virtnet_cpu_notifier = { + .notifier_call = virtnet_cpu_callback, +}; + static void virtnet_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) { @@ -1131,7 +1154,14 @@ static int virtnet_change_mtu(struct net_device *dev, int new_mtu) */ static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb) { - int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : + int txq; + + if (unlikely(cpu_hotplug == true)) { + virtnet_set_affinity(netdev_priv(dev), true); + cpu_hotplug = false; + } + + txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : smp_processor_id(); while (unlikely(txq >= dev->real_num_tx_queues)) @@ -1248,6 +1278,8 @@ static void virtnet_del_vqs(struct virtnet_info *vi) { struct virtio_device *vdev = vi->vdev; + unregister_hotcpu_notifier(&virtnet_cpu_notifier); + virtnet_set_affinity(vi, false); vdev->config->del_vqs(vdev); @@ -1372,6 +1404,11 @@ static int init_vqs(struct virtnet_info *vi) goto err_free; virtnet_set_affinity(vi, true); + + ret = register_hotcpu_notifier(&virtnet_cpu_notifier); + if (ret) + goto err_free; + return 0; err_free:
Add a cpu notifier to virtio-net, so that we can reset the virtqueue affinity if the cpu hotplug happens. It improve the performance through enabling or disabling the virtqueue affinity after doing cpu hotplug. Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: "Michael S. Tsirkin" <mst@redhat.com> Cc: Jason Wang <jasowang@redhat.com> Cc: virtualization@lists.linux-foundation.org Cc: netdev@vger.kernel.org Signed-off-by: Wanlong Gao <gaowanlong@cn.fujitsu.com> --- drivers/net/virtio_net.c | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-)