| Message ID | 1563031186-2101-2-git-send-email-yanhaishuang@cmss.chinamobile.com |
|---|---|
| State | Changes Requested |
| Delegated to: | Pablo Neira |
| Headers | show |
| Series | ipvs: speedup ipvs netns dismantle | expand |
Hello, On Sat, 13 Jul 2019, Haishuang Yan wrote: > It's better to batch __ip_vs_cleanup to speedup ipvs > connections dismantle. > > Signed-off-by: Haishuang Yan <yanhaishuang@cmss.chinamobile.com> > --- > include/net/ip_vs.h | 2 +- > net/netfilter/ipvs/ip_vs_core.c | 29 +++++++++++++++++------------ > net/netfilter/ipvs/ip_vs_ctl.c | 13 ++++++++++--- > 3 files changed, 28 insertions(+), 16 deletions(-) > > diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h > index 3759167..93e7a25 100644 > --- a/include/net/ip_vs.h > +++ b/include/net/ip_vs.h > @@ -1324,7 +1324,7 @@ static inline void ip_vs_control_del(struct ip_vs_conn *cp) > void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); > void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); > void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); > -void ip_vs_service_net_cleanup(struct netns_ipvs *ipvs); > +void ip_vs_service_nets_cleanup(struct list_head *net_list); > > /* IPVS application functions > * (from ip_vs_app.c) > diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c > index 46f06f9..b4d79b7 100644 > --- a/net/netfilter/ipvs/ip_vs_core.c > +++ b/net/netfilter/ipvs/ip_vs_core.c > @@ -2402,18 +2402,23 @@ static int __net_init __ip_vs_init(struct net *net) > return -ENOMEM; > } > > -static void __net_exit __ip_vs_cleanup(struct net *net) > +static void __net_exit __ip_vs_cleanup_batch(struct list_head *net_list) > { > - struct netns_ipvs *ipvs = net_ipvs(net); > - > - ip_vs_service_net_cleanup(ipvs); /* ip_vs_flush() with locks */ > - ip_vs_conn_net_cleanup(ipvs); > - ip_vs_app_net_cleanup(ipvs); > - ip_vs_protocol_net_cleanup(ipvs); > - ip_vs_control_net_cleanup(ipvs); > - ip_vs_estimator_net_cleanup(ipvs); > - IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); > - net->ipvs = NULL; > + struct netns_ipvs *ipvs; > + struct net *net; > + LIST_HEAD(list); > + > + ip_vs_service_nets_cleanup(net_list); /* ip_vs_flush() with locks */ > + list_for_each_entry(net, net_list, exit_list) { How much faster is to replace list_for_each_entry in ops_exit_list() with this one. IPVS can waste time in calls such as kthread_stop() and del_timer_sync() but I'm not sure we can solve it easily. What gain do you see in benchmarks? > + ipvs = net_ipvs(net); > + ip_vs_conn_net_cleanup(ipvs); > + ip_vs_app_net_cleanup(ipvs); > + ip_vs_protocol_net_cleanup(ipvs); > + ip_vs_control_net_cleanup(ipvs); > + ip_vs_estimator_net_cleanup(ipvs); > + IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); > + net->ipvs = NULL; > + } > } Regards -- Julian Anastasov <ja@ssi.bg>
> On 2019年7月16日, at 上午4:39, Julian Anastasov <ja@ssi.bg> wrote: > > > Hello, > > On Sat, 13 Jul 2019, Haishuang Yan wrote: > >> It's better to batch __ip_vs_cleanup to speedup ipvs >> connections dismantle. >> >> Signed-off-by: Haishuang Yan <yanhaishuang@cmss.chinamobile.com> >> --- >> include/net/ip_vs.h | 2 +- >> net/netfilter/ipvs/ip_vs_core.c | 29 +++++++++++++++++------------ >> net/netfilter/ipvs/ip_vs_ctl.c | 13 ++++++++++--- >> 3 files changed, 28 insertions(+), 16 deletions(-) >> >> diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h >> index 3759167..93e7a25 100644 >> --- a/include/net/ip_vs.h >> +++ b/include/net/ip_vs.h >> @@ -1324,7 +1324,7 @@ static inline void ip_vs_control_del(struct ip_vs_conn *cp) >> void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); >> void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); >> void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); >> -void ip_vs_service_net_cleanup(struct netns_ipvs *ipvs); >> +void ip_vs_service_nets_cleanup(struct list_head *net_list); >> >> /* IPVS application functions >> * (from ip_vs_app.c) >> diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c >> index 46f06f9..b4d79b7 100644 >> --- a/net/netfilter/ipvs/ip_vs_core.c >> +++ b/net/netfilter/ipvs/ip_vs_core.c >> @@ -2402,18 +2402,23 @@ static int __net_init __ip_vs_init(struct net *net) >> return -ENOMEM; >> } >> >> -static void __net_exit __ip_vs_cleanup(struct net *net) >> +static void __net_exit __ip_vs_cleanup_batch(struct list_head *net_list) >> { >> - struct netns_ipvs *ipvs = net_ipvs(net); >> - >> - ip_vs_service_net_cleanup(ipvs); /* ip_vs_flush() with locks */ >> - ip_vs_conn_net_cleanup(ipvs); >> - ip_vs_app_net_cleanup(ipvs); >> - ip_vs_protocol_net_cleanup(ipvs); >> - ip_vs_control_net_cleanup(ipvs); >> - ip_vs_estimator_net_cleanup(ipvs); >> - IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); >> - net->ipvs = NULL; >> + struct netns_ipvs *ipvs; >> + struct net *net; >> + LIST_HEAD(list); >> + >> + ip_vs_service_nets_cleanup(net_list); /* ip_vs_flush() with locks */ >> + list_for_each_entry(net, net_list, exit_list) { > > How much faster is to replace list_for_each_entry in > ops_exit_list() with this one. IPVS can waste time in calls > such as kthread_stop() and del_timer_sync() but I'm not sure > we can solve it easily. What gain do you see in benchmarks? Hi, As the following benchmark testing results show, there is a little performance improvement: $ cat add_del_unshare.sh #!/bin/bash for i in `seq 1 100` do (for j in `seq 1 40` ; do unshare -n ipvsadm -A -t 172.16.$i.$j:80 >/dev/null ; done) & done wait; grep net_namespace /proc/slabinfo Befor patch: $ time sh add_del_unshare.sh net_namespace 4020 4020 4736 6 8 : tunables 0 0 0 : slabdata 670 670 0 real 0m8.086s user 0m2.025s sys 0m36.956s After patch: $ time sh add_del_unshare.sh net_namespace 4020 4020 4736 6 8 : tunables 0 0 0 : slabdata 670 670 0 real 0m7.623s user 0m2.003s sys 0m32.935s > >> + ipvs = net_ipvs(net); >> + ip_vs_conn_net_cleanup(ipvs); >> + ip_vs_app_net_cleanup(ipvs); >> + ip_vs_protocol_net_cleanup(ipvs); >> + ip_vs_control_net_cleanup(ipvs); >> + ip_vs_estimator_net_cleanup(ipvs); >> + IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); >> + net->ipvs = NULL; >> + } >> } > > Regards > > -- > Julian Anastasov <ja@ssi.bg> >
Hello, On Thu, 18 Jul 2019, Haishuang Yan wrote: > As the following benchmark testing results show, there is a little performance improvement: OK, can you send v2 after removing the LIST_HEAD(list) from both patches, I guess, it is not needed. If you prefer, you can include these benchmark results too. > $ cat add_del_unshare.sh > #!/bin/bash > > for i in `seq 1 100` > do > (for j in `seq 1 40` ; do unshare -n ipvsadm -A -t 172.16.$i.$j:80 >/dev/null ; done) & > done > wait; grep net_namespace /proc/slabinfo > > Befor patch: > $ time sh add_del_unshare.sh > net_namespace 4020 4020 4736 6 8 : tunables 0 0 0 : slabdata 670 670 0 > > real 0m8.086s > user 0m2.025s > sys 0m36.956s > > After patch: > $ time sh add_del_unshare.sh > net_namespace 4020 4020 4736 6 8 : tunables 0 0 0 : slabdata 670 670 0 > > real 0m7.623s > user 0m2.003s > sys 0m32.935s > > > > > >> + ipvs = net_ipvs(net); > >> + ip_vs_conn_net_cleanup(ipvs); > >> + ip_vs_app_net_cleanup(ipvs); > >> + ip_vs_protocol_net_cleanup(ipvs); > >> + ip_vs_control_net_cleanup(ipvs); > >> + ip_vs_estimator_net_cleanup(ipvs); > >> + IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); > >> + net->ipvs = NULL; Regards -- Julian Anastasov <ja@ssi.bg>
diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 3759167..93e7a25 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -1324,7 +1324,7 @@ static inline void ip_vs_control_del(struct ip_vs_conn *cp) void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); -void ip_vs_service_net_cleanup(struct netns_ipvs *ipvs); +void ip_vs_service_nets_cleanup(struct list_head *net_list); /* IPVS application functions * (from ip_vs_app.c) diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 46f06f9..b4d79b7 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -2402,18 +2402,23 @@ static int __net_init __ip_vs_init(struct net *net) return -ENOMEM; } -static void __net_exit __ip_vs_cleanup(struct net *net) +static void __net_exit __ip_vs_cleanup_batch(struct list_head *net_list) { - struct netns_ipvs *ipvs = net_ipvs(net); - - ip_vs_service_net_cleanup(ipvs); /* ip_vs_flush() with locks */ - ip_vs_conn_net_cleanup(ipvs); - ip_vs_app_net_cleanup(ipvs); - ip_vs_protocol_net_cleanup(ipvs); - ip_vs_control_net_cleanup(ipvs); - ip_vs_estimator_net_cleanup(ipvs); - IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); - net->ipvs = NULL; + struct netns_ipvs *ipvs; + struct net *net; + LIST_HEAD(list); + + ip_vs_service_nets_cleanup(net_list); /* ip_vs_flush() with locks */ + list_for_each_entry(net, net_list, exit_list) { + ipvs = net_ipvs(net); + ip_vs_conn_net_cleanup(ipvs); + ip_vs_app_net_cleanup(ipvs); + ip_vs_protocol_net_cleanup(ipvs); + ip_vs_control_net_cleanup(ipvs); + ip_vs_estimator_net_cleanup(ipvs); + IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen); + net->ipvs = NULL; + } } static int __net_init __ip_vs_dev_init(struct net *net) @@ -2442,7 +2447,7 @@ static void __net_exit __ip_vs_dev_cleanup(struct net *net) static struct pernet_operations ipvs_core_ops = { .init = __ip_vs_init, - .exit = __ip_vs_cleanup, + .exit_batch = __ip_vs_cleanup_batch, .id = &ip_vs_net_id, .size = sizeof(struct netns_ipvs), }; diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index 07e0967..c8e652b 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -1607,14 +1607,21 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup) /* * Delete service by {netns} in the service table. - * Called by __ip_vs_cleanup() + * Called by __ip_vs_batch_cleanup() */ -void ip_vs_service_net_cleanup(struct netns_ipvs *ipvs) +void ip_vs_service_nets_cleanup(struct list_head *net_list) { + struct netns_ipvs *ipvs; + struct net *net; + LIST_HEAD(list); + EnterFunction(2); /* Check for "full" addressed entries */ mutex_lock(&__ip_vs_mutex); - ip_vs_flush(ipvs, true); + list_for_each_entry(net, net_list, exit_list) { + ipvs = net_ipvs(net); + ip_vs_flush(ipvs, true); + } mutex_unlock(&__ip_vs_mutex); LeaveFunction(2); }
It's better to batch __ip_vs_cleanup to speedup ipvs connections dismantle. Signed-off-by: Haishuang Yan <yanhaishuang@cmss.chinamobile.com> --- include/net/ip_vs.h | 2 +- net/netfilter/ipvs/ip_vs_core.c | 29 +++++++++++++++++------------ net/netfilter/ipvs/ip_vs_ctl.c | 13 ++++++++++--- 3 files changed, 28 insertions(+), 16 deletions(-)