[ovs-dev,v3] OVS-DPDK: Change "dpdk-socket-mem" default value.

From: Marcin Rybka <marcinx.rybka@intel.com>

When "dpdk-socket-mem" and "dpdk-alloc-mem" are not specified,
"dpdk-socket-mem" will be set to allocate 1024MB on each NUMA node.
This change will prevent OVS from failing when NIC is attached on
NUMA node 1 and higher. Patch contains documentation update.

Signed-off-by: Marcin Rybka <marcinx.rybka@intel.com>
Co-authored-by: Billy O'Mahony <billy.o.mahony@intel.com>
Signed-off-by: Billy O'Mahony <billy.o.mahony@intel.com>
---
 Documentation/intro/install/dpdk.rst |  3 ++-
 lib/dpdk.c                           | 28 +++++++++++++++++++++++++++-
 vswitchd/vswitch.xml                 |  7 ++++---
 3 files changed, 33 insertions(+), 5 deletions(-)

> -----Original Message-----
> From: ovs-dev-bounces@openvswitch.org [mailto:ovs-dev-
> bounces@openvswitch.org] On Behalf Of O Mahony, Billy
> Sent: Wednesday, May 9, 2018 11:14 AM
> To: dev@openvswitch.org
> Cc: Rybka, MarcinX <marcinx.rybka@intel.com>
> Subject: [ovs-dev] [PATCH v3] OVS-DPDK: Change "dpdk-socket-mem"
> default value.
> 
> From: Marcin Rybka <marcinx.rybka@intel.com>
> 
> When "dpdk-socket-mem" and "dpdk-alloc-mem" are not specified, "dpdk-
> socket-mem" will be set to allocate 1024MB on each NUMA node.
> This change will prevent OVS from failing when NIC is attached on NUMA
> node 1 and higher. Patch contains documentation update.
> 
I have tested and validated this patch and it works as expected and resolves the issue found.

Tested-by: Hariprasad Govindharajan <hariprasad.govindharajan@intel.com>

> Signed-off-by: Marcin Rybka <marcinx.rybka@intel.com>
> Co-authored-by: Billy O'Mahony <billy.o.mahony@intel.com>
> Signed-off-by: Billy O'Mahony <billy.o.mahony@intel.com>
> ---
>  Documentation/intro/install/dpdk.rst |  3 ++-
>  lib/dpdk.c                           | 28 +++++++++++++++++++++++++++-
>  vswitchd/vswitch.xml                 |  7 ++++---
>  3 files changed, 33 insertions(+), 5 deletions(-)
> 
> diff --git a/Documentation/intro/install/dpdk.rst
> b/Documentation/intro/install/dpdk.rst
> index fea4890..b68438d 100644
> --- a/Documentation/intro/install/dpdk.rst
> +++ b/Documentation/intro/install/dpdk.rst
> @@ -228,7 +228,8 @@ listed below. Defaults will be provided for all values
> not explicitly set.
> 
>  ``dpdk-socket-mem``
>    Comma separated list of memory to pre-allocate from hugepages on
> specific
> -  sockets.
> +  sockets. If not specified, 1024 MB will be set for each numa node by
> + default.
> 
>  ``dpdk-hugepage-dir``
>    Directory where hugetlbfs is mounted
> diff --git a/lib/dpdk.c b/lib/dpdk.c
> index 00dd974..40aa20f 100644
> --- a/lib/dpdk.c
> +++ b/lib/dpdk.c
> @@ -35,6 +35,7 @@
>  #include "netdev-dpdk.h"
>  #include "openvswitch/dynamic-string.h"
>  #include "openvswitch/vlog.h"
> +#include "ovs-numa.h"
>  #include "smap.h"
> 
>  VLOG_DEFINE_THIS_MODULE(dpdk);
> @@ -163,6 +164,28 @@ construct_dpdk_options(const struct smap
> *ovs_other_config,
>      return ret;
>  }
> 
> +static char *
> +construct_dpdk_socket_mem(void)
> +{
> +    int numa;
> +    const char *def_value = "1024";
> +    int numa_nodes = ovs_numa_get_n_numas();
> +
> +    if (numa_nodes == 0 || numa_nodes == OVS_NUMA_UNSPEC) {
> +        numa_nodes = 1;
> +    }
> +    /* Allocate enough memory for digits, comma-sep and terminator. */
> +    char *dpdk_socket_mem = xzalloc(numa_nodes * (strlen(def_value) +
> + 1));
> +
> +    strcat(dpdk_socket_mem, def_value);
> +    for (numa = 1; numa < numa_nodes; ++numa) {
> +        strcat(dpdk_socket_mem, ",");
> +        strcat(dpdk_socket_mem, def_value);
> +    }
> +
> +    return dpdk_socket_mem;
> +}
> +
>  #define MAX_DPDK_EXCL_OPTS 10
> 
>  static int
> @@ -170,6 +193,7 @@ construct_dpdk_mutex_options(const struct smap
> *ovs_other_config,
>                               char ***argv, const int initial_size,
>                               char **extra_args, const size_t extra_argc)  {
> +    char *default_dpdk_socket_mem = construct_dpdk_socket_mem();
>      struct dpdk_exclusive_options_map {
>          const char *category;
>          const char *ovs_dpdk_options[MAX_DPDK_EXCL_OPTS];
> @@ -180,7 +204,7 @@ construct_dpdk_mutex_options(const struct smap
> *ovs_other_config,
>          {"memory type",
>           {"dpdk-alloc-mem", "dpdk-socket-mem", NULL,},
>           {"-m",             "--socket-mem",    NULL,},
> -         "1024,0", 1
> +         default_dpdk_socket_mem, 1
>          },
>      };
> 
> @@ -227,6 +251,8 @@ construct_dpdk_mutex_options(const struct smap
> *ovs_other_config,
>          }
>      }
> 
> +    free(default_dpdk_socket_mem);
> +
>      return ret;
>  }
> 
> diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml index
> 9c2a826..d74f28b 100644
> --- a/vswitchd/vswitch.xml
> +++ b/vswitchd/vswitch.xml
> @@ -302,9 +302,10 @@
>          </p>
>          <p>
>            If dpdk-socket-mem and dpdk-alloc-mem are not specified, dpdk-
> socket-mem
> -          will be used and the default value is 1024,0. If dpdk-socket-mem and
> -          dpdk-alloc-mem are specified at same time, dpdk-socket-mem will be
> -          used as default. Changing this value requires restarting the daemon.
> +          will be used and the default value is 1024 for each numa node. If
> +          dpdk-socket-mem and dpdk-alloc-mem are specified at same time,
> +          dpdk-socket-mem will be used as default. Changing this value
> +          requires restarting the daemon.
>          </p>
>        </column>
> 
> --
> 2.7.4
> 
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev

> > -----Original Message-----
> > From: ovs-dev-bounces@openvswitch.org [mailto:ovs-dev-
> > bounces@openvswitch.org] On Behalf Of O Mahony, Billy
> > Sent: Wednesday, May 9, 2018 11:14 AM
> > To: dev@openvswitch.org
> > Cc: Rybka, MarcinX <marcinx.rybka@intel.com>
> > Subject: [ovs-dev] [PATCH v3] OVS-DPDK: Change "dpdk-socket-mem"
> > default value.
> >
> > From: Marcin Rybka <marcinx.rybka@intel.com>
> >
> > When "dpdk-socket-mem" and "dpdk-alloc-mem" are not specified, "dpdk-
> > socket-mem" will be set to allocate 1024MB on each NUMA node.
> > This change will prevent OVS from failing when NIC is attached on NUMA
> > node 1 and higher. Patch contains documentation update.
> >
> I have tested and validated this patch and it works as expected and
> resolves the issue found.
> 
> Tested-by: Hariprasad Govindharajan <hariprasad.govindharajan@intel.com>
> 

Thanks all for working on this. Will apply to DPDK_MERGE for the next pull request.

Ian
> > Signed-off-by: Marcin Rybka <marcinx.rybka@intel.com>
> > Co-authored-by: Billy O'Mahony <billy.o.mahony@intel.com>
> > Signed-off-by: Billy O'Mahony <billy.o.mahony@intel.com>
> > ---
> >  Documentation/intro/install/dpdk.rst |  3 ++-
> >  lib/dpdk.c                           | 28 +++++++++++++++++++++++++++-
> >  vswitchd/vswitch.xml                 |  7 ++++---
> >  3 files changed, 33 insertions(+), 5 deletions(-)
> >
> > diff --git a/Documentation/intro/install/dpdk.rst
> > b/Documentation/intro/install/dpdk.rst
> > index fea4890..b68438d 100644
> > --- a/Documentation/intro/install/dpdk.rst
> > +++ b/Documentation/intro/install/dpdk.rst
> > @@ -228,7 +228,8 @@ listed below. Defaults will be provided for all
> > values not explicitly set.
> >
> >  ``dpdk-socket-mem``
> >    Comma separated list of memory to pre-allocate from hugepages on
> > specific
> > -  sockets.
> > +  sockets. If not specified, 1024 MB will be set for each numa node
> > + by default.
> >
> >  ``dpdk-hugepage-dir``
> >    Directory where hugetlbfs is mounted diff --git a/lib/dpdk.c
> > b/lib/dpdk.c index 00dd974..40aa20f 100644
> > --- a/lib/dpdk.c
> > +++ b/lib/dpdk.c
> > @@ -35,6 +35,7 @@
> >  #include "netdev-dpdk.h"
> >  #include "openvswitch/dynamic-string.h"
> >  #include "openvswitch/vlog.h"
> > +#include "ovs-numa.h"
> >  #include "smap.h"
> >
> >  VLOG_DEFINE_THIS_MODULE(dpdk);
> > @@ -163,6 +164,28 @@ construct_dpdk_options(const struct smap
> > *ovs_other_config,
> >      return ret;
> >  }
> >
> > +static char *
> > +construct_dpdk_socket_mem(void)
> > +{
> > +    int numa;
> > +    const char *def_value = "1024";
> > +    int numa_nodes = ovs_numa_get_n_numas();
> > +
> > +    if (numa_nodes == 0 || numa_nodes == OVS_NUMA_UNSPEC) {
> > +        numa_nodes = 1;
> > +    }
> > +    /* Allocate enough memory for digits, comma-sep and terminator. */
> > +    char *dpdk_socket_mem = xzalloc(numa_nodes * (strlen(def_value) +
> > + 1));
> > +
> > +    strcat(dpdk_socket_mem, def_value);
> > +    for (numa = 1; numa < numa_nodes; ++numa) {
> > +        strcat(dpdk_socket_mem, ",");
> > +        strcat(dpdk_socket_mem, def_value);
> > +    }
> > +
> > +    return dpdk_socket_mem;
> > +}
> > +
> >  #define MAX_DPDK_EXCL_OPTS 10
> >
> >  static int
> > @@ -170,6 +193,7 @@ construct_dpdk_mutex_options(const struct smap
> > *ovs_other_config,
> >                               char ***argv, const int initial_size,
> >                               char **extra_args, const size_t
> > extra_argc)  {
> > +    char *default_dpdk_socket_mem = construct_dpdk_socket_mem();
> >      struct dpdk_exclusive_options_map {
> >          const char *category;
> >          const char *ovs_dpdk_options[MAX_DPDK_EXCL_OPTS];
> > @@ -180,7 +204,7 @@ construct_dpdk_mutex_options(const struct smap
> > *ovs_other_config,
> >          {"memory type",
> >           {"dpdk-alloc-mem", "dpdk-socket-mem", NULL,},
> >           {"-m",             "--socket-mem",    NULL,},
> > -         "1024,0", 1
> > +         default_dpdk_socket_mem, 1
> >          },
> >      };
> >
> > @@ -227,6 +251,8 @@ construct_dpdk_mutex_options(const struct smap
> > *ovs_other_config,
> >          }
> >      }
> >
> > +    free(default_dpdk_socket_mem);
> > +
> >      return ret;
> >  }
> >
> > diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml index
> > 9c2a826..d74f28b 100644
> > --- a/vswitchd/vswitch.xml
> > +++ b/vswitchd/vswitch.xml
> > @@ -302,9 +302,10 @@
> >          </p>
> >          <p>
> >            If dpdk-socket-mem and dpdk-alloc-mem are not specified,
> > dpdk- socket-mem
> > -          will be used and the default value is 1024,0. If dpdk-socket-
> mem and
> > -          dpdk-alloc-mem are specified at same time, dpdk-socket-mem
> will be
> > -          used as default. Changing this value requires restarting the
> daemon.
> > +          will be used and the default value is 1024 for each numa
> node. If
> > +          dpdk-socket-mem and dpdk-alloc-mem are specified at same
> time,
> > +          dpdk-socket-mem will be used as default. Changing this value
> > +          requires restarting the daemon.
> >          </p>
> >        </column>
> >
> > --
> > 2.7.4
> >
> > _______________________________________________
> > dev mailing list
> > dev@openvswitch.org
> > https://mail.openvswitch.org/mailman/listinfo/ovs-dev
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev