Message ID | 20171116225233.2697-3-victor.rodriguez.bahena@intel.com |
---|---|
State | New |
Headers | show |
Series | [1/3] benchtests: Wide range of tests names in schema.json | expand |
Hi, Almost there, comments inline. Please post the final version with those changes. Thanks, Siddhesh On Friday 17 November 2017 04:22 AM, Victor Rodriguez wrote: > This patch adds BENCHSET variable to benchtests/Makefile in order to > provide the capability to run a list of subsets of benchmark tests, ie; > > make bench BENCHSET="bench-pthread bench-math malloc-thread" > > This Helps users to benchmark specific glibc area > > Changelog: > 2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com> > > (VERSION): Set to 2.27 > * benchtests/Makefile:Add BENCHSET to allow subsets of > benchmarks to be run. > * benchtests/README: Add documentation for: Running subsets of > benchmarks. > > Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com> > Signed-off-by: Icarus Sparry <icarus.w.sparry@intel.com> > --- > ChangeLog | 9 +++++++ > benchtests/Makefile | 69 +++++++++++++++++++++++++++++++++++++---------------- > benchtests/README | 20 ++++++++++++++++ > 3 files changed, 78 insertions(+), 20 deletions(-) > > diff --git a/ChangeLog b/ChangeLog > index 6a6bb62..bf61c72 100644 > --- a/ChangeLog > +++ b/ChangeLog > @@ -1,3 +1,12 @@ > +Changelog: > +2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com> > + > + (VERSION): Set to 2.27 What is this line for? > + * benchtests/Makefile:Add BENCHSET to allow subsets of > + benchmarks to be run > + * benchtests/README: Add documentation for: Running subsets of > + benchmarks. > + > 2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com> > > (VERSION): Set to 2.27 > diff --git a/benchtests/Makefile b/benchtests/Makefile > index 37788e8..6712710 100644 > --- a/benchtests/Makefile > +++ b/benchtests/Makefile > @@ -29,7 +29,11 @@ bench-pthread := pthread_once thread_create > > bench-string := ffs ffsll > > +ifeq (${BENCHSET},) > bench := $(bench-math) $(bench-pthread) $(bench-string) > +else > +bench := $(foreach B,$(filter bench-%,${BENCHSET}), ${${B}}) > +endif OK. > > # String function benchmarks. > string-benchset := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \ > @@ -66,8 +70,12 @@ stdio-common-benchset := sprintf > > math-benchset := math-inlines > > +ifeq (${BENCHSET},) > benchset := $(string-benchset-all) $(stdlib-benchset) $(stdio-common-benchset) \ > $(math-benchset) > +else > +benchset := $(foreach B,$(filter %-benchset,${BENCHSET}), ${${B}}) > +endif OK. > > CFLAGS-bench-ffs.c += -fno-builtin > CFLAGS-bench-ffsll.c += -fno-builtin > @@ -77,7 +85,11 @@ CFLAGS-bench-fminf.c += -fno-builtin > CFLAGS-bench-fmax.c += -fno-builtin > CFLAGS-bench-fmaxf.c += -fno-builtin > > +ifeq (${BENCHSET},) > bench-malloc := malloc-thread > +else > +bench-malloc := $(filter malloc-%,${BENCHSET}) > +endif OK. > > $(addprefix $(objpfx)bench-,$(bench-math)): $(libm) > $(addprefix $(objpfx)bench-,$(math-benchset)): $(libm) > @@ -144,6 +156,19 @@ bench-clean: > rm -f $(timing-type) $(addsuffix .o,$(timing-type)) > rm -f $(addprefix $(objpfx),$(bench-extra-objs)) > > +# Validate the passed in BENCHSET > +ifneq ($(strip ${BENCHSET}),) > +VALIDBENCHSETNAMES := bench-pthread bench-math bench-string string-benchset \ > + wcsmbs-benchset stdlib-benchset stdio-common-benchset math-benchset \ > + malloc-thread > +INVALIDBENCHSETNAMES := $(filter-out ${VALIDBENCHSETNAMES},${BENCHSET}) > +ifneq (${INVALIDBENCHSETNAMES},) > +$(info The following values in BENCHSET are invalid: ${INVALIDBENCHSETNAMES}) > +$(info The valid ones are: ${VALIDBENCHSETNAMES}) > +$(error Invalid BENCHSET value) > +endif > +endif > + > # Define the bench target only if the target has a usable python installation. > ifdef PYTHON > bench: bench-build bench-set bench-func bench-malloc > @@ -170,10 +195,11 @@ bench-set: $(binaries-benchset) > done > > bench-malloc: $(binaries-bench-malloc) > - run=$(objpfx)bench-malloc-thread; \ > - for thr in 1 8 16 32; do \ > - echo "Running $${run} $${thr}"; \ > + for run in $^; do \ > + for thr in 1 8 16 32; do \ > + echo "Running $${run} $${thr}"; \ > $(run-bench) $${thr} > $${run}-$${thr}.out; \ > + done;\ > done OK. > > # Build and execute the benchmark functions. This target generates JSON > @@ -181,25 +207,28 @@ bench-malloc: $(binaries-bench-malloc) > # so one could even execute them individually and process it using any JSON > # capable language or tool. > bench-func: $(binaries-bench) > + if [ -n '$^' ] ; then \ > { timing_type=$$($(timing-type)); \ > - echo "{\"timing_type\": \"$${timing_type}\","; \ > - echo " \"functions\": {"; \ > - for run in $^; do \ > - if ! [ "x$${run}" = "x$<" ]; then \ > - echo ","; \ > + echo "{\"timing_type\": \"$${timing_type}\","; \ > + echo " \"functions\": {"; \ > + for run in $^; do \ > + if ! [ "x$${run}" = "x$<" ]; then \ > + echo ","; \ > + fi; \ > + echo "Running $${run}" >&2; \ > + $(run-bench) $(DETAILED_OPT); \ > + done; \ > + echo; \ > + echo " }"; \ > + echo "}"; \ > + } > $(objpfx)bench.out-tmp; \ > + if [ -f $(objpfx)bench.out ]; then \ > + mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \ > fi; \ > - echo "Running $${run}" >&2; \ > - $(run-bench) $(DETAILED_OPT); \ > - done; \ > - echo; \ > - echo " }"; \ > - echo "}"; } > $(objpfx)bench.out-tmp; \ > - if [ -f $(objpfx)bench.out ]; then \ > - mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \ > - fi; \ > - mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out > - $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \ > - scripts/benchout.schema.json > + mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out; \ > + $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \ > + scripts/benchout.schema.json; \ > + fi > > $(timing-type) $(binaries-bench) $(binaries-benchset) \ > $(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \ > diff --git a/benchtests/README b/benchtests/README > index b015acf..20942c7 100644 > --- a/benchtests/README > +++ b/benchtests/README > @@ -53,6 +53,26 @@ otherwise the above command may try to build the benchmark again. Benchmarks > that require generated code to be executed during the build are skipped when > cross-building. > > +Running subsets of benchmarks: > +============================== > + > +If a subset of benchmarks needs to be run one may run the benchmark by invoking > +make as follows: The following phrasing may be more succint: "To run only a subset of benchmarks, one may invoke make as follows" > + > + $ make bench BENCHSET="bench-pthread bench-math malloc-thread" > + > +Allowing to measure the performance of specific glibc area. Posible areas are: Drop this line and add: "where BENCHSET may be a space-separated list of the following values:" > + > + bench-math > + bench-pthread > + bench-string > + string-benchset > + wcsmbs-benchset > + stdlib-benchset > + stdio-common-benchset > + math-benchset > + malloc-thread > + > Adding a function to benchtests: > =============================== > >
diff --git a/ChangeLog b/ChangeLog index 6a6bb62..bf61c72 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +Changelog: +2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com> + + (VERSION): Set to 2.27 + * benchtests/Makefile:Add BENCHSET to allow subsets of + benchmarks to be run + * benchtests/README: Add documentation for: Running subsets of + benchmarks. + 2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com> (VERSION): Set to 2.27 diff --git a/benchtests/Makefile b/benchtests/Makefile index 37788e8..6712710 100644 --- a/benchtests/Makefile +++ b/benchtests/Makefile @@ -29,7 +29,11 @@ bench-pthread := pthread_once thread_create bench-string := ffs ffsll +ifeq (${BENCHSET},) bench := $(bench-math) $(bench-pthread) $(bench-string) +else +bench := $(foreach B,$(filter bench-%,${BENCHSET}), ${${B}}) +endif # String function benchmarks. string-benchset := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \ @@ -66,8 +70,12 @@ stdio-common-benchset := sprintf math-benchset := math-inlines +ifeq (${BENCHSET},) benchset := $(string-benchset-all) $(stdlib-benchset) $(stdio-common-benchset) \ $(math-benchset) +else +benchset := $(foreach B,$(filter %-benchset,${BENCHSET}), ${${B}}) +endif CFLAGS-bench-ffs.c += -fno-builtin CFLAGS-bench-ffsll.c += -fno-builtin @@ -77,7 +85,11 @@ CFLAGS-bench-fminf.c += -fno-builtin CFLAGS-bench-fmax.c += -fno-builtin CFLAGS-bench-fmaxf.c += -fno-builtin +ifeq (${BENCHSET},) bench-malloc := malloc-thread +else +bench-malloc := $(filter malloc-%,${BENCHSET}) +endif $(addprefix $(objpfx)bench-,$(bench-math)): $(libm) $(addprefix $(objpfx)bench-,$(math-benchset)): $(libm) @@ -144,6 +156,19 @@ bench-clean: rm -f $(timing-type) $(addsuffix .o,$(timing-type)) rm -f $(addprefix $(objpfx),$(bench-extra-objs)) +# Validate the passed in BENCHSET +ifneq ($(strip ${BENCHSET}),) +VALIDBENCHSETNAMES := bench-pthread bench-math bench-string string-benchset \ + wcsmbs-benchset stdlib-benchset stdio-common-benchset math-benchset \ + malloc-thread +INVALIDBENCHSETNAMES := $(filter-out ${VALIDBENCHSETNAMES},${BENCHSET}) +ifneq (${INVALIDBENCHSETNAMES},) +$(info The following values in BENCHSET are invalid: ${INVALIDBENCHSETNAMES}) +$(info The valid ones are: ${VALIDBENCHSETNAMES}) +$(error Invalid BENCHSET value) +endif +endif + # Define the bench target only if the target has a usable python installation. ifdef PYTHON bench: bench-build bench-set bench-func bench-malloc @@ -170,10 +195,11 @@ bench-set: $(binaries-benchset) done bench-malloc: $(binaries-bench-malloc) - run=$(objpfx)bench-malloc-thread; \ - for thr in 1 8 16 32; do \ - echo "Running $${run} $${thr}"; \ + for run in $^; do \ + for thr in 1 8 16 32; do \ + echo "Running $${run} $${thr}"; \ $(run-bench) $${thr} > $${run}-$${thr}.out; \ + done;\ done # Build and execute the benchmark functions. This target generates JSON @@ -181,25 +207,28 @@ bench-malloc: $(binaries-bench-malloc) # so one could even execute them individually and process it using any JSON # capable language or tool. bench-func: $(binaries-bench) + if [ -n '$^' ] ; then \ { timing_type=$$($(timing-type)); \ - echo "{\"timing_type\": \"$${timing_type}\","; \ - echo " \"functions\": {"; \ - for run in $^; do \ - if ! [ "x$${run}" = "x$<" ]; then \ - echo ","; \ + echo "{\"timing_type\": \"$${timing_type}\","; \ + echo " \"functions\": {"; \ + for run in $^; do \ + if ! [ "x$${run}" = "x$<" ]; then \ + echo ","; \ + fi; \ + echo "Running $${run}" >&2; \ + $(run-bench) $(DETAILED_OPT); \ + done; \ + echo; \ + echo " }"; \ + echo "}"; \ + } > $(objpfx)bench.out-tmp; \ + if [ -f $(objpfx)bench.out ]; then \ + mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \ fi; \ - echo "Running $${run}" >&2; \ - $(run-bench) $(DETAILED_OPT); \ - done; \ - echo; \ - echo " }"; \ - echo "}"; } > $(objpfx)bench.out-tmp; \ - if [ -f $(objpfx)bench.out ]; then \ - mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \ - fi; \ - mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out - $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \ - scripts/benchout.schema.json + mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out; \ + $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \ + scripts/benchout.schema.json; \ + fi $(timing-type) $(binaries-bench) $(binaries-benchset) \ $(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \ diff --git a/benchtests/README b/benchtests/README index b015acf..20942c7 100644 --- a/benchtests/README +++ b/benchtests/README @@ -53,6 +53,26 @@ otherwise the above command may try to build the benchmark again. Benchmarks that require generated code to be executed during the build are skipped when cross-building. +Running subsets of benchmarks: +============================== + +If a subset of benchmarks needs to be run one may run the benchmark by invoking +make as follows: + + $ make bench BENCHSET="bench-pthread bench-math malloc-thread" + +Allowing to measure the performance of specific glibc area. Posible areas are: + + bench-math + bench-pthread + bench-string + string-benchset + wcsmbs-benchset + stdlib-benchset + stdio-common-benchset + math-benchset + malloc-thread + Adding a function to benchtests: ===============================