[v3] trace: Multi-backend tracing

Adds support to compile QEMU with multiple tracing backends at the same time.

For example, you can compile QEMU with:

  $ ./configure --enable-trace-backends=ftrace,dtrace

Where 'ftrace' can be handy for having an in-flight record of events, and 'dtrace' can be later used to extract more information from the system.

This patch allows having both available without recompiling QEMU.

Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>
---
 .travis.yml                           |    8 +++--
 Makefile                              |    4 +--
 Makefile.target                       |    4 +--
 configure                             |   44 +++++++++++++-----------------
 docs/tracing.txt                      |    4 +--
 qemu-io.c                             |    2 +
 scripts/tracetool.py                  |   41 ++++++++++++++--------------
 scripts/tracetool/__init__.py         |   22 +++++++--------
 scripts/tracetool/backend/__init__.py |   15 +++++-----
 trace/Makefile.objs                   |   22 +++++++--------
 trace/control-internal.h              |    4 +--
 trace/control.c                       |   49 ++++++++++++++++++++++++++++++++-
 trace/control.h                       |   27 +++---------------
 trace/default.c                       |   40 ---------------------------
 trace/ftrace.c                        |   25 +----------------
 trace/ftrace.h                        |    5 +++
 trace/simple.c                        |   19 +------------
 trace/simple.h                        |    1 +
 trace/stderr.c                        |   30 --------------------
 vl.c                                  |    4 +--
 20 files changed, 144 insertions(+), 226 deletions(-)
 delete mode 100644 trace/default.c
 delete mode 100644 trace/stderr.c

Lluís Vilanova writes:

> Adds support to compile QEMU with multiple tracing backends at the same time.
> For example, you can compile QEMU with:

>   $ ./configure --enable-trace-backends=ftrace,dtrace

> Where 'ftrace' can be handy for having an in-flight record of events, and 'dtrace' can be later used to extract more information from the system.

> This patch allows having both available without recompiling QEMU.

> Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>
> ---

Changes in v3:

* Rebase on 43cbeff.

Changes in v2:

* Rebase on 3e890c7.
* Update ".travis.yml".
* Replace all uses of "TRACE_BACKEND" for "TRACE_BACKENDS".
* Replace "--backend" for "--backends" in "scripts/tracetool.py".
* Update "configure" help and "docs/tracing.txt".


Lluis


>  .travis.yml                           |    8 +++--
>  Makefile                              |    4 +--
>  Makefile.target                       |    4 +--
>  configure                             |   44 +++++++++++++-----------------
>  docs/tracing.txt                      |    4 +--
>  qemu-io.c                             |    2 +
>  scripts/tracetool.py                  |   41 ++++++++++++++--------------
>  scripts/tracetool/__init__.py         |   22 +++++++--------
>  scripts/tracetool/backend/__init__.py |   15 +++++-----
>  trace/Makefile.objs                   |   22 +++++++--------
>  trace/control-internal.h              |    4 +--
>  trace/control.c                       |   49 ++++++++++++++++++++++++++++++++-
>  trace/control.h                       |   27 +++---------------
>  trace/default.c                       |   40 ---------------------------
>  trace/ftrace.c                        |   25 +----------------
>  trace/ftrace.h                        |    5 +++
>  trace/simple.c                        |   19 +------------
>  trace/simple.h                        |    1 +
>  trace/stderr.c                        |   30 --------------------
>  vl.c                                  |    4 +--
>  20 files changed, 144 insertions(+), 226 deletions(-)
>  delete mode 100644 trace/default.c
>  delete mode 100644 trace/stderr.c

> diff --git a/.travis.yml b/.travis.yml
> index 04da973..89c30ae 100644
> --- a/.travis.yml
> +++ b/.travis.yml
> @@ -66,16 +66,16 @@ matrix:
>        compiler: gcc
>      # All the trace backends (apart from dtrace)
>      - env: TARGETS=i386-softmmu,x86_64-softmmu
> -           EXTRA_CONFIG="--enable-trace-backend=stderr"
> +           EXTRA_CONFIG="--enable-trace-backends=stderr"
>        compiler: gcc
>      - env: TARGETS=i386-softmmu,x86_64-softmmu
> -           EXTRA_CONFIG="--enable-trace-backend=simple"
> +           EXTRA_CONFIG="--enable-trace-backends=simple"
>        compiler: gcc
>      - env: TARGETS=i386-softmmu,x86_64-softmmu
> -           EXTRA_CONFIG="--enable-trace-backend=ftrace"
> +           EXTRA_CONFIG="--enable-trace-backends=ftrace"
>             TEST_CMD=""
>        compiler: gcc
>      - env: TARGETS=i386-softmmu,x86_64-softmmu
>            EXTRA_PKGS="liblttng-ust-dev liburcu-dev"
> -          EXTRA_CONFIG="--enable-trace-backend=ust"
> +          EXTRA_CONFIG="--enable-trace-backends=ust"
>        compiler: gcc
> diff --git a/Makefile b/Makefile
> index a120aab..609bf69 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -52,12 +52,12 @@ GENERATED_HEADERS += trace/generated-events.h
>  GENERATED_SOURCES += trace/generated-events.c
 
>  GENERATED_HEADERS += trace/generated-tracers.h
> -ifeq ($(TRACE_BACKEND),dtrace)
> +ifeq ($(findstring dtrace,$(TRACE_BACKENDS)),dtrace)
>  GENERATED_HEADERS += trace/generated-tracers-dtrace.h
>  endif
>  GENERATED_SOURCES += trace/generated-tracers.c
 
> -ifeq ($(TRACE_BACKEND),ust)
> +ifeq ($(findstring ust,$(TRACE_BACKENDS)),ust)
>  GENERATED_HEADERS += trace/generated-ust-provider.h
>  GENERATED_SOURCES += trace/generated-ust.c
>  endif
> diff --git a/Makefile.target b/Makefile.target
> index 6d8fde8..749b2e1 100644
> --- a/Makefile.target
> +++ b/Makefile.target
> @@ -46,7 +46,7 @@ endif
>  $(QEMU_PROG).stp-installed: $(SRC_PATH)/trace-events
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=stap \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		--binary=$(bindir)/$(QEMU_PROG) \
>  		--target-name=$(TARGET_NAME) \
>  		--target-type=$(TARGET_TYPE) \
> @@ -55,7 +55,7 @@ $(QEMU_PROG).stp-installed: $(SRC_PATH)/trace-events
>  $(QEMU_PROG).stp: $(SRC_PATH)/trace-events
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=stap \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		--binary=$(realpath .)/$(QEMU_PROG) \
>  		--target-name=$(TARGET_NAME) \
>  		--target-type=$(TARGET_TYPE) \
> diff --git a/configure b/configure
> index 46bc0c9..73b6539 100755
> --- a/configure
> +++ b/configure
> @@ -180,6 +180,10 @@ path_of() {
>      return 1
>  }
 
> +have_backend () {
> +    echo "$trace_backends" | grep "$1" >/dev/null
> +}
> +
>  # default parameters
>  source_path=`dirname "$0"`
>  cpu=""
> @@ -291,7 +295,7 @@ pkgversion=""
>  pie=""
>  zero_malloc=""
>  qom_cast_debug="yes"
> -trace_backend="nop"
> +trace_backends="nop"
>  trace_file="trace"
>  spice=""
>  rbd=""
> @@ -751,7 +755,7 @@ for opt do
>    ;;
>    --target-list=*) target_list="$optarg"
>    ;;
> -  --enable-trace-backend=*) trace_backend="$optarg"
> +  --enable-trace-backends=*) trace_backends="$optarg"
>    ;;
>    --with-trace-file=*) trace_file="$optarg"
>    ;;
> @@ -1318,7 +1322,7 @@ Advanced options (experts only):
>    --disable-docs           disable documentation build
>    --disable-vhost-net      disable vhost-net acceleration support
>    --enable-vhost-net       enable vhost-net acceleration support
> -  --enable-trace-backend=B Set trace backend
> +  --enable-trace-backends=B Set trace backend
>                             Available backends: $($python $source_path/scripts/tracetool.py --list-backends)
>    --with-trace-file=NAME   Full PATH,NAME of file to store traces
>                             Default:trace-<pid>
> @@ -3651,15 +3655,15 @@ fi
>  ##########################################
>  # check if trace backend exists
 
> -$python "$source_path/scripts/tracetool.py" "--backend=$trace_backend" --check-backend  > /dev/null 2> /dev/null
> +$python "$source_path/scripts/tracetool.py" "--backends=$trace_backends" --check-backends  > /dev/null 2> /dev/null
>  if test "$?" -ne 0 ; then
> -  error_exit "invalid trace backend" \
> -      "Please choose a supported trace backend."
> +  error_exit "invalid trace backends" \
> +      "Please choose supported trace backends."
>  fi
 
>  ##########################################
>  # For 'ust' backend, test if ust headers are present
> -if test "$trace_backend" = "ust"; then
> +if have_backend "ust"; then
>    cat > $TMPC << EOF
>  #include <lttng/tracepoint.h>
>  int main(void) { return 0; }
> @@ -3685,7 +3689,7 @@ fi
 
>  ##########################################
>  # For 'dtrace' backend, test if 'dtrace' command is present
> -if test "$trace_backend" = "dtrace"; then
> +if have_backend "dtrace"; then
>    if ! has 'dtrace' ; then
>      error_exit "dtrace command is not found in PATH $PATH"
>    fi
> @@ -4152,7 +4156,7 @@ echo "uuid support      $uuid"
>  echo "libcap-ng support $cap_ng"
>  echo "vhost-net support $vhost_net"
>  echo "vhost-scsi support $vhost_scsi"
> -echo "Trace backend     $trace_backend"
> +echo "Trace backends    $trace_backends"
>  if test "$trace_backend" = "simple"; then
>  echo "Trace output file $trace_file-<pid>"
>  fi
> @@ -4646,43 +4650,35 @@ if test "$tpm" = "yes"; then
>    fi
>  fi
 
> -# use default implementation for tracing backend-specific routines
> -trace_default=yes
> -echo "TRACE_BACKEND=$trace_backend" >> $config_host_mak
> -if test "$trace_backend" = "nop"; then
> +echo "TRACE_BACKENDS=$trace_backends" >> $config_host_mak
> +if have_backend "nop"; then
>    echo "CONFIG_TRACE_NOP=y" >> $config_host_mak
>  fi
> -if test "$trace_backend" = "simple"; then
> +if have_backend "simple"; then
>    echo "CONFIG_TRACE_SIMPLE=y" >> $config_host_mak
> -  trace_default=no
>    # Set the appropriate trace file.
>    trace_file="\"$trace_file-\" FMT_pid"
>  fi
> -if test "$trace_backend" = "stderr"; then
> +if have_backend "stderr"; then
>    echo "CONFIG_TRACE_STDERR=y" >> $config_host_mak
> -  trace_default=no
>  fi
> -if test "$trace_backend" = "ust"; then
> +if have_backend "ust"; then
>    echo "CONFIG_TRACE_UST=y" >> $config_host_mak
>  fi
> -if test "$trace_backend" = "dtrace"; then
> +if have_backend "dtrace"; then
>    echo "CONFIG_TRACE_DTRACE=y" >> $config_host_mak
>    if test "$trace_backend_stap" = "yes" ; then
>      echo "CONFIG_TRACE_SYSTEMTAP=y" >> $config_host_mak
>    fi
>  fi
> -if test "$trace_backend" = "ftrace"; then
> +if have_backend "ftrace"; then
>    if test "$linux" = "yes" ; then
>      echo "CONFIG_TRACE_FTRACE=y" >> $config_host_mak
> -    trace_default=no
>    else
>      feature_not_found "ftrace(trace backend)" "ftrace requires Linux"
>    fi
>  fi
>  echo "CONFIG_TRACE_FILE=$trace_file" >> $config_host_mak
> -if test "$trace_default" = "yes"; then
> -  echo "CONFIG_TRACE_DEFAULT=y" >> $config_host_mak
> -fi
 
>  if test "$rdma" = "yes" ; then
>    echo "CONFIG_RDMA=y" >> $config_host_mak
> diff --git a/docs/tracing.txt b/docs/tracing.txt
> index bf2e15c..c6ab1c1 100644
> --- a/docs/tracing.txt
> +++ b/docs/tracing.txt
> @@ -9,7 +9,7 @@ for debugging, profiling, and observing execution.
 
>  1. Build with the 'simple' trace backend:
 
> -    ./configure --enable-trace-backend=simple
> +    ./configure --enable-trace-backends=simple
>      make
 
>  2. Create a file with the events you want to trace:
> @@ -142,7 +142,7 @@ script.
>  The trace backend is chosen at configure time and only one trace backend can
>  be built into the binary:
 
> -    ./configure --trace-backend=simple
> +    ./configure --trace-backends=simple
 
>  For a list of supported trace backends, try ./configure --help or see below.
 
> diff --git a/qemu-io.c b/qemu-io.c
> index 9fcd72b..6b84b93 100644
> --- a/qemu-io.c
> +++ b/qemu-io.c
> @@ -425,7 +425,7 @@ int main(int argc, char **argv)
>              }
>              break;
>          case 'T':
> -            if (!trace_backend_init(optarg, NULL)) {
> +            if (!trace_init_backends(optarg, NULL)) {
>                  exit(1); /* error message will have been printed */
>              }
>              break;
> diff --git a/scripts/tracetool.py b/scripts/tracetool.py
> index 5f4890f..59c4810 100755
> --- a/scripts/tracetool.py
> +++ b/scripts/tracetool.py
> @@ -6,7 +6,7 @@ Command-line wrapper for the tracetool machinery.
>  """
 
>  __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
> -__copyright__  = "Copyright 2012, Lluís Vilanova <vilanova@ac.upc.edu>"
> +__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
>  __license__    = "GPL version 2 or (at your option) any later version"
 
>  __maintainer__ = "Stefan Hajnoczi"
> @@ -32,7 +32,7 @@ def error_opt(msg = None):
>      format_descr = "\n".join([ "    %-15s %s" % (n, d)
>                                 for n,d in tracetool.format.get_list() ])
>      error_write("""\
> -Usage: %(script)s --format=<format> --backend=<backend> [<options>]
> +Usage: %(script)s --format=<format> --backends=<backends> [<options>]
 
>  Backends:
>  %(backends)s
> @@ -65,16 +65,17 @@ def main(args):
>      global _SCRIPT
>      _SCRIPT = args[0]
 
> -    long_opts  = [ "backend=", "format=", "help", "list-backends", "check-backend" ]
> -    long_opts += [ "binary=", "target-type=", "target-name=", "probe-prefix=" ]
> +    long_opts = ["backends=", "format=", "help", "list-backends",
> +                 "check-backends"]
> +    long_opts += ["binary=", "target-type=", "target-name=", "probe-prefix="]
 
>      try:
>          opts, args = getopt.getopt(args[1:], "", long_opts)
>      except getopt.GetoptError, err:
>          error_opt(str(err))
 
> -    check_backend = False
> -    arg_backend = ""
> +    check_backends = False
> +    arg_backends = []
>      arg_format = ""
>      binary = None
>      target_type = None
> @@ -84,8 +85,8 @@ def main(args):
>          if opt == "--help":
>              error_opt()
 
> -        elif opt == "--backend":
> -            arg_backend = arg
> +        elif opt == "--backends":
> +            arg_backends = arg.split(",")
>          elif opt == "--format":
>              arg_format = arg
 
> @@ -93,8 +94,8 @@ def main(args):
>              public_backends = tracetool.backend.get_list(only_public = True)
>              out(", ".join([ b for b,_ in public_backends ]))
>              sys.exit(0)
> -        elif opt == "--check-backend":
> -            check_backend = True
> +        elif opt == "--check-backends":
> +            check_backends = True
 
>          elif opt == "--binary":
>              binary = arg
> @@ -108,14 +109,14 @@ def main(args):
>          else:
>              error_opt("unhandled option: %s" % opt)
 
> -    if arg_backend is None:
> -        error_opt("backend not set")
> +    if len(arg_backends) == 0:
> +        error_opt("no backends specified")
 
> -    if check_backend:
> -        if tracetool.backend.exists(arg_backend):
> -            sys.exit(0)
> -        else:
> -            sys.exit(1)
> +    if check_backends:
> +        for backend in arg_backends:
> +            if not tracetool.backend.exists(backend):
> +                sys.exit(1)
> +        sys.exit(0)
 
>      if arg_format == "stap":
>          if binary is None:
> @@ -126,11 +127,11 @@ def main(args):
>              error_opt("--target-name is required for SystemTAP tapset generator")
 
>          if probe_prefix is None:
> -            probe_prefix = ".".join([ "qemu", target_type, target_name ])
> +            probe_prefix = ".".join(["qemu", target_type, target_name])
 
>      try:
> -        tracetool.generate(sys.stdin, arg_format, arg_backend,
> -                           binary = binary, probe_prefix = probe_prefix)
> +        tracetool.generate(sys.stdin, arg_format, arg_backends,
> +                           binary=binary, probe_prefix=probe_prefix)
>      except tracetool.TracetoolError, e:
>          error_opt(str(e))
 
> diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py
> index eccf552..5cc73ac 100644
> --- a/scripts/tracetool/__init__.py
> +++ b/scripts/tracetool/__init__.py
> @@ -233,9 +233,9 @@ def try_import(mod_name, attr_name=None, attr_default=None):
>          return False, None
 
 
> -def generate(fevents, format, backend,
> +def generate(fevents, format, backends,
>               binary=None, probe_prefix=None):
> -    """Generate the output for the given (format, backend) pair.
> +    """Generate the output for the given (format, backends) pair.
 
>      Parameters
>      ----------
> @@ -243,8 +243,8 @@ def generate(fevents, format, backend,
>          Event description file.
>      format : str
>          Output format name.
> -    backend : str
> -        Output backend name.
> +    backends : list
> +        Output backend names.
>      binary : str or None
>          See tracetool.backend.dtrace.BINARY.
>      probe_prefix : str or None
> @@ -260,13 +260,13 @@ def generate(fevents, format, backend,
>          raise TracetoolError("unknown format: %s" % format)
>      format = format.replace("-", "_")
 
> -    backend = str(backend)
> -    if len(backend) is 0:
> -        raise TracetoolError("backend not set")
> -    if not tracetool.backend.exists(backend):
> -        raise TracetoolError("unknown backend: %s" % backend)
> -    backend = backend.replace("-", "_")
> -    backend = tracetool.backend.Wrapper(backend, format)
> +    if len(backends) is 0:
> +        raise TracetoolError("no backends specified")
> +    for backend in backends:
> +        if not tracetool.backend.exists(backend):
> +            raise TracetoolError("unknown backend: %s" % backend)
> +    backends = [backend.replace("-", "_") for backend in backends]
> +    backend = tracetool.backend.Wrapper(backends, format)
 
>      import tracetool.backend.dtrace
>      tracetool.backend.dtrace.BINARY = binary
> diff --git a/scripts/tracetool/backend/__init__.py b/scripts/tracetool/backend/__init__.py
> index 5e36f04..5bfa1ef 100644
> --- a/scripts/tracetool/backend/__init__.py
> +++ b/scripts/tracetool/backend/__init__.py
> @@ -99,17 +99,18 @@ def exists(name):
 
 
>  class Wrapper:
> -    def __init__(self, backend, format):
> -        self._backend = backend.replace("-", "_")
> +    def __init__(self, backends, format):
> +        self._backends = [backend.replace("-", "_") for backend in backends]
>          self._format = format.replace("-", "_")
> -        assert exists(self._backend)
> +        assert all(exists(backend) for backend in self._backends)
>          assert tracetool.format.exists(self._format)
 
>      def _run_function(self, name, *args, **kwargs):
> -        func = tracetool.try_import("tracetool.backend." + self._backend,
> -                                    name % self._format, None)[1]
> -        if func is not None:
> -            func(*args, **kwargs)
> +        for backend in self._backends:
> +            func = tracetool.try_import("tracetool.backend." + backend,
> +                                        name % self._format, None)[1]
> +            if func is not None:
> +                func(*args, **kwargs)
 
>      def generate_begin(self, events):
>          self._run_function("generate_%s_begin", events)
> diff --git a/trace/Makefile.objs b/trace/Makefile.objs
> index 6a30467..e4aa45c 100644
> --- a/trace/Makefile.objs
> +++ b/trace/Makefile.objs
> @@ -3,12 +3,12 @@
>  ######################################################################
>  # Auto-generated event descriptions for LTTng ust code
 
> -ifeq ($(TRACE_BACKEND),ust)
> +ifeq ($(findstring ust,$(TRACE_BACKENDS)),ust)
>  $(obj)/generated-ust-provider.h: $(obj)/generated-ust-provider.h-timestamp
>  $(obj)/generated-ust-provider.h-timestamp: $(SRC_PATH)/trace-events
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=ust-events-h \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
>  	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
> @@ -16,7 +16,7 @@ $(obj)/generated-ust.c: $(obj)/generated-ust.c-timestamp $(BUILD_DIR)/config-hos
>  $(obj)/generated-ust.c-timestamp: $(SRC_PATH)/trace-events
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=ust-events-c \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
>  	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
> @@ -31,7 +31,7 @@ $(obj)/generated-events.h: $(obj)/generated-events.h-timestamp
>  $(obj)/generated-events.h-timestamp: $(SRC_PATH)/trace-events
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=events-h \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
>  	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
> @@ -39,7 +39,7 @@ $(obj)/generated-events.c: $(obj)/generated-events.c-timestamp $(BUILD_DIR)/conf
>  $(obj)/generated-events.c-timestamp: $(SRC_PATH)/trace-events
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=events-c \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
>  	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
> @@ -54,19 +54,19 @@ $(obj)/generated-tracers.h: $(obj)/generated-tracers.h-timestamp
>  $(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=h \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
 
>  ######################################################################
>  # Auto-generated tracing routines (non-DTrace)
 
> -ifneq ($(TRACE_BACKEND),dtrace)
> +ifneq ($(findstring dtrace,$(TRACE_BACKENDS)),dtrace)
>  $(obj)/generated-tracers.c: $(obj)/generated-tracers.c-timestamp
>  	@cmp -s $< $@ || cp $< $@
>  $(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=c \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
 
>  $(obj)/generated-tracers.o: $(obj)/generated-tracers.c $(obj)/generated-tracers.h
> @@ -79,12 +79,12 @@ endif
>  # Normal practice is to name DTrace probe file with a '.d' extension
>  # but that gets picked up by QEMU's Makefile as an external dependency
>  # rule file. So we use '.dtrace' instead
> -ifeq ($(TRACE_BACKEND),dtrace)
> +ifeq ($(findstring dtrace,$(TRACE_BACKENDS)),dtrace)
>  $(obj)/generated-tracers.dtrace: $(obj)/generated-tracers.dtrace-timestamp
>  $(obj)/generated-tracers.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
>  	$(call quiet-command,$(TRACETOOL) \
>  		--format=d \
> -		--backend=$(TRACE_BACKEND) \
> +		--backends=$(TRACE_BACKENDS) \
>  		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
>  	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
> @@ -97,9 +97,7 @@ endif
>  ######################################################################
>  # Backend code
 
> -util-obj-$(CONFIG_TRACE_DEFAULT) += default.o
>  util-obj-$(CONFIG_TRACE_SIMPLE) += simple.o
> -util-obj-$(CONFIG_TRACE_STDERR) += stderr.o
>  util-obj-$(CONFIG_TRACE_FTRACE) += ftrace.o
>  util-obj-$(CONFIG_TRACE_UST) += generated-ust.o
>  util-obj-y += control.o
> diff --git a/trace/control-internal.h b/trace/control-internal.h
> index b3f587e..5a8df28 100644
> --- a/trace/control-internal.h
> +++ b/trace/control-internal.h
> @@ -1,7 +1,7 @@
>  /*
>   * Interface for configuring and controlling the state of tracing events.
>   *
> - * Copyright (C) 2011-2012 Lluís Vilanova <vilanova@ac.upc.edu>
> + * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
>   *
>   * This work is licensed under the terms of the GNU GPL, version 2 or later.
>   * See the COPYING file in the top-level directory.
> @@ -61,7 +61,7 @@ static inline void trace_event_set_state_dynamic(TraceEvent *ev, bool state)
>  {
>      assert(ev != NULL);
>      assert(trace_event_get_state_static(ev));
> -    return trace_event_set_state_dynamic_backend(ev, state);
> +    ev->dstate = state;
>  }
 
>  #endif  /* TRACE__CONTROL_INTERNAL_H */
> diff --git a/trace/control.c b/trace/control.c
> index 49f61e1..09c345c 100644
> --- a/trace/control.c
> +++ b/trace/control.c
> @@ -1,13 +1,19 @@
>  /*
>   * Interface for configuring and controlling the state of tracing events.
>   *
> - * Copyright (C) 2011-2012 Lluís Vilanova <vilanova@ac.upc.edu>
> + * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
>   *
>   * This work is licensed under the terms of the GNU GPL, version 2 or later.
>   * See the COPYING file in the top-level directory.
>   */
 
>  #include "trace/control.h"
> +#ifdef CONFIG_TRACE_SIMPLE
> +#include "trace/simple.h"
> +#endif
> +#ifdef CONFIG_TRACE_FTRACE
> +#include "trace/ftrace.h"
> +#endif
 
 
>  TraceEvent *trace_event_name(const char *name)
> @@ -79,7 +85,20 @@ TraceEvent *trace_event_pattern(const char *pat, TraceEvent *ev)
>      return NULL;
>  }
 
> -void trace_backend_init_events(const char *fname)
> +void trace_print_events(FILE *stream, fprintf_function stream_printf)
> +{
> +    TraceEventID i;
> +
> +    for (i = 0; i < trace_event_count(); i++) {
> +        TraceEvent *ev = trace_event_id(i);
> +        stream_printf(stream, "%s [Event ID %u] : state %u\n",
> +                      trace_event_get_name(ev), i,
> +                      trace_event_get_state_static(ev) &&
> +                      trace_event_get_state_dynamic(ev));
> +    }
> +}
> +
> +static void trace_init_events(const char *fname)
>  {
>      if (fname == NULL) {
>          return;
> @@ -130,3 +149,29 @@ void trace_backend_init_events(const char *fname)
>          exit(1);
>      }
>  }
> +
> +bool trace_init_backends(const char *events, const char *file)
> +{
> +#ifdef CONFIG_TRACE_SIMPLE
> +    if (!st_init(file)) {
> +            fprintf(stderr, "failed to initialize simple tracing backend.\n");
> +            return false;
> +    }
> +#else
> +    if (file) {
> +        fprintf(stderr, "error: -trace file=...: "
> +                "option not supported by the selected tracing backends\n");
> +        return false;
> +    }
> +#endif
> +
> +#ifdef CONFIG_TRACE_FTRACE
> +    if (!ftrace_init()) {
> +            fprintf(stderr, "failed to initialize ftrace backend.\n");
> +            return false;
> +    }
> +#endif
> +
> +    trace_init_events(events);
> +    return true;
> +}
> diff --git a/trace/control.h b/trace/control.h
> index cde8260..e1ec033 100644
> --- a/trace/control.h
> +++ b/trace/control.h
> @@ -1,7 +1,7 @@
>  /*
>   * Interface for configuring and controlling the state of tracing events.
>   *
> - * Copyright (C) 2011-2012 Lluís Vilanova <vilanova@ac.upc.edu>
> + * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
>   *
>   * This work is licensed under the terms of the GNU GPL, version 2 or later.
>   * See the COPYING file in the top-level directory.
> @@ -146,26 +146,17 @@ static bool trace_event_get_state_dynamic(TraceEvent *ev);
>   */
>  static void trace_event_set_state_dynamic(TraceEvent *ev, bool state);
 
> -/**
> - * trace_event_set_state_dynamic_backend:
> - *
> - * Warning: This function must be implemented by each tracing backend.
> - */
> -void trace_event_set_state_dynamic_backend(TraceEvent *ev, bool state);
> -
 
 
>  /**
>   * trace_print_events:
>   *
>   * Print the state of all events.
> - *
> - * Warning: This function must be implemented by each tracing backend.
>   */
>  void trace_print_events(FILE *stream, fprintf_function stream_printf);
 
>  /**
> - * trace_backend_init:
> + * trace_init_backends:
>   * @events: Name of file with events to be enabled at startup; may be NULL.
>   *          Corresponds to commandline option "-trace events=...".
>   * @file:   Name of trace output file; may be NULL.
> @@ -173,19 +164,9 @@ void trace_print_events(FILE *stream, fprintf_function stream_printf);
>   *
>   * Initialize the tracing backend.
>   *
> - * Warning: This function must be implemented by each tracing backend.
> - *
> - * Returns: Whether the backend could be successfully initialized.
> - */
> -bool trace_backend_init(const char *events, const char *file);
> -
> -/**
> - * trace_backend_init_events:
> - * @fname: Name of file with events to enable; may be NULL.
> - *
> - * Generic function to initialize the state of events.
> + * Returns: Whether the backends could be successfully initialized.
>   */
> -void trace_backend_init_events(const char *fname);
> +bool trace_init_backends(const char *events, const char *file);
 
 
>  #include "trace/control-internal.h"
> diff --git a/trace/default.c b/trace/default.c
> deleted file mode 100644
> index 6e07a47..0000000
> --- a/trace/default.c
> +++ /dev/null
> @@ -1,40 +0,0 @@
> -/*
> - * Default implementation for backend initialization from commandline.
> - *
> - * Copyright (C) 2011-2012 Lluís Vilanova <vilanova@ac.upc.edu>
> - *
> - * This work is licensed under the terms of the GNU GPL, version 2.  See
> - * the COPYING file in the top-level directory.
> - */
> -
> -#include "trace/control.h"
> -
> -
> -void trace_print_events(FILE *stream, fprintf_function stream_printf)
> -{
> -    fprintf(stderr, "warning: "
> -            "cannot print the trace events with the current backend\n");
> -    stream_printf(stream, "error: "
> -                  "operation not supported with the current backend\n");
> -}
> -
> -void trace_event_set_state_dynamic_backend(TraceEvent *ev, bool state)
> -{
> -    fprintf(stderr, "warning: "
> -            "cannot set the state of a trace event with the current backend\n");
> -}
> -
> -bool trace_backend_init(const char *events, const char *file)
> -{
> -    if (events) {
> -        fprintf(stderr, "error: -trace events=...: "
> -                "option not supported by the selected tracing backend\n");
> -        return false;
> -    }
> -    if (file) {
> -        fprintf(stderr, "error: -trace file=...: "
> -                "option not supported by the selected tracing backend\n");
> -        return false;
> -    }
> -    return true;
> -}
> diff --git a/trace/ftrace.c b/trace/ftrace.c
> index 46b7fdb..a7ae371 100644
> --- a/trace/ftrace.c
> +++ b/trace/ftrace.c
> @@ -42,35 +42,13 @@ static int find_debugfs(char *debugfs)
>      return 1;
>  }
 
> -void trace_print_events(FILE *stream, fprintf_function stream_printf)
> -{
> -    TraceEventID i;
> -
> -    for (i = 0; i < trace_event_count(); i++) {
> -        TraceEvent *ev = trace_event_id(i);
> -        stream_printf(stream, "%s [Event ID %u] : state %u\n",
> -                      trace_event_get_name(ev), i, trace_event_get_state_dynamic(ev));
> -    }
> -}
> -
> -void trace_event_set_state_dynamic_backend(TraceEvent *ev, bool state)
> -{
> -    ev->dstate = state;
> -}
> -
> -bool trace_backend_init(const char *events, const char *file)
> +bool ftrace_init(void)
>  {
>      char debugfs[PATH_MAX];
>      char path[PATH_MAX];
>      int debugfs_found;
>      int trace_fd = -1;
 
> -    if (file) {
> -        fprintf(stderr, "error: -trace file=...: "
> -                "option not supported by the selected tracing backend\n");
> -        return false;
> -    }
> -
>      debugfs_found = find_debugfs(debugfs);
>      if (debugfs_found) {
>          snprintf(path, PATH_MAX, "%s/tracing/tracing_on", debugfs);
> @@ -97,6 +75,5 @@ bool trace_backend_init(const char *events, const char *file)
>          return false;
>      }
 
> -    trace_backend_init_events(events);
>      return true;
>  }
> diff --git a/trace/ftrace.h b/trace/ftrace.h
> index 94cb8d5..ad18ccb 100644
> --- a/trace/ftrace.h
> +++ b/trace/ftrace.h
> @@ -1,10 +1,15 @@
>  #ifndef TRACE_FTRACE_H
>  #define TRACE_FTRACE_H
 
> +#include <stdint.h>
> +
> +
>  #define MAX_TRACE_STRLEN 512
>  #define _STR(x) #x
>  #define STR(x) _STR(x)
 
>  extern int trace_marker_fd;
 
> +bool ftrace_init(void);
> +
>  #endif /* ! TRACE_FTRACE_H */
> diff --git a/trace/simple.c b/trace/simple.c
> index bb0b52c..67e16e8 100644
> --- a/trace/simple.c
> +++ b/trace/simple.c
> @@ -366,22 +366,6 @@ void st_flush_trace_buffer(void)
>      flush_trace_file(true);
>  }
 
> -void trace_print_events(FILE *stream, fprintf_function stream_printf)
> -{
> -    unsigned int i;
> -
> -    for (i = 0; i < trace_event_count(); i++) {
> -        TraceEvent *ev = trace_event_id(i);
> -        stream_printf(stream, "%s [Event ID %u] : state %u\n",
> -                      trace_event_get_name(ev), i, trace_event_get_state_dynamic(ev));
> -    }
> -}
> -
> -void trace_event_set_state_dynamic_backend(TraceEvent *ev, bool state)
> -{
> -    ev->dstate = state;
> -}
> -
>  /* Helper function to create a thread with signals blocked.  Use glib's
>   * portable threads since QEMU abstractions cannot be used due to reentrancy in
>   * the tracer.  Also note the signal masking on POSIX hosts so that the thread
> @@ -410,7 +394,7 @@ static GThread *trace_thread_create(GThreadFunc fn)
>      return thread;
>  }
 
> -bool trace_backend_init(const char *events, const char *file)
> +bool st_init(const char *file)
>  {
>      GThread *thread;
 
> @@ -426,7 +410,6 @@ bool trace_backend_init(const char *events, const char *file)
>      }
 
>      atexit(st_flush_trace_buffer);
> -    trace_backend_init_events(events);
>      st_set_trace_file(file);
>      return true;
>  }
> diff --git a/trace/simple.h b/trace/simple.h
> index 5260d9a..6997996 100644
> --- a/trace/simple.h
> +++ b/trace/simple.h
> @@ -21,6 +21,7 @@
>  void st_print_trace_file_status(FILE *stream, fprintf_function stream_printf);
>  void st_set_trace_file_enabled(bool enable);
>  bool st_set_trace_file(const char *file);
> +bool st_init(const char *file);
>  void st_flush_trace_buffer(void);
 
>  typedef struct {
> diff --git a/trace/stderr.c b/trace/stderr.c
> deleted file mode 100644
> index e212efd..0000000
> --- a/trace/stderr.c
> +++ /dev/null
> @@ -1,30 +0,0 @@
> -#include "trace.h"
> -#include "trace/control.h"
> -
> -
> -void trace_print_events(FILE *stream, fprintf_function stream_printf)
> -{
> -    TraceEventID i;
> -
> -    for (i = 0; i < trace_event_count(); i++) {
> -        TraceEvent *ev = trace_event_id(i);
> -        stream_printf(stream, "%s [Event ID %u] : state %u\n",
> -                      trace_event_get_name(ev), i, trace_event_get_state_dynamic(ev));
> -    }
> -}
> -
> -void trace_event_set_state_dynamic_backend(TraceEvent *ev, bool state)
> -{
> -    ev->dstate = state;
> -}
> -
> -bool trace_backend_init(const char *events, const char *file)
> -{
> -    if (file) {
> -        fprintf(stderr, "error: -trace file=...: "
> -                "option not supported by the selected tracing backend\n");
> -        return false;
> -    }
> -    trace_backend_init_events(events);
> -    return true;
> -}
> diff --git a/vl.c b/vl.c
> index 73e0661..fbb2771 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -4034,7 +4034,7 @@ int main(int argc, char **argv, char **envp)
>      }
 
>      if (!is_daemonized()) {
> -        if (!trace_backend_init(trace_events, trace_file)) {
> +        if (!trace_init_backends(trace_events, trace_file)) {
>              exit(1);
>          }
>      }
> @@ -4549,7 +4549,7 @@ int main(int argc, char **argv, char **envp)
>      os_setup_post();
 
>      if (is_daemonized()) {
> -        if (!trace_backend_init(trace_events, trace_file)) {
> +        if (!trace_init_backends(trace_events, trace_file)) {
>              exit(1);
>          }
>      }

On Fri, May 09, 2014 at 04:38:51PM +0200, Lluís Vilanova wrote:

Nice.  A few minor points but overall it looks close.

> @@ -260,13 +260,13 @@ def generate(fevents, format, backend,
>          raise TracetoolError("unknown format: %s" % format)
>      format = format.replace("-", "_")
>  
> -    backend = str(backend)
> -    if len(backend) is 0:
> -        raise TracetoolError("backend not set")
> -    if not tracetool.backend.exists(backend):
> -        raise TracetoolError("unknown backend: %s" % backend)
> -    backend = backend.replace("-", "_")
> -    backend = tracetool.backend.Wrapper(backend, format)
> +    if len(backends) is 0:
> +        raise TracetoolError("no backends specified")
> +    for backend in backends:
> +        if not tracetool.backend.exists(backend):
> +            raise TracetoolError("unknown backend: %s" % backend)
> +    backends = [backend.replace("-", "_") for backend in backends]

This seems to be duplicated down...

> +    backend = tracetool.backend.Wrapper(backends, format)
>  
>      import tracetool.backend.dtrace
>      tracetool.backend.dtrace.BINARY = binary
> diff --git a/scripts/tracetool/backend/__init__.py b/scripts/tracetool/backend/__init__.py
> index 5e36f04..5bfa1ef 100644
> --- a/scripts/tracetool/backend/__init__.py
> +++ b/scripts/tracetool/backend/__init__.py
> @@ -99,17 +99,18 @@ def exists(name):
>  
>  
>  class Wrapper:
> -    def __init__(self, backend, format):
> -        self._backend = backend.replace("-", "_")
> +    def __init__(self, backends, format):
> +        self._backends = [backend.replace("-", "_") for backend in backends]

...here.

> @@ -130,3 +149,29 @@ void trace_backend_init_events(const char *fname)
>          exit(1);
>      }
>  }
> +
> +bool trace_init_backends(const char *events, const char *file)
> +{
> +#ifdef CONFIG_TRACE_SIMPLE
> +    if (!st_init(file)) {
> +            fprintf(stderr, "failed to initialize simple tracing backend.\n");
> +            return false;
> +    }
> +#else
> +    if (file) {
> +        fprintf(stderr, "error: -trace file=...: "
> +                "option not supported by the selected tracing backends\n");
> +        return false;
> +    }
> +#endif
> +
> +#ifdef CONFIG_TRACE_FTRACE
> +    if (!ftrace_init()) {
> +            fprintf(stderr, "failed to initialize ftrace backend.\n");
> +            return false;
> +    }
> +#endif

Please use scripts/checkpatch.pl to scan patches for coding style
issues.  QEMU uses 4-space indentation.

> diff --git a/trace/ftrace.h b/trace/ftrace.h
> index 94cb8d5..ad18ccb 100644
> --- a/trace/ftrace.h
> +++ b/trace/ftrace.h
> @@ -1,10 +1,15 @@
>  #ifndef TRACE_FTRACE_H
>  #define TRACE_FTRACE_H
>  
> +#include <stdint.h>

Why do you need this include?

> +
> +
>  #define MAX_TRACE_STRLEN 512
>  #define _STR(x) #x
>  #define STR(x) _STR(x)
>  
>  extern int trace_marker_fd;
>  
> +bool ftrace_init(void);

Missing #include <stdbool.h> for the bool type.