Patchwork [RFC] check: integrate with virt-test (formerly known as KVM Autotest)

login
register
mail settings
Submitter Paolo Bonzini
Date Feb. 28, 2013, 5:58 p.m.
Message ID <1362074323-26649-1-git-send-email-pbonzini@redhat.com>
Download mbox | patch
Permalink /patch/224124/
State New
Headers show

Comments

Paolo Bonzini - Feb. 28, 2013, 5:58 p.m.
This adds a new "check-autotest" target that calls out to the
Autotest-based virt-test testsuite.  The testsuite is very comprehensive,
however the current state of the "developer- oriented" tools only allows a
small subset of tests to be performed.  The tests take around 30 minutes
to run.

To use them, you need a clone of git://github.com/autotest/virt-test.git;
then you can point configure to that clone, and just run "make
check-autotest".  For now, the tests are just run on x86_64, but this
is just because no one bothered to add guest OS descriptions to Autotest
for other devices.  I plan to look into Highbank and/or Versatile Express
and/or pSeries, but help from maintainers is welcome. :)

After adding adding a few more architectures and tests, the full testsuite
shall take no more than 3-4 hours to run (1 hour per target).  The plan
is that it will run (at least for non-x86 targets) on both KVM and TCG.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 configure      |  4 ++++
 tests/Makefile | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 67 insertions(+), 6 deletions(-)

Patch

diff --git a/configure b/configure
index 8789324..00c5960 100755
--- a/configure
+++ b/configure
@@ -626,6 +626,8 @@  for opt do
   ;;
   --with-trace-file=*) trace_file="$optarg"
   ;;
+  --with-virt-tests=*) autotest_dir="$optarg"
+  ;;
   --enable-gprof) gprof="yes"
   ;;
   --enable-gcov) gcov="yes"
@@ -3330,6 +3332,7 @@  echo "libcap-ng support $cap_ng"
 echo "vhost-net support $vhost_net"
 echo "Trace backend     $trace_backend"
 echo "Trace output file $trace_file-<pid>"
+echo "virt-tests repo   ${autotest_dir:-no}"
 echo "spice support     $spice ($spice_protocol_version/$spice_server_version)"
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
@@ -3790,6 +3793,7 @@  echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
 echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
+echo "AUTOTEST_DIR=$autotest_dir" >> $config_host_mak
 if test "$gcov" = "yes" ; then
   echo "CONFIG_GCOV=y" >> $config_host_mak
   echo "GCOV=$gcov_tool" >> $config_host_mak
diff --git a/tests/Makefile b/tests/Makefile
index a2d62b8..c1f4560 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -140,16 +140,73 @@  qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
 qtest-obj-y += tests/libi2c.o tests/libi2c-omap.o
 $(check-qtest-y): $(qtest-obj-y)
 
+# Autotest rules
+
+AUTOTEST_TARGETS = x86_64
+AUTOTEST_TARGETS := $(filter $(AUTOTEST_TARGETS), $(TARGETS))
+
+# Other interesting tests that are currently broken:
+#	block_mirror
+#	cdrom_test
+#	chardev_hotplug
+#	vmstop
+#	block_hotplug
+#	ethtool (needs bridge, slirp is not enough; pretty long, too)
+#	guest_s4
+#	linux_s3
+#	mac_change
+#	nic_bonding
+#	ping.multi_nics
+#	nic_promisc
+#	vlan
+#
+# Some of them due to using slirp or for JeOS-specific failures.
+
+AUTOTEST_TESTS = \
+	block_stream \
+	floppy_test \
+	getfd \
+	migrate.default \
+	nic_hotplug.default \
+	numa_opts \
+	qemu_img \
+	qmp_basic \
+	seabios \
+	system_powerdown \
+	system_reset \
+	usb..usb_tablet \
+	usb..usb_storage \
+	virtio_console.spread_linear.specifiable.virtserialport.virtio_console_smoke \
+	boot \
+	reboot \
+	shutdown
+
+ifneq ($(AUTOTEST_DIR),)
+check-autotest: $(patsubst %, check-autotest-%,$(AUTOTEST_TARGETS))
+$(patsubst %, check-autotest-%,$(AUTOTEST_TARGETS)): check-autotest-%:
+	mkdir -p $(BUILD_DIR)/log-autotest-$* && \
+	  cd $(AUTOTEST_DIR) && \
+	  git clean -Xf qemu/cfg && \
+	  ./run -tqemu --arch=$* -m 256 --smp=1 \
+	    --logs-dir=$(BUILD_DIR)/log-autotest-$* \
+	    --qemu-bin=$(BUILD_DIR)/$*-softmmu/qemu-system-$* \
+	    --tests '$(AUTOTEST_TESTS)'
+endif
+
 .PHONY: check-help
 check-help:
 	@echo "Regression testing targets:"
 	@echo
-	@echo " make check                Run all tests"
-	@echo " make check-qtest-TARGET   Run qtest tests for given target"
-	@echo " make check-qtest          Run qtest tests"
-	@echo " make check-unit           Run qobject tests"
-	@echo " make check-block          Run block tests"
-	@echo " make check-report.html    Generates an HTML test report"
+	@echo " make check                 Run all tests"
+	@echo " make check-qtest-TARGET    Run qtest tests for given target"
+	@echo " make check-qtest           Run qtest tests"
+	@echo " make check-unit            Run qobject tests"
+	@echo " make check-block           Run block tests"
+ifneq ($(AUTOTEST_DIR),)
+	@echo " make check-autotest-TARGET Run virt-test suite for given target"
+	@echo " make check-autotest        Run virt-test suite"
+endif
+	@echo " make check-report.html     Generates an HTML test report"
 	@echo
 	@echo "Please note that HTML reports do not regenerate if the unit tests"
 	@echo "has not changed."