@@ -64,3 +64,6 @@
[submodule "roms/vbootrom"]
path = roms/vbootrom
url = https://gitlab.com/qemu-project/vbootrom.git
+[submodule "tests/lcitool/libvirt-ci"]
+ path = tests/lcitool/libvirt-ci
+ url = http://gitlab.com/libvirt/libvirt-ci
@@ -287,6 +287,7 @@ cscope:
# Needed by "meson install"
export DESTDIR
+include $(SRC_PATH)/tests/lcitool/Makefile.include
include $(SRC_PATH)/tests/docker/Makefile.include
include $(SRC_PATH)/tests/vm/Makefile.include
@@ -316,6 +317,7 @@ endif
@echo 'Test targets:'
$(call print-help,check,Run all tests (check-help for details))
$(call print-help,bench,Run all benchmarks)
+ $(call print-help,lcitool-help,Help about targets for managing build environment manifests)
$(call print-help,docker-help,Help about targets running tests inside containers)
$(call print-help,vm-help,Help about targets running tests inside VM)
@echo ''
@@ -382,14 +382,112 @@ Along with many other images, the ``centos8`` image is defined in a Dockerfile
in ``tests/docker/dockerfiles/``, called ``centos8.docker``. ``make docker-help``
command will list all the available images.
-To add a new image, simply create a new ``.docker`` file under the
-``tests/docker/dockerfiles/`` directory.
-
A ``.pre`` script can be added beside the ``.docker`` file, which will be
executed before building the image under the build context directory. This is
mainly used to do necessary host side setup. One such setup is ``binfmt_misc``,
for example, to make qemu-user powered cross build containers work.
+Most of the existing Dockerfiles were written by hand, simply by creating a
+a new ``.docker`` file under the ``tests/docker/dockerfiles/`` directory.
+This has led to an inconsistent set of packages being present across the
+different containers.
+
+Thus going forward, QEMU is aiming to automatically generate the Dockerfiles
+using the ``lcitool`` program provided by the ``libvirt-ci`` project:
+
+ https://gitlab.com/libvirt/libvirt-ci
+
+In that project, there is a ``mappings.yml`` file defining the distro native
+package names for a wide variety of third party projects. This is processed
+in combination with a project defined list of build pre-requisites to determine
+the list of native packages to install on each distribution. This can be used
+to generate dockerfiles, VM package lists and Cirrus CI variables needed to
+setup build environments across OS distributions with a consistent set of
+packages present.
+
+When preparing a patch series that adds a new build pre-requisite to QEMU,
+updates to various lcitool data files may be required.
+
+
+Adding new build pre-requisites
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the simple case where the pre-requisite is already known to ``libvirt-ci``
+the following steps are needed
+
+ * Edit ``tests/lcitool/projects/qemu.yml`` and add the pre-requisite
+
+ * Run ``make lcitool-refresh`` to re-generate all relevant build environment
+ manifests
+
+In some cases ``libvirt-ci`` will not know about the build pre-requisite and
+thus some extra preparation steps will be required first
+
+ * Fork the ``libvirt-ci`` project on gitlab
+
+ * Edit the ``mappings.yml`` change to add an entry for the new build
+ prerequisite, listing its native package name on as many OS distros
+ as practical.
+
+ * Commit the ``mappings.yml`` change and submit a merge request to
+ the ``libvirt-ci`` project, noting in the description that this
+ is a new build pre-requisite desired for use with QEMU
+
+ * CI pipeline will run to validate that the changes to ``mappings.yml``
+ are correct, by attempting to install the newly listed package on
+ all OS distributions supported by ``libvirt-ci``.
+
+ * Once the merge request is accepted, go back to QEMU and update
+ the ``libvirt-ci`` submodule to point to a commit that contains
+ the ``mappings.yml`` update.
+
+
+Adding new OS distros
+^^^^^^^^^^^^^^^^^^^^^
+
+In some cases ``libvirt-ci`` will not know about the OS distro that is
+desired to be tested. Before adding a new OS distro, discuss the proposed
+addition:
+
+ * Send a mail to qemu-devel, copying people listed in the
+ MAINTAINERS file for ``Build and test automation``.
+
+ There are limited CI compute resources available to QEMU, so the
+ cost/benefit tradeoff of adding new OS distros needs to be considered.
+
+ * File an issue at https://gitlab.com/libvirt/libvirt-ci/-/issues
+ pointing to the qemu-devel mail thread in the archives.
+
+ This alerts other people who might be interested in the work
+ to avoid duplication, as well as to get feedback from libvirt-ci
+ maintainers on any tips to ease the addition
+
+Assuming there is agreement to add a new OS distro then
+
+ * Fork the ``libvirt-ci`` project on gitlab
+
+ * Add metadata under ``guests/lcitool/lcitool/ansible/group_vars/``
+ for the new OS distro. There might be code changes required if
+ the OS distro uses a package format not currently known. The
+ ``libvirt-ci`` maintainers can advise on this when the issue
+ is file.
+
+ * Edit the ``mappings.yml`` change to update all the existing package
+ entries, providing details of the new OS distro
+
+ * Commit the ``mappings.yml`` change and submit a merge request to
+ the ``libvirt-ci`` project, noting in the description that this
+ is a new build pre-requisite desired for use with QEMU
+
+ * CI pipeline will run to validate that the changes to ``mappings.yml``
+ are correct, by attempting to install the newly listed package on
+ all OS distributions supported by ``libvirt-ci``.
+
+ * Once the merge request is accepted, go back to QEMU and update
+ the ``libvirt-ci`` submodule to point to a commit that contains
+ the ``mappings.yml`` update.
+
+
Tests
~~~~~
new file mode 100644
@@ -0,0 +1,17 @@
+
+LCITOOL_REFRESH = $(SRC_PATH)/tests/lcitool/refresh
+
+lcitool:
+ @echo 'Manage build environment manifests'
+ @echo
+ @echo 'Available targets:'
+ @echo
+ @echo ' lcitool: Print this help.'
+ @echo ' lcitool-refresh: Re-generate all build environment manifests.'
+ @echo
+
+lcitool-help: lcitool
+
+lcitool-refresh:
+ $(call quiet-command, git submodule update --init $(SRC_PATH)/tests/lcitool/libvirt-ci)
+ $(call quiet-command, $(LCITOOL_REFRESH))
new file mode 160000
@@ -0,0 +1 @@
+Subproject commit 29cec2153b9a4dbb2e66f1cbc9866a4eff519cfd
new file mode 100644
@@ -0,0 +1,115 @@
+---
+packages:
+ - alsa
+ - bash
+ - bc
+ - brlapi
+ - bzip2
+ - bzip2-libs
+ - capstone
+ - ccache
+ - clang
+ - column
+ - ctags
+ - cyrus-sasl
+ - daxctl
+ - dbus-daemon
+ - device-mapper-multipath
+ - diffutils
+ - dtrace
+ - findutils
+ - g++
+ - gcc
+ - gcovr
+ - gettext
+ - genisoimage
+ - glib2
+ - glib2-static
+ - glibc-static
+ - glusterfs
+ - gnutls
+ - gtk3
+ - hostname
+ - libaio
+ - libattr
+ - libasan
+ - libbpf
+ - libcacard
+ - libcap-ng
+ - libcurl
+ - libdrm
+ - libepoxy
+ - libfdt
+ - libffi
+ - libgcrypt
+ - libibverbs
+ - libiscsi
+ - libjemalloc
+ - libjpeg
+ - libnfs
+ - libnuma
+ - libpmem
+ - libpng
+ - librbd
+ - librdmacm
+ - libseccomp
+ - libselinux
+ - libslirp
+ - libssh
+ - libtasn1
+ - libubsan
+ - libudev
+ - liburing
+ - libusbx
+ - libvdeplug
+ - libxml2
+ - libzstd
+ - llvm
+ - lttng-ust
+ - lzo
+ - netcat
+ - nettle
+ - ninja
+ - nsis
+ - make
+ - mesa-libgbm
+ - meson
+ - ncursesw
+ - pam
+ - pcre-static
+ - perl
+ - perl-Test-Harness
+ - pixman
+ - pkg-config
+ - pulseaudio
+ - python3
+ - python3-PyYAML
+ - python3-numpy
+ - python3-opencv
+ - python3-pillow
+ - python3-pip
+ - python3-sphinx
+ - python3-sphinx-rtd-theme
+ - python3-virtualenv
+ - rpm2cpio
+ - sdl2
+ - sdl2-image
+ - sed
+ - snappy
+ - sparse
+ - spice-protocol
+ - spice-server
+ - ssh-client
+ - systemd
+ - tar
+ - tesseract
+ - tesseract-eng
+ - texinfo
+ - usbredir
+ - virglrenderer
+ - vte
+ - which
+ - xen
+ - xfsprogs
+ - zlib
+ - zlib-static
new file mode 100755
@@ -0,0 +1,67 @@
+#!/usr/bin/python3
+#
+# Re-generate container recipes
+#
+# This script uses the "lcitool" available from
+#
+# https://gitlab.com/libvirt/libvirt-ci
+#
+# Copyright (c) 2020 Red Hat Inc.
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+import sys
+import os
+import subprocess
+
+from pathlib import Path
+
+if len(sys.argv) != 1:
+ print("syntax: %s" % sys.argv[0], file=sys.stderr)
+ sys.exit(1)
+
+self_dir = Path(__file__).parent
+src_dir = self_dir.parent.parent
+dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles")
+
+lcitool_path = Path(self_dir, "libvirt-ci", "lcitool")
+
+lcitool_cmd = [lcitool_path, "--data-dir", self_dir]
+
+def atomic_write(filename, content):
+ tmp = filename.with_suffix(filename.suffix + ".tmp")
+ try:
+ with tmp.open("w") as fp:
+ print(content, file=fp, end="")
+ tmp.rename(filename)
+ except Exception as ex:
+ tmp.unlink()
+ raise
+
+def generate(filename, cmd, trailer):
+ print("Generate %s" % filename)
+ lcitool=subprocess.run(cmd, capture_output=True)
+
+ if lcitool.returncode != 0:
+ raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr))
+
+ content = lcitool.stdout.decode("utf8")
+ if trailer is not None:
+ content += trailer
+ atomic_write(filename, content)
+
+def generate_dockerfile(host, target, cross=None, trailer=None):
+ filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker")
+ cmd = lcitool_cmd + ["dockerfile"]
+ if cross is not None:
+ cmd.extend(["--cross", cross])
+ cmd.extend([target, "qemu"])
+ generate(filename, cmd, trailer)
+
+try:
+ sys.exit(0)
+except Exception as ex:
+ print(str(ex), file=sys.stderr)
+ sys.exit(1)