[v4,03/10] tests: Add vm test lib

This is the common code to implement a "VM test" to

  1) Download and initialize a pre-defined VM that has necessary
  dependencies to build QEMU and SSH access.

  2) Archive $SRC_PATH to a .tar file.

  3) Boot the VM, and pass the source tar file to the guest.

  4) SSH into the VM, untar the source tarball, build from the source.

Signed-off-by: Fam Zheng <famz@redhat.com>
---
 tests/vm/basevm.py | 287 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 287 insertions(+)
 create mode 100755 tests/vm/basevm.py

Hi Fam,

On 08/28/2017 02:47 PM, Fam Zheng wrote:
> This is the common code to implement a "VM test" to
> 
>    1) Download and initialize a pre-defined VM that has necessary
>    dependencies to build QEMU and SSH access.
> 
>    2) Archive $SRC_PATH to a .tar file.
> 
>    3) Boot the VM, and pass the source tar file to the guest.
> 
>    4) SSH into the VM, untar the source tarball, build from the source.
> 
> Signed-off-by: Fam Zheng <famz@redhat.com>
> ---
>   tests/vm/basevm.py | 287 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 287 insertions(+)
>   create mode 100755 tests/vm/basevm.py
> 
> diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
> new file mode 100755
> index 0000000000..d0095c5332
> --- /dev/null
> +++ b/tests/vm/basevm.py
> @@ -0,0 +1,287 @@
> +#!/usr/bin/env python
> +#
> +# VM testing base class
> +#
> +# Copyright (C) 2017 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2.  See
> +# the COPYING file in the top-level directory.
> +#
> +
> +import os
> +import sys
> +import logging
> +import time
> +import datetime
> +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
> +from qemu import QEMUMachine
> +import subprocess
> +import hashlib
> +import optparse
> +import atexit
> +import tempfile
> +import shutil
> +import multiprocessing
> +import traceback
> +
> +SSH_KEY = """\
> +-----BEGIN RSA PRIVATE KEY-----
> +MIIEowIBAAKCAQEAopAuOlmLV6LVHdFBj8/eeOwI9CqguIJPp7eAQSZvOiB4Ag/R
> +coEhl/RBbrV5Yc/SmSD4PTpJO/iM10RwliNjDb4a3I8q3sykRJu9c9PI/YsH8WN9
> ++NH2NjKPtJIcKTu287IM5JYxyB6nDoOzILbTyJ1TDR/xH6qYEfBAyiblggdjcvhA
> +RTf93QIn39F/xLypXvT1K2O9BJEsnJ8lEUvB2UXhKo/JTfSeZF8wPBeowaP9EONk
> +7b+nuJOWHGg68Ji6wVi62tjwl2Szch6lxIhZBpnV7QNRKMfYHP6eIyF4pusazzZq
> +Telsq6xI2ghecWLzb/MF5A+rklsGx2FNuJSAJwIDAQABAoIBAHHi4o/8VZNivz0x
> +cWXn8erzKV6tUoWQvW85Lj/2RiwJvSlsnYZDkx5af1CpEE2HA/pFT8PNRqsd+MWC
> +7AEy710cVsM4BYerBFYQaYxwzblaoojo88LSjVPw3h5Z0iLM8+IMVd36nwuc9dpE
> +R8TecMZ1+U4Tl6BgqkK+9xToZRdPKdjS8L5MoFhGN+xY0vRbbJbGaV9Q0IHxLBkB
> +rEBV7T1mUynneCHRUQlJQEwJmKpT8MH3IjsUXlG5YvnuuvcQJSNTaW2iDLxuOKp8
> +cxW8+qL88zpb1D5dppoIu6rlrugN0azSq70ruFJQPc/A8GQrDKoGgRQiagxNY3u+
> +vHZzXlECgYEA0dKO3gfkSxsDBb94sQwskMScqLhcKhztEa8kPxTx6Yqh+x8/scx3
> +XhJyOt669P8U1v8a/2Al+s81oZzzfQSzO1Q7gEwSrgBcRMSIoRBUw9uYcy02ngb/
> +j/ng3DGivfJztjjiSJwb46FHkJ2JR8mF2UisC6UMXk3NgFY/3vWQx78CgYEAxlcG
> +T3hfSWSmTgKRczMJuHQOX9ULfTBIqwP5VqkkkiavzigGRirzb5lgnmuTSPTpF0LB
> +XVPjR2M4q+7gzP0Dca3pocrvLEoxjwIKnCbYKnyyvnUoE9qHv4Kr+vDbgWpa2LXG
> +JbLmE7tgTCIp20jOPPT4xuDvlbzQZBJ5qCQSoZkCgYEAgrotSSihlCnAOFSTXbu4
> +CHp3IKe8xIBBNENq0eK61kcJpOxTQvOha3sSsJsU4JAM6+cFaxb8kseHIqonCj1j
> +bhOM/uJmwQJ4el/4wGDsbxriYOBKpyq1D38gGhDS1IW6kk3erl6VAb36WJ/OaGum
> +eTpN9vNeQWM4Jj2WjdNx4QECgYAwTdd6mU1TmZCrJRL5ZG+0nYc2rbMrnQvFoqUi
> +BvWiJovggHzur90zy73tNzPaq9Ls2FQxf5G1vCN8NCRJqEEjeYCR59OSDMu/EXc2
> +CnvQ9SevHOdS1oEDEjcCWZCMFzPi3XpRih1gptzQDe31uuiHjf3cqcGPzTlPdfRt
> +D8P92QKBgC4UaBvIRwREVJsdZzpIzm224Bpe8LOmA7DeTnjlT0b3lkGiBJ36/Q0p
> +VhYh/6cjX4/iuIs7gJbGon7B+YPB8scmOi3fj0+nkJAONue1mMfBNkba6qQTc6Y2
> +5mEKw2/O7/JpND7ucU3OK9plcw/qnrWDgHxl0Iz95+OzUIIagxne
> +-----END RSA PRIVATE KEY-----
> +"""
> +SSH_PUB_KEY = """\
> +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCikC46WYtXotUd0UGPz9547Aj0KqC4gk+nt4BBJm86IHgCD9FygSGX9EFutXlhz9KZIPg9Okk7+IzXRHCWI2MNvhrcjyrezKREm71z08j9iwfxY3340fY2Mo+0khwpO7bzsgzkljHIHqcOg7MgttPInVMNH/EfqpgR8EDKJuWCB2Ny+EBFN/3dAiff0X/EvKle9PUrY70EkSycnyURS8HZReEqj8lN9J5kXzA8F6jBo/0Q42Ttv6e4k5YcaDrwmLrBWLra2PCXZLNyHqXEiFkGmdXtA1Eox9gc/p4jIXim6xrPNmpN6WyrrEjaCF5xYvNv8wXkD6uSWwbHYU24lIAn qemu-vm-key
> +"""
> +
> +class BaseVM(object):
> +    GUEST_USER = "qemu"
> +    GUEST_PASS = "qemupass"
> +    ROOT_PASS = "qemupass"
> +
> +    # The script to run in the guest that builds QEMU
> +    BUILD_SCRIPT = ""
> +    # The guest name, to be overridden by subclasses
> +    name = "#base"
> +    def __init__(self, debug=False, vcpus=None):
> +        self._guest = None
> +        self._tmpdir = tempfile.mkdtemp(prefix="qemu-vm-")
> +        atexit.register(shutil.rmtree, self._tmpdir)
> +
> +        self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa")
> +        open(self._ssh_key_file, "w").write(SSH_KEY)
> +        subprocess.check_call(["chmod", "600", self._ssh_key_file])
> +
> +        self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
> +        open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY)
> +
> +        self.debug = debug
> +        self._stderr = sys.stderr
> +        self._devnull = open("/dev/null", "w")
> +        if self.debug:
> +            self._stdout = sys.stdout
> +        else:
> +            self._stdout = self._devnull
> +        self._args = [ \
> +            "-nodefaults", "-m", "2G",
> +            "-cpu", "host",
> +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
> +            "-device", "virtio-net-pci,netdev=vnet",
> +            "-vnc", ":0,to=20",
> +            "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
> +        if vcpus:
> +            self._args += ["-smp", str(vcpus)]
> +        if os.access("/dev/kvm", os.R_OK | os.W_OK):
> +            self._args += ["-enable-kvm"]
> +        else:
> +            logging.info("KVM not available, not using -enable-kvm")
> +        self._data_args = []
> +
> +    def _download_with_cache(self, url, sha256sum=None):
> +        def check_sha256sum(fname):
> +            if not sha256sum:
> +                return True
> +            checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
> +            return sha256sum == checksum
> +
> +        cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
> +        if not os.path.exists(cache_dir):
> +            os.makedirs(cache_dir)
> +        fname = os.path.join(cache_dir, hashlib.sha1(url).hexdigest())
> +        if os.path.exists(fname) and check_sha256sum(fname):
> +            return fname
> +        logging.debug("Downloading %s to %s...", url, fname)
> +        subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"],
> +                              stdout=self._stdout, stderr=self._stderr)
> +        os.rename(fname + ".download", fname)
> +        return fname
> +
> +    def _ssh_do(self, user, cmd, check, interactive=False):
> +        ssh_cmd = ["ssh", "-q",
> +                   "-o", "StrictHostKeyChecking=no",
> +                   "-o", "UserKnownHostsFile=/dev/null",
> +                   "-o", "ConnectTimeout=1",
> +                   "-p", self.ssh_port, "-i", self._ssh_key_file]
> +        if interactive:
> +            ssh_cmd += ['-t']
> +        assert not isinstance(cmd, str)
> +        ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd)
> +        logging.debug("ssh_cmd: %s", " ".join(ssh_cmd))
> +        r = subprocess.call(ssh_cmd,
> +                            stdin=sys.stdin if interactive else self._devnull,
> +                            stdout=sys.stdout if interactive else self._stdout,
> +                            stderr=sys.stderr if interactive else self._stderr)
> +        if check and r != 0:
> +            raise Exception("SSH command failed: %s" % cmd)
> +        return r
> +
> +    def ssh(self, *cmd):
> +        return self._ssh_do(self.GUEST_USER, cmd, False)
> +
> +    def ssh_interactive(self, *cmd):
> +        return self._ssh_do(self.GUEST_USER, cmd, False, True)
> +
> +    def ssh_root(self, *cmd):
> +        return self._ssh_do("root", cmd, False)
> +
> +    def ssh_check(self, *cmd):
> +        self._ssh_do(self.GUEST_USER, cmd, True)
> +
> +    def ssh_root_check(self, *cmd):
> +        self._ssh_do("root", cmd, True)
> +
> +    def build_image(self, img):
> +        raise NotImplementedError
> +
> +    def add_source_dir(self, data_dir):
> +        name = "data-" + hashlib.sha1(data_dir).hexdigest()[:5]
> +        tarfile = os.path.join(self._tmpdir, name + ".tar")
> +        logging.debug("Creating archive %s for data dir: %s", tarfile, data_dir)
> +        if subprocess.call("type gtar", stdout=self._devnull,
> +                           stderr=self._devnull, shell=True) == 0:
> +            tar_cmd = "gtar"
> +        else:
> +            tar_cmd = "tar"
> +        subprocess.check_call([tar_cmd,
> +                               "--exclude-vcs",
> +                               "--exclude=tests/vm/*.img",
> +                               "--exclude=tests/vm/*.img.*",
> +                               "--exclude=*.d",
> +                               "--exclude=*.o",
> +                               "--exclude=docker-src.*",
> +                               "-cf", tarfile, '.'], cwd=data_dir,

I'm not happy with this command :/
My distrib uses tmpfs for /tmp and suddently the whole X window became 
irresponsive until this script failing after filling 8G of /tmp and swap:

...
DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for data 
dir: .
tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
tar: Error is not recoverable: exiting now
Failed to prepare guest environment

Then I figured out my workdir is full of testing stuff, debug images, 
firmwares, coredumps, etc.

I'll think of another way.

> +                              stdin=self._devnull, stdout=self._stdout)
> +        self._data_args += ["-drive",
> +                            "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
> +                                    (tarfile, name),
> +                            "-device",
> +                            "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)]
> +
> +    def boot(self, img, extra_args=[]):
> +        args = self._args + [
> +            "-device", "VGA",
> +            "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img,
> +            "-device", "virtio-blk,drive=drive0,bootindex=0"]
> +        args += self._data_args + extra_args
> +        logging.debug("QEMU args: %s", " ".join(args))
> +        guest = QEMUMachine(binary=os.environ.get("QEMU", "qemu-system-x86_64"),
> +                            args=args)
> +        guest.launch()
> +        atexit.register(self.shutdown)
> +        self._guest = guest
> +        usernet_info = guest.qmp("human-monitor-command",
> +                                 command_line="info usernet")
> +        self.ssh_port = None
> +        for l in usernet_info["return"].splitlines():
> +            fields = l.split()
> +            if "TCP[HOST_FORWARD]" in fields and "22" in fields:
> +                self.ssh_port = l.split()[3]
> +        if not self.ssh_port:
> +            raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
> +                            usernet_info)
> +
> +    def wait_ssh(self, seconds=120):
> +        starttime = datetime.datetime.now()
> +        guest_up = False
> +        while (datetime.datetime.now() - starttime).total_seconds() < seconds:
> +            if self.ssh("exit 0") == 0:
> +                guest_up = True
> +                break
> +            time.sleep(1)
> +        if not guest_up:
> +            raise TimeoutError("Timeout while waiting for guest ssh")
> +
> +    def shutdown(self):
> +        self._guest.shutdown()
> +
> +    def wait(self):
> +        self._guest.wait()
> +
> +    def qmp(self, *args, **kwargs):
> +        return self._guest.qmp(*args, **kwargs)
> +
> +def parse_args(vm_name):
> +    parser = optparse.OptionParser(description="""
> +    VM test utility.  Exit codes: 0 = success, 1 = command line error, 2 = environment initialization failed, 3 = test command failed""")
> +    parser.add_option("--debug", "-D", action="store_true",
> +                      help="enable debug output")
> +    parser.add_option("--image", "-i", default="%s.img" % vm_name,
> +                      help="image file name")
> +    parser.add_option("--force", "-f", action="store_true",
> +                      help="force build image even if image exists")
> +    parser.add_option("--jobs", type=int, default=multiprocessing.cpu_count(),
> +                      help="number of virtual CPUs")
> +    parser.add_option("--build-image", "-b", action="store_true",
> +                      help="build image")
> +    parser.add_option("--build-qemu",
> +                      help="build QEMU from source in guest")
> +    parser.add_option("--interactive", "-I", action="store_true",
> +                      help="Interactively run command")
> +    parser.disable_interspersed_args()
> +    return parser.parse_args()
> +
> +def main(vmcls):
> +    try:
> +        args, argv = parse_args(vmcls.name)
> +        if not argv and not args.build_qemu and not args.build_image:
> +            print "Nothing to do?"
> +            return 1
> +        if args.debug:
> +            logging.getLogger().setLevel(logging.DEBUG)
> +        vm = vmcls(debug=args.debug, vcpus=args.jobs)
> +        if args.build_image:
> +            if os.path.exists(args.image) and not args.force:
> +                sys.stderr.writelines(["Image file exists: %s\n" % args.image,
> +                                      "Use --force option to overwrite\n"])
> +                return 1
> +            return vm.build_image(args.image)
> +        if args.build_qemu:
> +            vm.add_source_dir(args.build_qemu)
> +            cmd = [vm.BUILD_SCRIPT.format(
> +                   configure_opts = " ".join(argv),
> +                   jobs=args.jobs)]
> +        else:
> +            cmd = argv
> +        vm.boot(args.image + ",snapshot=on")
> +        vm.wait_ssh()
> +    except Exception as e:
> +        if isinstance(e, SystemExit) and e.code == 0:
> +            return 0
> +        sys.stderr.write("Failed to prepare guest environment\n")
> +        traceback.print_exc()
> +        return 2
> +
> +    if args.interactive:
> +        if vm.ssh_interactive(*cmd) == 0:
> +            return 0
> +        vm.ssh_interactive()
> +        return 3
> +    else:
> +        if vm.ssh(*cmd) != 0:
> +            return 3
>

On Tue, Aug 29, 2017 at 09:06:48AM -0300, Philippe Mathieu-Daudé wrote:
> Hi Fam,
> 
> On 08/28/2017 02:47 PM, Fam Zheng wrote:
> > This is the common code to implement a "VM test" to
> > 
> >    1) Download and initialize a pre-defined VM that has necessary
> >    dependencies to build QEMU and SSH access.
> > 
> >    2) Archive $SRC_PATH to a .tar file.
> > 
> >    3) Boot the VM, and pass the source tar file to the guest.
> > 
> >    4) SSH into the VM, untar the source tarball, build from the source.
> > 
> > Signed-off-by: Fam Zheng <famz@redhat.com>
> > ---
> >   tests/vm/basevm.py | 287 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> >   1 file changed, 287 insertions(+)
> >   create mode 100755 tests/vm/basevm.py
> > 
> > diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
> > new file mode 100755
> > index 0000000000..d0095c5332
> > --- /dev/null
> > +++ b/tests/vm/basevm.py

> > +    def add_source_dir(self, data_dir):
> > +        name = "data-" + hashlib.sha1(data_dir).hexdigest()[:5]
> > +        tarfile = os.path.join(self._tmpdir, name + ".tar")
> > +        logging.debug("Creating archive %s for data dir: %s", tarfile, data_dir)
> > +        if subprocess.call("type gtar", stdout=self._devnull,
> > +                           stderr=self._devnull, shell=True) == 0:
> > +            tar_cmd = "gtar"
> > +        else:
> > +            tar_cmd = "tar"
> > +        subprocess.check_call([tar_cmd,
> > +                               "--exclude-vcs",
> > +                               "--exclude=tests/vm/*.img",
> > +                               "--exclude=tests/vm/*.img.*",
> > +                               "--exclude=*.d",
> > +                               "--exclude=*.o",
> > +                               "--exclude=docker-src.*",
> > +                               "-cf", tarfile, '.'], cwd=data_dir,
> 
> I'm not happy with this command :/
> My distrib uses tmpfs for /tmp and suddently the whole X window became
> irresponsive until this script failing after filling 8G of /tmp and swap:
> 
> ...
> DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for data dir:
> .
> tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
> tar: Error is not recoverable: exiting now
> Failed to prepare guest environment
> 
> Then I figured out my workdir is full of testing stuff, debug images,
> firmwares, coredumps, etc.
> 
> I'll think of another way.

Yeah, /tmp should never be used for anything which has significant
size. Could go for /var/tmp instead, but IMHO just use the QEMU build
dir, as is done for (almost) all other build & test artifacts and
thus avoid any global dirs.


Regards,
Daniel

On 08/28/2017 02:47 PM, Fam Zheng wrote:
> This is the common code to implement a "VM test" to
> 
>    1) Download and initialize a pre-defined VM that has necessary
>    dependencies to build QEMU and SSH access.
> 
>    2) Archive $SRC_PATH to a .tar file.
> 
>    3) Boot the VM, and pass the source tar file to the guest.
> 
>    4) SSH into the VM, untar the source tarball, build from the source.
> 
> Signed-off-by: Fam Zheng <famz@redhat.com>
> ---
>   tests/vm/basevm.py | 287 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 287 insertions(+)
>   create mode 100755 tests/vm/basevm.py
> 
> diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
> new file mode 100755
> index 0000000000..d0095c5332
> --- /dev/null
> +++ b/tests/vm/basevm.py
> @@ -0,0 +1,287 @@
> +#!/usr/bin/env python
> +#
> +# VM testing base class
> +#
> +# Copyright (C) 2017 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2.  See
> +# the COPYING file in the top-level directory.
> +#
> +
> +import os
> +import sys
> +import logging
> +import time
> +import datetime
> +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
> +from qemu import QEMUMachine
> +import subprocess
> +import hashlib
> +import optparse
> +import atexit
> +import tempfile
> +import shutil
> +import multiprocessing
> +import traceback
> +
> +SSH_KEY = """\
> +-----BEGIN RSA PRIVATE KEY-----
> +MIIEowIBAAKCAQEAopAuOlmLV6LVHdFBj8/eeOwI9CqguIJPp7eAQSZvOiB4Ag/R
> +coEhl/RBbrV5Yc/SmSD4PTpJO/iM10RwliNjDb4a3I8q3sykRJu9c9PI/YsH8WN9
> ++NH2NjKPtJIcKTu287IM5JYxyB6nDoOzILbTyJ1TDR/xH6qYEfBAyiblggdjcvhA
> +RTf93QIn39F/xLypXvT1K2O9BJEsnJ8lEUvB2UXhKo/JTfSeZF8wPBeowaP9EONk
> +7b+nuJOWHGg68Ji6wVi62tjwl2Szch6lxIhZBpnV7QNRKMfYHP6eIyF4pusazzZq
> +Telsq6xI2ghecWLzb/MF5A+rklsGx2FNuJSAJwIDAQABAoIBAHHi4o/8VZNivz0x
> +cWXn8erzKV6tUoWQvW85Lj/2RiwJvSlsnYZDkx5af1CpEE2HA/pFT8PNRqsd+MWC
> +7AEy710cVsM4BYerBFYQaYxwzblaoojo88LSjVPw3h5Z0iLM8+IMVd36nwuc9dpE
> +R8TecMZ1+U4Tl6BgqkK+9xToZRdPKdjS8L5MoFhGN+xY0vRbbJbGaV9Q0IHxLBkB
> +rEBV7T1mUynneCHRUQlJQEwJmKpT8MH3IjsUXlG5YvnuuvcQJSNTaW2iDLxuOKp8
> +cxW8+qL88zpb1D5dppoIu6rlrugN0azSq70ruFJQPc/A8GQrDKoGgRQiagxNY3u+
> +vHZzXlECgYEA0dKO3gfkSxsDBb94sQwskMScqLhcKhztEa8kPxTx6Yqh+x8/scx3
> +XhJyOt669P8U1v8a/2Al+s81oZzzfQSzO1Q7gEwSrgBcRMSIoRBUw9uYcy02ngb/
> +j/ng3DGivfJztjjiSJwb46FHkJ2JR8mF2UisC6UMXk3NgFY/3vWQx78CgYEAxlcG
> +T3hfSWSmTgKRczMJuHQOX9ULfTBIqwP5VqkkkiavzigGRirzb5lgnmuTSPTpF0LB
> +XVPjR2M4q+7gzP0Dca3pocrvLEoxjwIKnCbYKnyyvnUoE9qHv4Kr+vDbgWpa2LXG
> +JbLmE7tgTCIp20jOPPT4xuDvlbzQZBJ5qCQSoZkCgYEAgrotSSihlCnAOFSTXbu4
> +CHp3IKe8xIBBNENq0eK61kcJpOxTQvOha3sSsJsU4JAM6+cFaxb8kseHIqonCj1j
> +bhOM/uJmwQJ4el/4wGDsbxriYOBKpyq1D38gGhDS1IW6kk3erl6VAb36WJ/OaGum
> +eTpN9vNeQWM4Jj2WjdNx4QECgYAwTdd6mU1TmZCrJRL5ZG+0nYc2rbMrnQvFoqUi
> +BvWiJovggHzur90zy73tNzPaq9Ls2FQxf5G1vCN8NCRJqEEjeYCR59OSDMu/EXc2
> +CnvQ9SevHOdS1oEDEjcCWZCMFzPi3XpRih1gptzQDe31uuiHjf3cqcGPzTlPdfRt
> +D8P92QKBgC4UaBvIRwREVJsdZzpIzm224Bpe8LOmA7DeTnjlT0b3lkGiBJ36/Q0p
> +VhYh/6cjX4/iuIs7gJbGon7B+YPB8scmOi3fj0+nkJAONue1mMfBNkba6qQTc6Y2
> +5mEKw2/O7/JpND7ucU3OK9plcw/qnrWDgHxl0Iz95+OzUIIagxne
> +-----END RSA PRIVATE KEY-----
> +"""
> +SSH_PUB_KEY = """\
> +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCikC46WYtXotUd0UGPz9547Aj0KqC4gk+nt4BBJm86IHgCD9FygSGX9EFutXlhz9KZIPg9Okk7+IzXRHCWI2MNvhrcjyrezKREm71z08j9iwfxY3340fY2Mo+0khwpO7bzsgzkljHIHqcOg7MgttPInVMNH/EfqpgR8EDKJuWCB2Ny+EBFN/3dAiff0X/EvKle9PUrY70EkSycnyURS8HZReEqj8lN9J5kXzA8F6jBo/0Q42Ttv6e4k5YcaDrwmLrBWLra2PCXZLNyHqXEiFkGmdXtA1Eox9gc/p4jIXim6xrPNmpN6WyrrEjaCF5xYvNv8wXkD6uSWwbHYU24lIAn qemu-vm-key
> +"""
> +
> +class BaseVM(object):
> +    GUEST_USER = "qemu"
> +    GUEST_PASS = "qemupass"
> +    ROOT_PASS = "qemupass"
> +
> +    # The script to run in the guest that builds QEMU
> +    BUILD_SCRIPT = ""
> +    # The guest name, to be overridden by subclasses
> +    name = "#base"
> +    def __init__(self, debug=False, vcpus=None):
> +        self._guest = None
> +        self._tmpdir = tempfile.mkdtemp(prefix="qemu-vm-")
> +        atexit.register(shutil.rmtree, self._tmpdir)
> +
> +        self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa")
> +        open(self._ssh_key_file, "w").write(SSH_KEY)
> +        subprocess.check_call(["chmod", "600", self._ssh_key_file])
> +
> +        self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
> +        open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY)
> +
> +        self.debug = debug
> +        self._stderr = sys.stderr
> +        self._devnull = open("/dev/null", "w")
> +        if self.debug:
> +            self._stdout = sys.stdout
> +        else:
> +            self._stdout = self._devnull
> +        self._args = [ \
> +            "-nodefaults", "-m", "2G",
> +            "-cpu", "host",
> +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",

Testing with debian/unstable:

$ make vm-build-netbsd V=1
./tests/vm/netbsd  --debug   --image "tests/vm/netbsd.img" --build-qemu .
DEBUG:root:Creating archive /tmp/qemu-vm-PxfXNv/data-3a52c.tar for data 
dir: .
DEBUG:root:QEMU args: -nodefaults -m 2G -cpu host -netdev 
user,id=vnet,hostfwd=:0.0.0.0:0-:22 -device virtio-net-pci,netdev=vnet 
-vnc :0,to=20 -serial file:/tmp/qemu-vm-PxfXNv/serial.out -smp 4 
-enable-kvm -device VGA -drive 
file=tests/vm/netbsd.img,snapshot=on,if=none,id=drive0,cache=writeback 
-device virtio-blk,drive=drive0,bootindex=0 -drive 
file=/tmp/qemu-vm-PxfXNv/data-3a52c.tar,if=none,id=data-3a52c,cache=writeback,format=raw 
-device virtio-blk,drive=data-3a52c,serial=data-3a52c,bootindex=1
Failed to prepare guest environment
Traceback (most recent call last):
   File "/source/qemu/tests/vm/basevm.py", line 274, in main
     vm.boot(args.image + ",snapshot=on")
   File "/source/qemu/tests/vm/basevm.py", line 198, in boot
     guest.launch()
   File "/source/qemu/tests/vm/../../scripts/qemu.py", line 137, in launch
     self._post_launch()
   File "/source/qemu/tests/vm/../../scripts/qemu.py", line 121, in 
_post_launch
     self._qmp.accept()
   File "/source/qemu/tests/vm/../../scripts/qmp/qmp.py", line 147, in 
accept
     return self.__negotiate_capabilities()
   File "/source/qemu/tests/vm/../../scripts/qmp/qmp.py", line 60, in 
__negotiate_capabilities
     raise QMPConnectError
QMPConnectError
tests/vm/Makefile.include:32: recipe for target 'vm-build-netbsd' failed
make: *** [vm-build-netbsd] Error 2

Having:

$ qemu-system-x86_64 -version
QEMU emulator version 2.8.1(Debian 1:2.8+dfsg-7)

Hopefully I could get it working with a /master build:

$ QEMU=/source/qemu/build/x86_64-softmmu/qemu-system-x86_64 make 
vm-build-netbsd
...

> +            "-device", "virtio-net-pci,netdev=vnet",
> +            "-vnc", ":0,to=20",
> +            "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
> +        if vcpus:
> +            self._args += ["-smp", str(vcpus)]
> +        if os.access("/dev/kvm", os.R_OK | os.W_OK):
> +            self._args += ["-enable-kvm"]
> +        else:
> +            logging.info("KVM not available, not using -enable-kvm")
> +        self._data_args = []
> +
> +    def _download_with_cache(self, url, sha256sum=None):
> +        def check_sha256sum(fname):
> +            if not sha256sum:
> +                return True
> +            checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
> +            return sha256sum == checksum
> +
> +        cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
> +        if not os.path.exists(cache_dir):
> +            os.makedirs(cache_dir)
> +        fname = os.path.join(cache_dir, hashlib.sha1(url).hexdigest())
> +        if os.path.exists(fname) and check_sha256sum(fname):
> +            return fname
> +        logging.debug("Downloading %s to %s...", url, fname)
> +        subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"],
> +                              stdout=self._stdout, stderr=self._stderr)
> +        os.rename(fname + ".download", fname)
> +        return fname
> +
> +    def _ssh_do(self, user, cmd, check, interactive=False):
> +        ssh_cmd = ["ssh", "-q",
> +                   "-o", "StrictHostKeyChecking=no",
> +                   "-o", "UserKnownHostsFile=/dev/null",
> +                   "-o", "ConnectTimeout=1",
> +                   "-p", self.ssh_port, "-i", self._ssh_key_file]
> +        if interactive:
> +            ssh_cmd += ['-t']
> +        assert not isinstance(cmd, str)
> +        ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd)
> +        logging.debug("ssh_cmd: %s", " ".join(ssh_cmd))
> +        r = subprocess.call(ssh_cmd,
> +                            stdin=sys.stdin if interactive else self._devnull,
> +                            stdout=sys.stdout if interactive else self._stdout,
> +                            stderr=sys.stderr if interactive else self._stderr)
> +        if check and r != 0:
> +            raise Exception("SSH command failed: %s" % cmd)
> +        return r
> +
> +    def ssh(self, *cmd):
> +        return self._ssh_do(self.GUEST_USER, cmd, False)
> +
> +    def ssh_interactive(self, *cmd):
> +        return self._ssh_do(self.GUEST_USER, cmd, False, True)
> +
> +    def ssh_root(self, *cmd):
> +        return self._ssh_do("root", cmd, False)
> +
> +    def ssh_check(self, *cmd):
> +        self._ssh_do(self.GUEST_USER, cmd, True)
> +
> +    def ssh_root_check(self, *cmd):
> +        self._ssh_do("root", cmd, True)
> +
> +    def build_image(self, img):
> +        raise NotImplementedError
> +
> +    def add_source_dir(self, data_dir):
> +        name = "data-" + hashlib.sha1(data_dir).hexdigest()[:5]
> +        tarfile = os.path.join(self._tmpdir, name + ".tar")
> +        logging.debug("Creating archive %s for data dir: %s", tarfile, data_dir)
> +        if subprocess.call("type gtar", stdout=self._devnull,
> +                           stderr=self._devnull, shell=True) == 0:
> +            tar_cmd = "gtar"
> +        else:
> +            tar_cmd = "tar"
> +        subprocess.check_call([tar_cmd,
> +                               "--exclude-vcs",
> +                               "--exclude=tests/vm/*.img",
> +                               "--exclude=tests/vm/*.img.*",
> +                               "--exclude=*.d",
> +                               "--exclude=*.o",
> +                               "--exclude=docker-src.*",
> +                               "-cf", tarfile, '.'], cwd=data_dir,
> +                              stdin=self._devnull, stdout=self._stdout)
> +        self._data_args += ["-drive",
> +                            "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
> +                                    (tarfile, name),
> +                            "-device",
> +                            "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)]
> +
> +    def boot(self, img, extra_args=[]):
> +        args = self._args + [
> +            "-device", "VGA",
> +            "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img,
> +            "-device", "virtio-blk,drive=drive0,bootindex=0"]
> +        args += self._data_args + extra_args
> +        logging.debug("QEMU args: %s", " ".join(args))
> +        guest = QEMUMachine(binary=os.environ.get("QEMU", "qemu-system-x86_64"),
> +                            args=args)
> +        guest.launch()
> +        atexit.register(self.shutdown)
> +        self._guest = guest
> +        usernet_info = guest.qmp("human-monitor-command",
> +                                 command_line="info usernet")
> +        self.ssh_port = None
> +        for l in usernet_info["return"].splitlines():
> +            fields = l.split()
> +            if "TCP[HOST_FORWARD]" in fields and "22" in fields:
> +                self.ssh_port = l.split()[3]
> +        if not self.ssh_port:
> +            raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
> +                            usernet_info)
> +
> +    def wait_ssh(self, seconds=120):
> +        starttime = datetime.datetime.now()
> +        guest_up = False
> +        while (datetime.datetime.now() - starttime).total_seconds() < seconds:
> +            if self.ssh("exit 0") == 0:
> +                guest_up = True
> +                break
> +            time.sleep(1)
> +        if not guest_up:
> +            raise TimeoutError("Timeout while waiting for guest ssh")
> +
> +    def shutdown(self):
> +        self._guest.shutdown()
> +
> +    def wait(self):
> +        self._guest.wait()
> +
> +    def qmp(self, *args, **kwargs):
> +        return self._guest.qmp(*args, **kwargs)
> +
> +def parse_args(vm_name):
> +    parser = optparse.OptionParser(description="""
> +    VM test utility.  Exit codes: 0 = success, 1 = command line error, 2 = environment initialization failed, 3 = test command failed""")
> +    parser.add_option("--debug", "-D", action="store_true",
> +                      help="enable debug output")
> +    parser.add_option("--image", "-i", default="%s.img" % vm_name,
> +                      help="image file name")
> +    parser.add_option("--force", "-f", action="store_true",
> +                      help="force build image even if image exists")
> +    parser.add_option("--jobs", type=int, default=multiprocessing.cpu_count(),
> +                      help="number of virtual CPUs")
> +    parser.add_option("--build-image", "-b", action="store_true",
> +                      help="build image")
> +    parser.add_option("--build-qemu",
> +                      help="build QEMU from source in guest")
> +    parser.add_option("--interactive", "-I", action="store_true",
> +                      help="Interactively run command")
> +    parser.disable_interspersed_args()
> +    return parser.parse_args()
> +
> +def main(vmcls):
> +    try:
> +        args, argv = parse_args(vmcls.name)
> +        if not argv and not args.build_qemu and not args.build_image:
> +            print "Nothing to do?"
> +            return 1
> +        if args.debug:
> +            logging.getLogger().setLevel(logging.DEBUG)
> +        vm = vmcls(debug=args.debug, vcpus=args.jobs)
> +        if args.build_image:
> +            if os.path.exists(args.image) and not args.force:
> +                sys.stderr.writelines(["Image file exists: %s\n" % args.image,
> +                                      "Use --force option to overwrite\n"])
> +                return 1
> +            return vm.build_image(args.image)
> +        if args.build_qemu:
> +            vm.add_source_dir(args.build_qemu)
> +            cmd = [vm.BUILD_SCRIPT.format(
> +                   configure_opts = " ".join(argv),
> +                   jobs=args.jobs)]
> +        else:
> +            cmd = argv
> +        vm.boot(args.image + ",snapshot=on")
> +        vm.wait_ssh()
> +    except Exception as e:
> +        if isinstance(e, SystemExit) and e.code == 0:
> +            return 0
> +        sys.stderr.write("Failed to prepare guest environment\n")
> +        traceback.print_exc()
> +        return 2
> +
> +    if args.interactive:
> +        if vm.ssh_interactive(*cmd) == 0:
> +            return 0
> +        vm.ssh_interactive()
> +        return 3
> +    else:
> +        if vm.ssh(*cmd) != 0:
> +            return 3
>

Hi Fam, Kamil,

> On 08/28/2017 02:47 PM, Fam Zheng wrote:
[...]
>> +        subprocess.check_call([tar_cmd,
>> +                               "--exclude-vcs",
>> +                               "--exclude=tests/vm/*.img",
>> +                               "--exclude=tests/vm/*.img.*",
>> +                               "--exclude=*.d",
>> +                               "--exclude=*.o",
>> +                               "--exclude=docker-src.*",
>> +                               "-cf", tarfile, '.'], cwd=data_dir,
> 
> I'm not happy with this command :/
> My distrib uses tmpfs for /tmp and suddently the whole X window became 
> irresponsive until this script failing after filling 8G of /tmp and swap:
> 
> ...
> DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for data 
> dir: .
> tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
> tar: Error is not recoverable: exiting now
> Failed to prepare guest environment
> 
> Then I figured out my workdir is full of testing stuff, debug images, 
> firmwares, coredumps, etc.
> 
> I'll think of another way.

Using:

(git ls-files; git submodule foreach --recursive "git ls-files | sed 
s_^_\$sm_path/_" | sed /^Entering/d) > files.txt

and:

         subprocess.check_call([tar_cmd,
                                "-cf", tarfile,
                                "-v" if self.debug else "",
                                "-T", "files.txt"], cwd=data_dir,
                               stdin=self._devnull, stdout=self._stdout)

Current /master and submodules generated tarball is ~305MB uncompressed.

Kamil do you mind testing this command on NetBSD?

(obviously 'files.txt' goes in tmpdir).

Regards,

Phil.

On 29.08.2017 15:10, Philippe Mathieu-Daudé wrote:
> Hi Fam, Kamil,
> 
>> On 08/28/2017 02:47 PM, Fam Zheng wrote:
> [...]
>>> +        subprocess.check_call([tar_cmd,
>>> +                               "--exclude-vcs",
>>> +                               "--exclude=tests/vm/*.img",
>>> +                               "--exclude=tests/vm/*.img.*",
>>> +                               "--exclude=*.d",
>>> +                               "--exclude=*.o",
>>> +                               "--exclude=docker-src.*",
>>> +                               "-cf", tarfile, '.'], cwd=data_dir,
>>
>> I'm not happy with this command :/
>> My distrib uses tmpfs for /tmp and suddently the whole X window became
>> irresponsive until this script failing after filling 8G of /tmp and swap:
>>
>> ...
>> DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for
>> data dir: .
>> tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
>> tar: Error is not recoverable: exiting now
>> Failed to prepare guest environment
>>
>> Then I figured out my workdir is full of testing stuff, debug images,
>> firmwares, coredumps, etc.
>>
>> I'll think of another way.
> 
> Using:
> 
> (git ls-files; git submodule foreach --recursive "git ls-files | sed
> s_^_\$sm_path/_" | sed /^Entering/d) > files.txt
> 
> and:
> 
>         subprocess.check_call([tar_cmd,
>                                "-cf", tarfile,
>                                "-v" if self.debug else "",
>                                "-T", "files.txt"], cwd=data_dir,
>                               stdin=self._devnull, stdout=self._stdout)
> 
> Current /master and submodules generated tarball is ~305MB uncompressed.
> 
> Kamil do you mind testing this command on NetBSD?
> 
> (obviously 'files.txt' goes in tmpdir).
> 

Result:

http://www.netbsd.org/~kamil/qemu/files.txt

I've tested this command line on a checkout from Jul 17th rev. 77031ee1ce4c

> Regards,
> 
> Phil.
>

On 08/29/2017 10:22 AM, Kamil Rytarowski wrote:
> On 29.08.2017 15:10, Philippe Mathieu-Daudé wrote:
>> Hi Fam, Kamil,
>>
>>> On 08/28/2017 02:47 PM, Fam Zheng wrote:
>> [...]
>>>> +        subprocess.check_call([tar_cmd,
>>>> +                               "--exclude-vcs",
>>>> +                               "--exclude=tests/vm/*.img",
>>>> +                               "--exclude=tests/vm/*.img.*",
>>>> +                               "--exclude=*.d",
>>>> +                               "--exclude=*.o",
>>>> +                               "--exclude=docker-src.*",
>>>> +                               "-cf", tarfile, '.'], cwd=data_dir,
>>>
>>> I'm not happy with this command :/
>>> My distrib uses tmpfs for /tmp and suddently the whole X window became
>>> irresponsive until this script failing after filling 8G of /tmp and swap:
>>>
>>> ...
>>> DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for
>>> data dir: .
>>> tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
>>> tar: Error is not recoverable: exiting now
>>> Failed to prepare guest environment
>>>
>>> Then I figured out my workdir is full of testing stuff, debug images,
>>> firmwares, coredumps, etc.
>>>
>>> I'll think of another way.
>>
>> Using:
>>
>> (git ls-files; git submodule foreach --recursive "git ls-files | sed
>> s_^_\$sm_path/_" | sed /^Entering/d) > files.txt
>>
>> and:
>>
>>          subprocess.check_call([tar_cmd,
>>                                 "-cf", tarfile,
>>                                 "-v" if self.debug else "",
>>                                 "-T", "files.txt"], cwd=data_dir,
>>                                stdin=self._devnull, stdout=self._stdout)
>>
>> Current /master and submodules generated tarball is ~305MB uncompressed.
>>
>> Kamil do you mind testing this command on NetBSD?
>>
>> (obviously 'files.txt' goes in tmpdir).
>>
> 
> Result:
> 
> http://www.netbsd.org/~kamil/qemu/files.txt

\o/

> I've tested this command line on a checkout from Jul 17th rev. 77031ee1ce4c

Thanks Kamil :)

Fam are you OK using this command instead?

Regards,

Phil.

On 08/28/2017 02:47 PM, Fam Zheng wrote:
[...]
> +    def __init__(self, debug=False, vcpus=None):
> +        self._guest = None
> +        self._tmpdir = tempfile.mkdtemp(prefix="qemu-vm-")
> +        atexit.register(shutil.rmtree, self._tmpdir)
> +
> +        self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa")
> +        open(self._ssh_key_file, "w").write(SSH_KEY)
> +        subprocess.check_call(["chmod", "600", self._ssh_key_file])
> +
> +        self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
> +        open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY)
> +
> +        self.debug = debug
> +        self._stderr = sys.stderr
> +        self._devnull = open("/dev/null", "w")
> +        if self.debug:
> +            self._stdout = sys.stdout
> +        else:
> +            self._stdout = self._devnull
> +        self._args = [ \
> +            "-nodefaults", "-m", "2G",
> +            "-cpu", "host",
> +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
> +            "-device", "virtio-net-pci,netdev=vnet",
> +            "-vnc", ":0,to=20",
> +            "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
> +        if vcpus:
> +            self._args += ["-smp", str(vcpus)]

What about enabling mttcg which isn't default?

             self._args += ["--accel", "tcg,thread=multi"]

> +        if os.access("/dev/kvm", os.R_OK | os.W_OK):
> +            self._args += ["-enable-kvm"]
> +        else:
> +            logging.info("KVM not available, not using -enable-kvm")
> +        self._data_args = []
[...]

On Tue, 08/29 13:11, Daniel P. Berrange wrote:
> On Tue, Aug 29, 2017 at 09:06:48AM -0300, Philippe Mathieu-Daudé wrote:
> > Hi Fam,
> > 
> > On 08/28/2017 02:47 PM, Fam Zheng wrote:
> > > This is the common code to implement a "VM test" to
> > > 
> > >    1) Download and initialize a pre-defined VM that has necessary
> > >    dependencies to build QEMU and SSH access.
> > > 
> > >    2) Archive $SRC_PATH to a .tar file.
> > > 
> > >    3) Boot the VM, and pass the source tar file to the guest.
> > > 
> > >    4) SSH into the VM, untar the source tarball, build from the source.
> > > 
> > > Signed-off-by: Fam Zheng <famz@redhat.com>
> > > ---
> > >   tests/vm/basevm.py | 287 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> > >   1 file changed, 287 insertions(+)
> > >   create mode 100755 tests/vm/basevm.py
> > > 
> > > diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py
> > > new file mode 100755
> > > index 0000000000..d0095c5332
> > > --- /dev/null
> > > +++ b/tests/vm/basevm.py
> 
> > > +    def add_source_dir(self, data_dir):
> > > +        name = "data-" + hashlib.sha1(data_dir).hexdigest()[:5]
> > > +        tarfile = os.path.join(self._tmpdir, name + ".tar")
> > > +        logging.debug("Creating archive %s for data dir: %s", tarfile, data_dir)
> > > +        if subprocess.call("type gtar", stdout=self._devnull,
> > > +                           stderr=self._devnull, shell=True) == 0:
> > > +            tar_cmd = "gtar"
> > > +        else:
> > > +            tar_cmd = "tar"
> > > +        subprocess.check_call([tar_cmd,
> > > +                               "--exclude-vcs",
> > > +                               "--exclude=tests/vm/*.img",
> > > +                               "--exclude=tests/vm/*.img.*",
> > > +                               "--exclude=*.d",
> > > +                               "--exclude=*.o",
> > > +                               "--exclude=docker-src.*",
> > > +                               "-cf", tarfile, '.'], cwd=data_dir,
> > 
> > I'm not happy with this command :/
> > My distrib uses tmpfs for /tmp and suddently the whole X window became
> > irresponsive until this script failing after filling 8G of /tmp and swap:
> > 
> > ...
> > DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for data dir:
> > .
> > tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
> > tar: Error is not recoverable: exiting now
> > Failed to prepare guest environment
> > 
> > Then I figured out my workdir is full of testing stuff, debug images,
> > firmwares, coredumps, etc.
> > 
> > I'll think of another way.
> 
> Yeah, /tmp should never be used for anything which has significant
> size. Could go for /var/tmp instead, but IMHO just use the QEMU build
> dir, as is done for (almost) all other build & test artifacts and
> thus avoid any global dirs.

Thanks, I'll fix it. Using current dir would be fine.

Fam

> 
> 
> Regards,
> Daniel
> -- 
> |: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org         -o-            https://fstop138.berrange.com :|
> |: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

On Tue, 08/29 09:15, Philippe Mathieu-Daudé wrote:
> > +        self._args = [ \
> > +            "-nodefaults", "-m", "2G",
> > +            "-cpu", "host",
> > +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
> 
> Testing with debian/unstable:
> 
> $ make vm-build-netbsd V=1
> ./tests/vm/netbsd  --debug   --image "tests/vm/netbsd.img" --build-qemu .
> DEBUG:root:Creating archive /tmp/qemu-vm-PxfXNv/data-3a52c.tar for data dir:
> .
> DEBUG:root:QEMU args: -nodefaults -m 2G -cpu host -netdev
> user,id=vnet,hostfwd=:0.0.0.0:0-:22 -device virtio-net-pci,netdev=vnet -vnc
> :0,to=20 -serial file:/tmp/qemu-vm-PxfXNv/serial.out -smp 4 -enable-kvm
> -device VGA -drive
> file=tests/vm/netbsd.img,snapshot=on,if=none,id=drive0,cache=writeback
> -device virtio-blk,drive=drive0,bootindex=0 -drive file=/tmp/qemu-vm-PxfXNv/data-3a52c.tar,if=none,id=data-3a52c,cache=writeback,format=raw
> -device virtio-blk,drive=data-3a52c,serial=data-3a52c,bootindex=1
> Failed to prepare guest environment

Can you please look into the stderr of the QEMU command line to see what
arguments went wrong? (I hope the qemu.py improvement patches on the list can
give a better error message in such cases.)

Fam

On Tue, 08/29 14:34, Philippe Mathieu-Daudé wrote:
> > +        self._args = [ \
> > +            "-nodefaults", "-m", "2G",
> > +            "-cpu", "host",
> > +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
> > +            "-device", "virtio-net-pci,netdev=vnet",
> > +            "-vnc", ":0,to=20",
> > +            "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
> > +        if vcpus:
> > +            self._args += ["-smp", str(vcpus)]
> 
> What about enabling mttcg which isn't default?
> 
>             self._args += ["--accel", "tcg,thread=multi"]

Any specific reason to enable it? I think it is not available on older QEMU.

> 
> > +        if os.access("/dev/kvm", os.R_OK | os.W_OK):
> > +            self._args += ["-enable-kvm"]
> > +        else:
> > +            logging.info("KVM not available, not using -enable-kvm")
> > +        self._data_args = []
> [...]

Fam

On Wed, 08/30 11:29, Fam Zheng wrote:
> On Tue, 08/29 09:15, Philippe Mathieu-Daudé wrote:
> > > +        self._args = [ \
> > > +            "-nodefaults", "-m", "2G",
> > > +            "-cpu", "host",
> > > +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
> > 
> > Testing with debian/unstable:
> > 
> > $ make vm-build-netbsd V=1
> > ./tests/vm/netbsd  --debug   --image "tests/vm/netbsd.img" --build-qemu .
> > DEBUG:root:Creating archive /tmp/qemu-vm-PxfXNv/data-3a52c.tar for data dir:
> > .
> > DEBUG:root:QEMU args: -nodefaults -m 2G -cpu host -netdev
> > user,id=vnet,hostfwd=:0.0.0.0:0-:22 -device virtio-net-pci,netdev=vnet -vnc
> > :0,to=20 -serial file:/tmp/qemu-vm-PxfXNv/serial.out -smp 4 -enable-kvm
> > -device VGA -drive
> > file=tests/vm/netbsd.img,snapshot=on,if=none,id=drive0,cache=writeback
> > -device virtio-blk,drive=drive0,bootindex=0 -drive file=/tmp/qemu-vm-PxfXNv/data-3a52c.tar,if=none,id=data-3a52c,cache=writeback,format=raw
> > -device virtio-blk,drive=data-3a52c,serial=data-3a52c,bootindex=1
> > Failed to prepare guest environment
> 
> Can you please look into the stderr of the QEMU command line to see what
> arguments went wrong? (I hope the qemu.py improvement patches on the list can
> give a better error message in such cases.)

I've tested with Fedora 26's qemu-system-x86-2.9.0-5.fc26.x86_64 and got the
failure due to the hostfwd syntax:

qemu-system-x86_64: -netdev user,id=vnet,hostfwd=:0.0.0.0:0-:22: invalid host forwarding rule ':0.0.0.0:0-:22'
qemu-system-x86_64: -netdev user,id=vnet,hostfwd=:0.0.0.0:0-:22: Device 'user' could not be initialized

But it makes sense to use dynamic port allocation to avoid collision.

Since it's about developing QEMU, using cutting edge QEMU features is not that
bad. Let's keep this one.

Fam

On Tue, 08/29 10:35, Philippe Mathieu-Daudé wrote:
> On 08/29/2017 10:22 AM, Kamil Rytarowski wrote:
> > On 29.08.2017 15:10, Philippe Mathieu-Daudé wrote:
> > > Hi Fam, Kamil,
> > > 
> > > > On 08/28/2017 02:47 PM, Fam Zheng wrote:
> > > [...]
> > > > > +        subprocess.check_call([tar_cmd,
> > > > > +                               "--exclude-vcs",
> > > > > +                               "--exclude=tests/vm/*.img",
> > > > > +                               "--exclude=tests/vm/*.img.*",
> > > > > +                               "--exclude=*.d",
> > > > > +                               "--exclude=*.o",
> > > > > +                               "--exclude=docker-src.*",
> > > > > +                               "-cf", tarfile, '.'], cwd=data_dir,
> > > > 
> > > > I'm not happy with this command :/
> > > > My distrib uses tmpfs for /tmp and suddently the whole X window became
> > > > irresponsive until this script failing after filling 8G of /tmp and swap:
> > > > 
> > > > ...
> > > > DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for
> > > > data dir: .
> > > > tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
> > > > tar: Error is not recoverable: exiting now
> > > > Failed to prepare guest environment
> > > > 
> > > > Then I figured out my workdir is full of testing stuff, debug images,
> > > > firmwares, coredumps, etc.
> > > > 
> > > > I'll think of another way.
> > > 
> > > Using:
> > > 
> > > (git ls-files; git submodule foreach --recursive "git ls-files | sed
> > > s_^_\$sm_path/_" | sed /^Entering/d) > files.txt
> > > 
> > > and:
> > > 
> > >          subprocess.check_call([tar_cmd,
> > >                                 "-cf", tarfile,
> > >                                 "-v" if self.debug else "",
> > >                                 "-T", "files.txt"], cwd=data_dir,
> > >                                stdin=self._devnull, stdout=self._stdout)
> > > 
> > > Current /master and submodules generated tarball is ~305MB uncompressed.
> > > 
> > > Kamil do you mind testing this command on NetBSD?
> > > 
> > > (obviously 'files.txt' goes in tmpdir).
> > > 
> > 
> > Result:
> > 
> > http://www.netbsd.org/~kamil/qemu/files.txt
> 
> \o/
> 
> > I've tested this command line on a checkout from Jul 17th rev. 77031ee1ce4c
> 
> Thanks Kamil :)
> 
> Fam are you OK using this command instead?

Looks good, will use it! Thanks.

Fam

On Wed, 08/30 18:16, Fam Zheng wrote:
> On Tue, 08/29 10:35, Philippe Mathieu-Daudé wrote:
> > On 08/29/2017 10:22 AM, Kamil Rytarowski wrote:
> > > On 29.08.2017 15:10, Philippe Mathieu-Daudé wrote:
> > > > Hi Fam, Kamil,
> > > > 
> > > > > On 08/28/2017 02:47 PM, Fam Zheng wrote:
> > > > [...]
> > > > > > +        subprocess.check_call([tar_cmd,
> > > > > > +                               "--exclude-vcs",
> > > > > > +                               "--exclude=tests/vm/*.img",
> > > > > > +                               "--exclude=tests/vm/*.img.*",
> > > > > > +                               "--exclude=*.d",
> > > > > > +                               "--exclude=*.o",
> > > > > > +                               "--exclude=docker-src.*",
> > > > > > +                               "-cf", tarfile, '.'], cwd=data_dir,
> > > > > 
> > > > > I'm not happy with this command :/
> > > > > My distrib uses tmpfs for /tmp and suddently the whole X window became
> > > > > irresponsive until this script failing after filling 8G of /tmp and swap:
> > > > > 
> > > > > ...
> > > > > DEBUG:root:Creating archive /tmp/qemu-vm-F7CY9O/data-3a52c.tar for
> > > > > data dir: .
> > > > > tar: /tmp/qemu-vm-F7CY9O/data-3a52c.tar: Wrote only 4096 of 10240 bytes
> > > > > tar: Error is not recoverable: exiting now
> > > > > Failed to prepare guest environment
> > > > > 
> > > > > Then I figured out my workdir is full of testing stuff, debug images,
> > > > > firmwares, coredumps, etc.
> > > > > 
> > > > > I'll think of another way.
> > > > 
> > > > Using:
> > > > 
> > > > (git ls-files; git submodule foreach --recursive "git ls-files | sed
> > > > s_^_\$sm_path/_" | sed /^Entering/d) > files.txt
> > > > 
> > > > and:
> > > > 
> > > >          subprocess.check_call([tar_cmd,
> > > >                                 "-cf", tarfile,
> > > >                                 "-v" if self.debug else "",
> > > >                                 "-T", "files.txt"], cwd=data_dir,
> > > >                                stdin=self._devnull, stdout=self._stdout)
> > > > 
> > > > Current /master and submodules generated tarball is ~305MB uncompressed.
> > > > 
> > > > Kamil do you mind testing this command on NetBSD?
> > > > 
> > > > (obviously 'files.txt' goes in tmpdir).
> > > > 
> > > 
> > > Result:
> > > 
> > > http://www.netbsd.org/~kamil/qemu/files.txt
> > 
> > \o/
> > 
> > > I've tested this command line on a checkout from Jul 17th rev. 77031ee1ce4c
> > 
> > Thanks Kamil :)
> > 
> > Fam are you OK using this command instead?
> 
> Looks good, will use it! Thanks.

Hmm, there is a complication. "git ls-files" output includes submodule names,
which are directory names from the "tar -T" PoV, and the whole directory is
archived.

I'm going to stop fighting on this and refactor the docker tests's "git archive"
rules, to reuse in this script.

Fam

On 08/30/2017 12:34 AM, Fam Zheng wrote:
> On Tue, 08/29 14:34, Philippe Mathieu-Daudé wrote:
>>> +        self._args = [ \
>>> +            "-nodefaults", "-m", "2G",
>>> +            "-cpu", "host",
>>> +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
>>> +            "-device", "virtio-net-pci,netdev=vnet",
>>> +            "-vnc", ":0,to=20",
>>> +            "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
>>> +        if vcpus:
>>> +            self._args += ["-smp", str(vcpus)]
>>
>> What about enabling mttcg which isn't default?
>>
>>              self._args += ["--accel", "tcg,thread=multi"]
> 
> Any specific reason to enable it? I think it is not available on older QEMU.

Neither is dynamic portfwd :)

I see 2 reasons:
- faster test
- cover mttcg

> 
>>
>>> +        if os.access("/dev/kvm", os.R_OK | os.W_OK):
>>> +            self._args += ["-enable-kvm"]
>>> +        else:
>>> +            logging.info("KVM not available, not using -enable-kvm")
>>> +        self._data_args = []
>> [...]
> 
> Fam
>

On Fri, 09/01 16:29, Philippe Mathieu-Daudé wrote:
> On 08/30/2017 12:34 AM, Fam Zheng wrote:
> > On Tue, 08/29 14:34, Philippe Mathieu-Daudé wrote:
> > > > +        self._args = [ \
> > > > +            "-nodefaults", "-m", "2G",
> > > > +            "-cpu", "host",
> > > > +            "-netdev", "user,id=vnet,hostfwd=:0.0.0.0:0-:22",
> > > > +            "-device", "virtio-net-pci,netdev=vnet",
> > > > +            "-vnc", ":0,to=20",
> > > > +            "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
> > > > +        if vcpus:
> > > > +            self._args += ["-smp", str(vcpus)]
> > > 
> > > What about enabling mttcg which isn't default?
> > > 
> > >              self._args += ["--accel", "tcg,thread=multi"]
> > 
> > Any specific reason to enable it? I think it is not available on older QEMU.
> 
> Neither is dynamic portfwd :)

I figured, but portfwd is strongly justified, whereas ...

> 
> I see 2 reasons:
> - faster test

Any data? And if it is noticably faster, I doubt anyone is going to actually use
it, because it probably take a whole day to run one build.

> - cover mttcg

Testing mttcg is good, but we don't want to test mttcg and building at the same
time. Again, it can take a whole day.

But anyway --accel and any other options should be possible to get passed as
command line options.

Fam

> 
> > 
> > > 
> > > > +        if os.access("/dev/kvm", os.R_OK | os.W_OK):
> > > > +            self._args += ["-enable-kvm"]
> > > > +        else:
> > > > +            logging.info("KVM not available, not using -enable-kvm")
> > > > +        self._data_args = []
> > > [...]
> > 
> > Fam
> > 
>