{"id":808043,"url":"http://patchwork.ozlabs.org/api/1.0/patches/808043/?format=json","project":{"id":14,"url":"http://patchwork.ozlabs.org/api/1.0/projects/14/?format=json","name":"QEMU Development","link_name":"qemu-devel","list_id":"qemu-devel.nongnu.org","list_email":"qemu-devel@nongnu.org","web_url":"","scm_url":"","webscm_url":""},"msgid":"<20170831064302.14427-5-famz@redhat.com>","date":"2017-08-31T06:42:54","name":"[v5,04/12] tests: Add vm test lib","commit_ref":null,"pull_url":null,"state":"new","archived":false,"hash":"d25e86d67b211a1bf98d48da0973392199d1a90a","submitter":{"id":24872,"url":"http://patchwork.ozlabs.org/api/1.0/people/24872/?format=json","name":"Fam Zheng","email":"famz@redhat.com"},"delegate":null,"mbox":"http://patchwork.ozlabs.org/project/qemu-devel/patch/20170831064302.14427-5-famz@redhat.com/mbox/","series":[{"id":753,"url":"http://patchwork.ozlabs.org/api/1.0/series/753/?format=json","date":"2017-08-31T06:42:50","name":"tests: Add VM based build tests (for non-x86_64 and/or non-Linux)","version":5,"mbox":"http://patchwork.ozlabs.org/series/753/mbox/"}],"check":"pending","checks":"http://patchwork.ozlabs.org/api/patches/808043/checks/","tags":{},"headers":{"Return-Path":"","X-Original-To":"incoming@patchwork.ozlabs.org","Delivered-To":"patchwork-incoming@bilbo.ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=pass (mailfrom) smtp.mailfrom=nongnu.org\n\t(client-ip=2001:4830:134:3::11; helo=lists.gnu.org;\n\tenvelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org;\n\treceiver=)","ext-mx02.extmail.prod.ext.phx2.redhat.com;\n\tdmarc=none (p=none dis=none) header.from=redhat.com","ext-mx02.extmail.prod.ext.phx2.redhat.com;\n\tspf=fail smtp.mailfrom=famz@redhat.com"],"Received":["from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11])\n\t(using TLSv1 with cipher AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 3xjXss2Jx8z9s7F\n\tfor ;\n\tThu, 31 Aug 2017 16:44:33 +1000 (AEST)","from localhost ([::1]:54157 helo=lists.gnu.org)\n\tby lists.gnu.org with esmtp (Exim 4.71) (envelope-from\n\t)\n\tid 1dnJDL-00073T-Ao\n\tfor incoming@patchwork.ozlabs.org; Thu, 31 Aug 2017 02:44:31 -0400","from eggs.gnu.org ([2001:4830:134:3::10]:51935)\n\tby lists.gnu.org with esmtp (Exim 4.71)\n\t(envelope-from ) id 1dnJCa-000718-RZ\n\tfor qemu-devel@nongnu.org; Thu, 31 Aug 2017 02:43:46 -0400","from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71)\n\t(envelope-from ) id 1dnJCZ-0006Z5-3x\n\tfor qemu-devel@nongnu.org; Thu, 31 Aug 2017 02:43:44 -0400","from mx1.redhat.com ([209.132.183.28]:43096)\n\tby eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32)\n\t(Exim 4.71) (envelope-from ) id 1dnJCY-0006Yj-R0\n\tfor qemu-devel@nongnu.org; Thu, 31 Aug 2017 02:43:43 -0400","from smtp.corp.redhat.com\n\t(int-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.12])\n\t(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby mx1.redhat.com (Postfix) with ESMTPS id B3C67883CA;\n\tThu, 31 Aug 2017 06:43:41 +0000 (UTC)","from lemon.redhat.com (ovpn-12-59.pek2.redhat.com [10.72.12.59])\n\tby smtp.corp.redhat.com (Postfix) with ESMTP id 250011810B;\n\tThu, 31 Aug 2017 06:43:36 +0000 (UTC)"],"DMARC-Filter":"OpenDMARC Filter v1.3.2 mx1.redhat.com B3C67883CA","From":"Fam Zheng ","To":"qemu-devel@nongnu.org","Date":"Thu, 31 Aug 2017 14:42:54 +0800","Message-Id":"<20170831064302.14427-5-famz@redhat.com>","In-Reply-To":"<20170831064302.14427-1-famz@redhat.com>","References":"<20170831064302.14427-1-famz@redhat.com>","X-Scanned-By":"MIMEDefang 2.79 on 10.5.11.12","X-Greylist":"Sender IP whitelisted, not delayed by milter-greylist-4.5.16\n\t(mx1.redhat.com [10.5.110.26]);\n\tThu, 31 Aug 2017 06:43:41 +0000 (UTC)","X-detected-operating-system":"by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic]\n\t[fuzzy]","X-Received-From":"209.132.183.28","Subject":"[Qemu-devel] [PATCH v5 04/12] tests: Add vm test lib","X-BeenThere":"qemu-devel@nongnu.org","X-Mailman-Version":"2.1.21","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"Peter Maydell , Fam Zheng ,\n\t=?utf-8?q?Philippe_Mathieu-Daud=C3=A9?= ,\n\tKamil Rytarowski , pbonzini@redhat.com, \n\tstefanha@redhat.com, Cleber Rosa ,\n\t=?utf-8?q?Alex_Benn=C3=A9e?= ","Errors-To":"qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org","Sender":"\"Qemu-devel\"\n\t"},"content":"This is the common code to implement a \"VM test\" to\n\n 1) Download and initialize a pre-defined VM that has necessary\n dependencies to build QEMU and SSH access.\n\n 2) Archive $SRC_PATH to a .tar file.\n\n 3) Boot the VM, and pass the source tar file to the guest.\n\n 4) SSH into the VM, untar the source tarball, build from the source.\n\nSigned-off-by: Fam Zheng \n---\n tests/vm/basevm.py | 276 +++++++++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 276 insertions(+)\n create mode 100755 tests/vm/basevm.py","diff":"diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py\nnew file mode 100755\nindex 0000000000..9db91d61fa\n--- /dev/null\n+++ b/tests/vm/basevm.py\n@@ -0,0 +1,276 @@\n+#!/usr/bin/env python\n+#\n+# VM testing base class\n+#\n+# Copyright (C) 2017 Red Hat Inc.\n+#\n+# Authors:\n+# Fam Zheng \n+#\n+# This work is licensed under the terms of the GNU GPL, version 2. See\n+# the COPYING file in the top-level directory.\n+#\n+\n+import os\n+import sys\n+import logging\n+import time\n+import datetime\n+sys.path.append(os.path.join(os.path.dirname(__file__), \"..\", \"..\", \"scripts\"))\n+from qemu import QEMUMachine\n+import subprocess\n+import hashlib\n+import optparse\n+import atexit\n+import tempfile\n+import shutil\n+import multiprocessing\n+import traceback\n+\n+SSH_KEY = \"\"\"\\\n+-----BEGIN RSA PRIVATE KEY-----\n+MIIEowIBAAKCAQEAopAuOlmLV6LVHdFBj8/eeOwI9CqguIJPp7eAQSZvOiB4Ag/R\n+coEhl/RBbrV5Yc/SmSD4PTpJO/iM10RwliNjDb4a3I8q3sykRJu9c9PI/YsH8WN9\n++NH2NjKPtJIcKTu287IM5JYxyB6nDoOzILbTyJ1TDR/xH6qYEfBAyiblggdjcvhA\n+RTf93QIn39F/xLypXvT1K2O9BJEsnJ8lEUvB2UXhKo/JTfSeZF8wPBeowaP9EONk\n+7b+nuJOWHGg68Ji6wVi62tjwl2Szch6lxIhZBpnV7QNRKMfYHP6eIyF4pusazzZq\n+Telsq6xI2ghecWLzb/MF5A+rklsGx2FNuJSAJwIDAQABAoIBAHHi4o/8VZNivz0x\n+cWXn8erzKV6tUoWQvW85Lj/2RiwJvSlsnYZDkx5af1CpEE2HA/pFT8PNRqsd+MWC\n+7AEy710cVsM4BYerBFYQaYxwzblaoojo88LSjVPw3h5Z0iLM8+IMVd36nwuc9dpE\n+R8TecMZ1+U4Tl6BgqkK+9xToZRdPKdjS8L5MoFhGN+xY0vRbbJbGaV9Q0IHxLBkB\n+rEBV7T1mUynneCHRUQlJQEwJmKpT8MH3IjsUXlG5YvnuuvcQJSNTaW2iDLxuOKp8\n+cxW8+qL88zpb1D5dppoIu6rlrugN0azSq70ruFJQPc/A8GQrDKoGgRQiagxNY3u+\n+vHZzXlECgYEA0dKO3gfkSxsDBb94sQwskMScqLhcKhztEa8kPxTx6Yqh+x8/scx3\n+XhJyOt669P8U1v8a/2Al+s81oZzzfQSzO1Q7gEwSrgBcRMSIoRBUw9uYcy02ngb/\n+j/ng3DGivfJztjjiSJwb46FHkJ2JR8mF2UisC6UMXk3NgFY/3vWQx78CgYEAxlcG\n+T3hfSWSmTgKRczMJuHQOX9ULfTBIqwP5VqkkkiavzigGRirzb5lgnmuTSPTpF0LB\n+XVPjR2M4q+7gzP0Dca3pocrvLEoxjwIKnCbYKnyyvnUoE9qHv4Kr+vDbgWpa2LXG\n+JbLmE7tgTCIp20jOPPT4xuDvlbzQZBJ5qCQSoZkCgYEAgrotSSihlCnAOFSTXbu4\n+CHp3IKe8xIBBNENq0eK61kcJpOxTQvOha3sSsJsU4JAM6+cFaxb8kseHIqonCj1j\n+bhOM/uJmwQJ4el/4wGDsbxriYOBKpyq1D38gGhDS1IW6kk3erl6VAb36WJ/OaGum\n+eTpN9vNeQWM4Jj2WjdNx4QECgYAwTdd6mU1TmZCrJRL5ZG+0nYc2rbMrnQvFoqUi\n+BvWiJovggHzur90zy73tNzPaq9Ls2FQxf5G1vCN8NCRJqEEjeYCR59OSDMu/EXc2\n+CnvQ9SevHOdS1oEDEjcCWZCMFzPi3XpRih1gptzQDe31uuiHjf3cqcGPzTlPdfRt\n+D8P92QKBgC4UaBvIRwREVJsdZzpIzm224Bpe8LOmA7DeTnjlT0b3lkGiBJ36/Q0p\n+VhYh/6cjX4/iuIs7gJbGon7B+YPB8scmOi3fj0+nkJAONue1mMfBNkba6qQTc6Y2\n+5mEKw2/O7/JpND7ucU3OK9plcw/qnrWDgHxl0Iz95+OzUIIagxne\n+-----END RSA PRIVATE KEY-----\n+\"\"\"\n+SSH_PUB_KEY = \"\"\"\\\n+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCikC46WYtXotUd0UGPz9547Aj0KqC4gk+nt4BBJm86IHgCD9FygSGX9EFutXlhz9KZIPg9Okk7+IzXRHCWI2MNvhrcjyrezKREm71z08j9iwfxY3340fY2Mo+0khwpO7bzsgzkljHIHqcOg7MgttPInVMNH/EfqpgR8EDKJuWCB2Ny+EBFN/3dAiff0X/EvKle9PUrY70EkSycnyURS8HZReEqj8lN9J5kXzA8F6jBo/0Q42Ttv6e4k5YcaDrwmLrBWLra2PCXZLNyHqXEiFkGmdXtA1Eox9gc/p4jIXim6xrPNmpN6WyrrEjaCF5xYvNv8wXkD6uSWwbHYU24lIAn qemu-vm-key\n+\"\"\"\n+\n+class BaseVM(object):\n+ GUEST_USER = \"qemu\"\n+ GUEST_PASS = \"qemupass\"\n+ ROOT_PASS = \"qemupass\"\n+\n+ # The script to run in the guest that builds QEMU\n+ BUILD_SCRIPT = \"\"\n+ # The guest name, to be overridden by subclasses\n+ name = \"#base\"\n+ def __init__(self, debug=False, vcpus=None):\n+ self._guest = None\n+ self._tmpdir = tempfile.mkdtemp(prefix=\"vm-test-\", suffix=\".tmp\", dir=\".\")\n+ atexit.register(shutil.rmtree, self._tmpdir)\n+\n+ self._ssh_key_file = os.path.join(self._tmpdir, \"id_rsa\")\n+ open(self._ssh_key_file, \"w\").write(SSH_KEY)\n+ subprocess.check_call([\"chmod\", \"600\", self._ssh_key_file])\n+\n+ self._ssh_pub_key_file = os.path.join(self._tmpdir, \"id_rsa.pub\")\n+ open(self._ssh_pub_key_file, \"w\").write(SSH_PUB_KEY)\n+\n+ self.debug = debug\n+ self._stderr = sys.stderr\n+ self._devnull = open(\"/dev/null\", \"w\")\n+ if self.debug:\n+ self._stdout = sys.stdout\n+ else:\n+ self._stdout = self._devnull\n+ self._args = [ \\\n+ \"-nodefaults\", \"-m\", \"2G\",\n+ \"-cpu\", \"host\",\n+ \"-netdev\", \"user,id=vnet,hostfwd=:0.0.0.0:0-:22\",\n+ \"-device\", \"virtio-net-pci,netdev=vnet\",\n+ \"-vnc\", \":0,to=20\",\n+ \"-serial\", \"file:%s\" % os.path.join(self._tmpdir, \"serial.out\")]\n+ if vcpus:\n+ self._args += [\"-smp\", str(vcpus)]\n+ if os.access(\"/dev/kvm\", os.R_OK | os.W_OK):\n+ self._args += [\"-enable-kvm\"]\n+ else:\n+ logging.info(\"KVM not available, not using -enable-kvm\")\n+ self._data_args = []\n+\n+ def _download_with_cache(self, url, sha256sum=None):\n+ def check_sha256sum(fname):\n+ if not sha256sum:\n+ return True\n+ checksum = subprocess.check_output([\"sha256sum\", fname]).split()[0]\n+ return sha256sum == checksum\n+\n+ cache_dir = os.path.expanduser(\"~/.cache/qemu-vm/download\")\n+ if not os.path.exists(cache_dir):\n+ os.makedirs(cache_dir)\n+ fname = os.path.join(cache_dir, hashlib.sha1(url).hexdigest())\n+ if os.path.exists(fname) and check_sha256sum(fname):\n+ return fname\n+ logging.debug(\"Downloading %s to %s...\", url, fname)\n+ subprocess.check_call([\"wget\", \"-c\", url, \"-O\", fname + \".download\"],\n+ stdout=self._stdout, stderr=self._stderr)\n+ os.rename(fname + \".download\", fname)\n+ return fname\n+\n+ def _ssh_do(self, user, cmd, check, interactive=False):\n+ ssh_cmd = [\"ssh\", \"-q\",\n+ \"-o\", \"StrictHostKeyChecking=no\",\n+ \"-o\", \"UserKnownHostsFile=/dev/null\",\n+ \"-o\", \"ConnectTimeout=1\",\n+ \"-p\", self.ssh_port, \"-i\", self._ssh_key_file]\n+ if interactive:\n+ ssh_cmd += ['-t']\n+ assert not isinstance(cmd, str)\n+ ssh_cmd += [\"%s@127.0.0.1\" % user] + list(cmd)\n+ logging.debug(\"ssh_cmd: %s\", \" \".join(ssh_cmd))\n+ r = subprocess.call(ssh_cmd,\n+ stdin=sys.stdin if interactive else self._devnull,\n+ stdout=sys.stdout if interactive else self._stdout,\n+ stderr=sys.stderr if interactive else self._stderr)\n+ if check and r != 0:\n+ raise Exception(\"SSH command failed: %s\" % cmd)\n+ return r\n+\n+ def ssh(self, *cmd):\n+ return self._ssh_do(self.GUEST_USER, cmd, False)\n+\n+ def ssh_interactive(self, *cmd):\n+ return self._ssh_do(self.GUEST_USER, cmd, False, True)\n+\n+ def ssh_root(self, *cmd):\n+ return self._ssh_do(\"root\", cmd, False)\n+\n+ def ssh_check(self, *cmd):\n+ self._ssh_do(self.GUEST_USER, cmd, True)\n+\n+ def ssh_root_check(self, *cmd):\n+ self._ssh_do(\"root\", cmd, True)\n+\n+ def build_image(self, img):\n+ raise NotImplementedError\n+\n+ def add_source_dir(self, src_dir):\n+ name = \"data-\" + hashlib.sha1(src_dir).hexdigest()[:5]\n+ tarfile = os.path.join(self._tmpdir, name + \".tar\")\n+ logging.debug(\"Creating archive %s for src_dir dir: %s\", tarfile, src_dir)\n+ subprocess.check_call([\"./scripts/archive-source.sh\", tarfile],\n+ cwd=src_dir, stdin=self._devnull,\n+ stdout=self._stdout, stderr=self._stderr)\n+ self._data_args += [\"-drive\",\n+ \"file=%s,if=none,id=%s,cache=writeback,format=raw\" % \\\n+ (tarfile, name),\n+ \"-device\",\n+ \"virtio-blk,drive=%s,serial=%s,bootindex=1\" % (name, name)]\n+\n+ def boot(self, img, extra_args=[]):\n+ args = self._args + [\n+ \"-device\", \"VGA\",\n+ \"-drive\", \"file=%s,if=none,id=drive0,cache=writeback\" % img,\n+ \"-device\", \"virtio-blk,drive=drive0,bootindex=0\"]\n+ args += self._data_args + extra_args\n+ logging.debug(\"QEMU args: %s\", \" \".join(args))\n+ guest = QEMUMachine(binary=os.environ.get(\"QEMU\", \"qemu-system-x86_64\"),\n+ args=args)\n+ guest.launch()\n+ atexit.register(self.shutdown)\n+ self._guest = guest\n+ usernet_info = guest.qmp(\"human-monitor-command\",\n+ command_line=\"info usernet\")\n+ self.ssh_port = None\n+ for l in usernet_info[\"return\"].splitlines():\n+ fields = l.split()\n+ if \"TCP[HOST_FORWARD]\" in fields and \"22\" in fields:\n+ self.ssh_port = l.split()[3]\n+ if not self.ssh_port:\n+ raise Exception(\"Cannot find ssh port from 'info usernet':\\n%s\" % \\\n+ usernet_info)\n+\n+ def wait_ssh(self, seconds=120):\n+ starttime = datetime.datetime.now()\n+ guest_up = False\n+ while (datetime.datetime.now() - starttime).total_seconds() < seconds:\n+ if self.ssh(\"exit 0\") == 0:\n+ guest_up = True\n+ break\n+ time.sleep(1)\n+ if not guest_up:\n+ raise Exception(\"Timeout while waiting for guest ssh\")\n+\n+ def shutdown(self):\n+ self._guest.shutdown()\n+\n+ def wait(self):\n+ self._guest.wait()\n+\n+ def qmp(self, *args, **kwargs):\n+ return self._guest.qmp(*args, **kwargs)\n+\n+def parse_args(vm_name):\n+ parser = optparse.OptionParser(description=\"\"\"\n+ VM test utility. Exit codes: 0 = success, 1 = command line error, 2 = environment initialization failed, 3 = test command failed\"\"\")\n+ parser.add_option(\"--debug\", \"-D\", action=\"store_true\",\n+ help=\"enable debug output\")\n+ parser.add_option(\"--image\", \"-i\", default=\"%s.img\" % vm_name,\n+ help=\"image file name\")\n+ parser.add_option(\"--force\", \"-f\", action=\"store_true\",\n+ help=\"force build image even if image exists\")\n+ parser.add_option(\"--jobs\", type=int, default=multiprocessing.cpu_count() / 2,\n+ help=\"number of virtual CPUs\")\n+ parser.add_option(\"--build-image\", \"-b\", action=\"store_true\",\n+ help=\"build image\")\n+ parser.add_option(\"--build-qemu\",\n+ help=\"build QEMU from source in guest\")\n+ parser.add_option(\"--interactive\", \"-I\", action=\"store_true\",\n+ help=\"Interactively run command\")\n+ parser.disable_interspersed_args()\n+ return parser.parse_args()\n+\n+def main(vmcls):\n+ try:\n+ args, argv = parse_args(vmcls.name)\n+ if not argv and not args.build_qemu and not args.build_image:\n+ print \"Nothing to do?\"\n+ return 1\n+ if args.debug:\n+ logging.getLogger().setLevel(logging.DEBUG)\n+ vm = vmcls(debug=args.debug, vcpus=args.jobs)\n+ if args.build_image:\n+ if os.path.exists(args.image) and not args.force:\n+ sys.stderr.writelines([\"Image file exists: %s\\n\" % args.image,\n+ \"Use --force option to overwrite\\n\"])\n+ return 1\n+ return vm.build_image(args.image)\n+ if args.build_qemu:\n+ vm.add_source_dir(args.build_qemu)\n+ cmd = [vm.BUILD_SCRIPT.format(\n+ configure_opts = \" \".join(argv),\n+ jobs=args.jobs)]\n+ else:\n+ cmd = argv\n+ vm.boot(args.image + \",snapshot=on\")\n+ vm.wait_ssh()\n+ except Exception as e:\n+ if isinstance(e, SystemExit) and e.code == 0:\n+ return 0\n+ sys.stderr.write(\"Failed to prepare guest environment\\n\")\n+ traceback.print_exc()\n+ return 2\n+\n+ if args.interactive:\n+ if vm.ssh_interactive(*cmd) == 0:\n+ return 0\n+ vm.ssh_interactive()\n+ return 3\n+ else:\n+ if vm.ssh(*cmd) != 0:\n+ return 3\n","prefixes":["v5","04/12"]}