[v2,01/15] tests: Add utilities for docker testing

docker_run: A wrapper for "docker run" (or "sudo -n docker run" if
necessary), which takes care of killing and removing the running
container at SIGINT.

docker_clean: A tool to tear down all the containers including inactive
ones that are started by docker_run.

docker_build: A tool to compare an image from given dockerfile and
rebuild it if they're different.

Signed-off-by: Fam Zheng <famz@redhat.com>
---
 tests/docker/docker.py    | 113 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/docker/docker_build |  42 +++++++++++++++++
 tests/docker/docker_clean |  22 +++++++++
 tests/docker/docker_run   |  29 ++++++++++++
 4 files changed, 206 insertions(+)
 create mode 100755 tests/docker/docker.py
 create mode 100755 tests/docker/docker_build
 create mode 100755 tests/docker/docker_clean
 create mode 100755 tests/docker/docker_run

Fam Zheng <famz@redhat.com> writes:

> docker_run: A wrapper for "docker run" (or "sudo -n docker run" if
> necessary), which takes care of killing and removing the running
> container at SIGINT.
>
> docker_clean: A tool to tear down all the containers including inactive
> ones that are started by docker_run.
>
> docker_build: A tool to compare an image from given dockerfile and
> rebuild it if they're different.
>
> Signed-off-by: Fam Zheng <famz@redhat.com>
> ---
>  tests/docker/docker.py    | 113 ++++++++++++++++++++++++++++++++++++++++++++++
>  tests/docker/docker_build |  42 +++++++++++++++++
>  tests/docker/docker_clean |  22 +++++++++
>  tests/docker/docker_run   |  29 ++++++++++++
>  4 files changed, 206 insertions(+)
>  create mode 100755 tests/docker/docker.py
>  create mode 100755 tests/docker/docker_build
>  create mode 100755 tests/docker/docker_clean
>  create mode 100755 tests/docker/docker_run
>
> diff --git a/tests/docker/docker.py b/tests/docker/docker.py
> new file mode 100755
> index 0000000..d175a86
> --- /dev/null
> +++ b/tests/docker/docker.py
> @@ -0,0 +1,113 @@
> +#!/usr/bin/env python2 -B
> +#
> +# Docker controlling module
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import os
> +import subprocess
> +import json
> +import hashlib
> +import atexit
> +import uuid
> +
> +class ContainerTerminated(Exception):
> +    """ Raised if the container has already existed """
> +    pass
> +
> +class Docker(object):
> +    """ Running Docker commands """
> +    def __init__(self):
> +        self._command = self._guess_command()
> +        self._instances = []
> +        atexit.register(self._kill_instances)
> +
> +    def _do(self, cmd, quiet=True, **kwargs):
> +        if quiet:
> +            kwargs["stdout"] = subprocess.PIPE
> +        return subprocess.call(self._command + cmd, **kwargs)
> +
> +    def _do_kill_instances(self, only_known, only_active=True):
> +        cmd = ["ps", "-q"]

Hmm ps -q barfs on my command line:

16:04 alex@zen/x86_64  [qemu.git/mttcg/multi_tcg_v8_ajb-r2] >ps -q
error: unsupported SysV option

Is there not a more portable way of doing this, even if it is a standard
library?

> +        if not only_active:
> +            cmd.append("-a")
> +        for i in self._output(cmd).split():
> +            resp = self._output(["inspect", i])
> +            labels = json.loads(resp)[0]["Config"]["Labels"]
> +            active = json.loads(resp)[0]["State"]["Running"]
> +            if not labels:
> +                continue
> +            instance_uuid = labels.get("com.qemu.instance.uuid", None)
> +            if not instance_uuid:
> +                continue
> +            if only_known and instance_uuid not in self._instances:
> +                continue
> +            print "Terminating", i
> +            if active:
> +                self._do(["kill", i])
> +            self._do(["rm", i])
> +
> +    def clean(self):
> +        self._do_kill_instances(False, False)
> +        return 0
> +
> +    def _kill_instances(self):
> +        return self._do_kill_instances(True)
> +
> +    def _output(self, cmd, **kwargs):
> +        return subprocess.check_output(self._command + cmd,
> +                                       stderr=subprocess.STDOUT,
> +                                       **kwargs)
> +
> +    def _guess_command(self):
> +        commands = [["docker"], ["sudo", "-n", "docker"]]
> +        for cmd in commands:
> +            if subprocess.call(cmd + ["images"],
> +                               stdout=subprocess.PIPE,
> +                               stderr=subprocess.PIPE) == 0:
> +                return cmd
> +        commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
> +        raise Exception("Cannot find working docker command. Tried:\n%s" % commands_txt)
> +
> +    def get_image_dockerfile_checksum(self, tag):
> +        resp = self._output(["inspect", tag])
> +        labels = json.loads(resp)[0]["Config"].get("Labels", {})
> +        return labels.get("com.qemu.dockerfile-checksum", "")
> +
> +    def checksum(self, text):
> +        return hashlib.sha1(text).hexdigest()
> +
> +    def build_image(self, tag, dockerfile, df, quiet=True):
> +        tmp = dockerfile + "\n" + \
> +              "LABEL com.qemu.dockerfile-checksum=%s" % self.checksum(dockerfile)
> +        tmp_df = df + ".tmp"
> +        tmp_file = open(tmp_df, "wb")
> +        tmp_file.write(tmp)
> +        tmp_file.close()
> +        self._do(["build", "-t", tag, "-f", tmp_df, os.path.dirname(df)],
> +                 quiet=quiet)
> +        os.unlink(tmp_df)
> +
> +    def image_matches_dockerfile(self, tag, dockerfile):
> +        try:
> +            checksum = self.get_image_dockerfile_checksum(tag)
> +        except:
> +            return False
> +        return checksum == self.checksum(dockerfile)
> +
> +    def run(self, cmd, keep, quiet):
> +        label = uuid.uuid1().hex
> +        if not keep:
> +            self._instances.append(label)
> +        ret = self._do(["run", "--label", "com.qemu.instance.uuid=" + label] + cmd, quiet=quiet)
> +        if not keep:
> +            self._instances.remove(label)
> +        return ret

I think it might be useful to catch some arguments here for testing
things. It is likely to be the first script someone runs while poking
around so some help text would be useful even if it just points at the
other commands.

In fact I'm not sure why all the various commands aren't in one script
for now given this does most of the heavy lifting.

> +
> diff --git a/tests/docker/docker_build b/tests/docker/docker_build
> new file mode 100755
> index 0000000..6948e2c
> --- /dev/null
> +++ b/tests/docker/docker_build
> @@ -0,0 +1,42 @@
> +#!/usr/bin/env python2
> +#
> +# Compare to Dockerfile and rebuild a docker image if necessary.
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import sys
> +import docker
> +import argparse
> +
> +def main():
> +    parser = argparse.ArgumentParser()
> +    parser.add_argument("tag",
> +                        help="Image Tag")
> +    parser.add_argument("dockerfile",
> +                        help="Dockerfile name")
> +    parser.add_argument("--verbose", "-v", action="store_true",
> +                        help="Print verbose information")
> +    args = parser.parse_args()
> +
> +    dockerfile = open(args.dockerfile, "rb").read()
> +    tag = args.tag
> +
> +    dkr = docker.Docker()
> +    if dkr.image_matches_dockerfile(tag, dockerfile):
> +        if args.verbose:
> +            print "Image is up to date."
> +        return 0
> +
> +    quiet = not args.verbose
> +    dkr.build_image(tag, dockerfile, args.dockerfile, quiet=quiet)
> +    return 0
> +
> +if __name__ == "__main__":
> +    sys.exit(main())
> diff --git a/tests/docker/docker_clean b/tests/docker/docker_clean
> new file mode 100755
> index 0000000..88cdba6
> --- /dev/null
> +++ b/tests/docker/docker_clean
> @@ -0,0 +1,22 @@
> +#!/usr/bin/env python2
> +#
> +# Clean up uselsee containers.
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import sys
> +import docker
> +
> +def main():
> +    docker.Docker().clean()
> +    return 0
> +
> +if __name__ == "__main__":
> +    sys.exit(main())

Of all the scripts run if you call with --help this just does something
straight away. It should at least attempt a usage() text to prevent
accidents.

> diff --git a/tests/docker/docker_run b/tests/docker/docker_run
> new file mode 100755
> index 0000000..4c46d90
> --- /dev/null
> +++ b/tests/docker/docker_run
> @@ -0,0 +1,29 @@
> +#!/usr/bin/env python2
> +#
> +# Wrapper for "docker run" with automatical clean up if the execution is
> +# iterrupted.
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import sys
> +import argparse
> +import docker
> +
> +def main():
> +    parser = argparse.ArgumentParser()
> +    parser.add_argument("--keep", action="store_true",
> +                        help="Don't remove image when the command completes")
> +    parser.add_argument("--quiet", action="store_true",
> +                        help="Run quietly unless an error occured")
> +    args, argv = parser.parse_known_args()
> +    return docker.Docker().run(argv, args.keep, quiet=args.quiet)
> +
> +if __name__ == "__main__":
> +    sys.exit(main())


--
Alex Bennée

On Mon, 02/29 16:46, Alex Bennée wrote:
> 
> Fam Zheng <famz@redhat.com> writes:
> 
> > docker_run: A wrapper for "docker run" (or "sudo -n docker run" if
> > necessary), which takes care of killing and removing the running
> > container at SIGINT.
> >
> > docker_clean: A tool to tear down all the containers including inactive
> > ones that are started by docker_run.
> >
> > docker_build: A tool to compare an image from given dockerfile and
> > rebuild it if they're different.
> >
> > Signed-off-by: Fam Zheng <famz@redhat.com>
> > ---
> >  tests/docker/docker.py    | 113 ++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/docker/docker_build |  42 +++++++++++++++++
> >  tests/docker/docker_clean |  22 +++++++++
> >  tests/docker/docker_run   |  29 ++++++++++++
> >  4 files changed, 206 insertions(+)
> >  create mode 100755 tests/docker/docker.py
> >  create mode 100755 tests/docker/docker_build
> >  create mode 100755 tests/docker/docker_clean
> >  create mode 100755 tests/docker/docker_run
> >
> > diff --git a/tests/docker/docker.py b/tests/docker/docker.py
> > new file mode 100755
> > index 0000000..d175a86
> > --- /dev/null
> > +++ b/tests/docker/docker.py
> > @@ -0,0 +1,113 @@
> > +#!/usr/bin/env python2 -B
> > +#
> > +# Docker controlling module
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import os
> > +import subprocess
> > +import json
> > +import hashlib
> > +import atexit
> > +import uuid
> > +
> > +class ContainerTerminated(Exception):
> > +    """ Raised if the container has already existed """
> > +    pass
> > +
> > +class Docker(object):
> > +    """ Running Docker commands """
> > +    def __init__(self):
> > +        self._command = self._guess_command()
> > +        self._instances = []
> > +        atexit.register(self._kill_instances)
> > +
> > +    def _do(self, cmd, quiet=True, **kwargs):
> > +        if quiet:
> > +            kwargs["stdout"] = subprocess.PIPE
> > +        return subprocess.call(self._command + cmd, **kwargs)
> > +
> > +    def _do_kill_instances(self, only_known, only_active=True):
> > +        cmd = ["ps", "-q"]
> 
> Hmm ps -q barfs on my command line:
> 
> 16:04 alex@zen/x86_64  [qemu.git/mttcg/multi_tcg_v8_ajb-r2] >ps -q
> error: unsupported SysV option
> 
> Is there not a more portable way of doing this, even if it is a standard
> library?

Down the road this is "sudo docker ps" command. :)

> 
> > +        if not only_active:
> > +            cmd.append("-a")
> > +        for i in self._output(cmd).split():
> > +            resp = self._output(["inspect", i])
> > +            labels = json.loads(resp)[0]["Config"]["Labels"]
> > +            active = json.loads(resp)[0]["State"]["Running"]
> > +            if not labels:
> > +                continue
> > +            instance_uuid = labels.get("com.qemu.instance.uuid", None)
> > +            if not instance_uuid:
> > +                continue
> > +            if only_known and instance_uuid not in self._instances:
> > +                continue
> > +            print "Terminating", i
> > +            if active:
> > +                self._do(["kill", i])
> > +            self._do(["rm", i])
> > +
> > +    def clean(self):
> > +        self._do_kill_instances(False, False)
> > +        return 0
> > +
> > +    def _kill_instances(self):
> > +        return self._do_kill_instances(True)
> > +
> > +    def _output(self, cmd, **kwargs):
> > +        return subprocess.check_output(self._command + cmd,
> > +                                       stderr=subprocess.STDOUT,
> > +                                       **kwargs)
> > +
> > +    def _guess_command(self):
> > +        commands = [["docker"], ["sudo", "-n", "docker"]]
> > +        for cmd in commands:
> > +            if subprocess.call(cmd + ["images"],
> > +                               stdout=subprocess.PIPE,
> > +                               stderr=subprocess.PIPE) == 0:
> > +                return cmd
> > +        commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
> > +        raise Exception("Cannot find working docker command. Tried:\n%s" % commands_txt)
> > +
> > +    def get_image_dockerfile_checksum(self, tag):
> > +        resp = self._output(["inspect", tag])
> > +        labels = json.loads(resp)[0]["Config"].get("Labels", {})
> > +        return labels.get("com.qemu.dockerfile-checksum", "")
> > +
> > +    def checksum(self, text):
> > +        return hashlib.sha1(text).hexdigest()
> > +
> > +    def build_image(self, tag, dockerfile, df, quiet=True):
> > +        tmp = dockerfile + "\n" + \
> > +              "LABEL com.qemu.dockerfile-checksum=%s" % self.checksum(dockerfile)
> > +        tmp_df = df + ".tmp"
> > +        tmp_file = open(tmp_df, "wb")
> > +        tmp_file.write(tmp)
> > +        tmp_file.close()
> > +        self._do(["build", "-t", tag, "-f", tmp_df, os.path.dirname(df)],
> > +                 quiet=quiet)
> > +        os.unlink(tmp_df)
> > +
> > +    def image_matches_dockerfile(self, tag, dockerfile):
> > +        try:
> > +            checksum = self.get_image_dockerfile_checksum(tag)
> > +        except:
> > +            return False
> > +        return checksum == self.checksum(dockerfile)
> > +
> > +    def run(self, cmd, keep, quiet):
> > +        label = uuid.uuid1().hex
> > +        if not keep:
> > +            self._instances.append(label)
> > +        ret = self._do(["run", "--label", "com.qemu.instance.uuid=" + label] + cmd, quiet=quiet)
> > +        if not keep:
> > +            self._instances.remove(label)
> > +        return ret
> 
> I think it might be useful to catch some arguments here for testing
> things. It is likely to be the first script someone runs while poking
> around so some help text would be useful even if it just points at the
> other commands.

Sure, I can do that.

> 
> In fact I'm not sure why all the various commands aren't in one script
> for now given this does most of the heavy lifting.

OK, I can merge them into one script.


> 
> > +
> > diff --git a/tests/docker/docker_build b/tests/docker/docker_build
> > new file mode 100755
> > index 0000000..6948e2c
> > --- /dev/null
> > +++ b/tests/docker/docker_build
> > @@ -0,0 +1,42 @@
> > +#!/usr/bin/env python2
> > +#
> > +# Compare to Dockerfile and rebuild a docker image if necessary.
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import sys
> > +import docker
> > +import argparse
> > +
> > +def main():
> > +    parser = argparse.ArgumentParser()
> > +    parser.add_argument("tag",
> > +                        help="Image Tag")
> > +    parser.add_argument("dockerfile",
> > +                        help="Dockerfile name")
> > +    parser.add_argument("--verbose", "-v", action="store_true",
> > +                        help="Print verbose information")
> > +    args = parser.parse_args()
> > +
> > +    dockerfile = open(args.dockerfile, "rb").read()
> > +    tag = args.tag
> > +
> > +    dkr = docker.Docker()
> > +    if dkr.image_matches_dockerfile(tag, dockerfile):
> > +        if args.verbose:
> > +            print "Image is up to date."
> > +        return 0
> > +
> > +    quiet = not args.verbose
> > +    dkr.build_image(tag, dockerfile, args.dockerfile, quiet=quiet)
> > +    return 0
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> > diff --git a/tests/docker/docker_clean b/tests/docker/docker_clean
> > new file mode 100755
> > index 0000000..88cdba6
> > --- /dev/null
> > +++ b/tests/docker/docker_clean
> > @@ -0,0 +1,22 @@
> > +#!/usr/bin/env python2
> > +#
> > +# Clean up uselsee containers.
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import sys
> > +import docker
> > +
> > +def main():
> > +    docker.Docker().clean()
> > +    return 0
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> 
> Of all the scripts run if you call with --help this just does something
> straight away. It should at least attempt a usage() text to prevent
> accidents.

Yes. Will address.

Fam

> 
> > diff --git a/tests/docker/docker_run b/tests/docker/docker_run
> > new file mode 100755
> > index 0000000..4c46d90
> > --- /dev/null
> > +++ b/tests/docker/docker_run
> > @@ -0,0 +1,29 @@
> > +#!/usr/bin/env python2
> > +#
> > +# Wrapper for "docker run" with automatical clean up if the execution is
> > +# iterrupted.
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import sys
> > +import argparse
> > +import docker
> > +
> > +def main():
> > +    parser = argparse.ArgumentParser()
> > +    parser.add_argument("--keep", action="store_true",
> > +                        help="Don't remove image when the command completes")
> > +    parser.add_argument("--quiet", action="store_true",
> > +                        help="Run quietly unless an error occured")
> > +    args, argv = parser.parse_known_args()
> > +    return docker.Docker().run(argv, args.keep, quiet=args.quiet)
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> 
> 
> --
> Alex Bennée