[01/12] tests: Add utilities for docker testing

docker_run: A wrapper for "docker run" (or "sudo -n docker run" if
necessary), which takes care of killing and removing the running
container at SIGINT.

docker_clean: A tool to tear down all the containers including inactive
ones that are started by docker_run.

docker_build: A tool to compare an image from given dockerfile and
rebuild it if they're different.

Signed-off-by: Fam Zheng <famz@redhat.com>
---
 tests/docker/docker.py    | 108 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/docker/docker_build |  42 ++++++++++++++++++
 tests/docker/docker_clean |  22 ++++++++++
 tests/docker/docker_run   |  28 ++++++++++++
 4 files changed, 200 insertions(+)
 create mode 100755 tests/docker/docker.py
 create mode 100755 tests/docker/docker_build
 create mode 100755 tests/docker/docker_clean
 create mode 100755 tests/docker/docker_run

On 02/05/2016 04:24 AM, Fam Zheng wrote:
> docker_run: A wrapper for "docker run" (or "sudo -n docker run" if
> necessary), which takes care of killing and removing the running
> container at SIGINT.
> 
> docker_clean: A tool to tear down all the containers including inactive
> ones that are started by docker_run.
> 
> docker_build: A tool to compare an image from given dockerfile and
> rebuild it if they're different.
> 
> Signed-off-by: Fam Zheng <famz@redhat.com>
> ---
>  tests/docker/docker.py    | 108 ++++++++++++++++++++++++++++++++++++++++++++++
>  tests/docker/docker_build |  42 ++++++++++++++++++
>  tests/docker/docker_clean |  22 ++++++++++
>  tests/docker/docker_run   |  28 ++++++++++++
>  4 files changed, 200 insertions(+)
>  create mode 100755 tests/docker/docker.py
>  create mode 100755 tests/docker/docker_build
>  create mode 100755 tests/docker/docker_clean
>  create mode 100755 tests/docker/docker_run
> 
> diff --git a/tests/docker/docker.py b/tests/docker/docker.py
> new file mode 100755
> index 0000000..e513da0
> --- /dev/null
> +++ b/tests/docker/docker.py
> @@ -0,0 +1,108 @@
> +#!/usr/bin/env python2 -B
> +#
> +# Docker controlling module
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import os
> +import subprocess
> +import json
> +import hashlib
> +import atexit
> +import time
> +import uuid
> +
> +class ContainerTerminated(Exception):
> +    pass
> +
> +class Docker(object):
> +    def __init__(self):
> +        self._command = self._guess_command()
> +        self._instances = []
> +        atexit.register(self._kill_instances)
> +
> +    def _do(self, cmd, quiet=True, **kwargs):
> +        if quiet:
> +            kwargs["stdout"] = subprocess.PIPE
> +        return subprocess.call(self._command + cmd, **kwargs)
> +
> +    def _do_kill_instances(self, only_known, only_active=True):
> +        cmd = ["ps", "-q"]
> +        if not only_active:
> +            cmd.append("-a")
> +        for i in self._output(cmd).split():
> +            r = self._output(["inspect", i])
> +            labels = json.loads(r)[0]["Config"]["Labels"]
> +            active = json.loads(r)[0]["State"]["Running"]
> +            if not labels:
> +                continue
> +            u = labels.get("com.qemu.instance.uuid", None)
> +            if not u:
> +                continue
> +            if only_known and u not in self._instances:
> +                continue
> +            print "Terminating", i
> +            if active:
> +                self._do(["kill", i])
> +            self._do(["rm", i])
> +
> +    def clean(self):
> +        self._do_kill_instances(False, False)
> +        return 0
> +
> +    def _kill_instances(self):
> +        return self._do_kill_instances(True)
> +
> +    def _output(self, cmd, **kwargs):
> +        return subprocess.check_output(self._command + cmd,
> +                                       stderr=subprocess.STDOUT,
> +                                       **kwargs)
> +
> +    def _guess_command(self):
> +        for c in [["docker"], ["sudo", "-n", "docker"]]:

If the sudo version fails (Say, because a password prompt shows up) we
get the unhelpful error "Cannot find working docker command."

Does your sudo not prompt you in your dev environment?

> +            if subprocess.call(c + ["images"],
> +                               stdout=subprocess.PIPE,
> +                               stderr=subprocess.PIPE) == 0:
> +                return c
> +        raise Exception("Cannot find working docker command")
> +
> +    def get_image_dockerfile_checksum(self, tag):
> +        resp = self._output(["inspect", tag])
> +        t = json.loads(resp)[0]
> +        return t["Config"].get("Labels", {}).get("com.qemu.dockerfile-checksum", "")
> +
> +    def checksum(self, text):
> +        return hashlib.sha1(text).hexdigest()
> +
> +    def build_image(self, tag, dockerfile, df, quiet=True):
> +        tmp = dockerfile + "\n" + \
> +              "LABEL com.qemu.dockerfile-checksum=%s" % self.checksum(dockerfile)
> +        tmp_df = df + ".tmp"
> +        f = open(tmp_df, "wb")
> +        f.write(tmp)
> +        f.close()
> +        self._do(["build", "-t", tag, "-f", tmp_df, os.path.dirname(df)],
> +                 quiet=quiet)
> +        os.unlink(tmp_df)
> +
> +    def image_matches_dockerfile(self, tag, dockerfile):
> +        try:
> +            a = self.get_image_dockerfile_checksum(tag)
> +        except:
> +            return False
> +        return a == self.checksum(dockerfile)
> +
> +    def run(self, cmd, quiet, **kwargs):
> +        label = uuid.uuid1().hex
> +        self._instances.append(label)
> +        r = self._do(["run", "--label", "com.qemu.instance.uuid=" + label] + cmd, quiet=quiet)
> +        self._instances.remove(label)
> +        return r
> +
> diff --git a/tests/docker/docker_build b/tests/docker/docker_build
> new file mode 100755
> index 0000000..b4f0dec
> --- /dev/null
> +++ b/tests/docker/docker_build
> @@ -0,0 +1,42 @@
> +#!/usr/bin/env python2
> +#
> +# Compare to Dockerfile and rebuild a docker image if necessary.
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import sys
> +import docker
> +import argparse
> +
> +def main():
> +    parser = argparse.ArgumentParser()
> +    parser.add_argument("tag",
> +                        help="Image Tag")
> +    parser.add_argument("dockerfile",
> +                        help="Dockerfile name")
> +    parser.add_argument("--verbose", "-v", action="store_true",
> +                        help="Print verbose information")
> +    args = parser.parse_args()
> +
> +    dockerfile = open(args.dockerfile, "rb").read()
> +    tag = args.tag
> +
> +    d = docker.Docker()
> +    if d.image_matches_dockerfile(tag, dockerfile):
> +        if args.verbose:
> +            print "Image is up to date."
> +        return 0
> +
> +    quiet = not args.verbose
> +    d.build_image(tag, dockerfile, args.dockerfile, quiet=quiet)
> +    return 0
> +
> +if __name__ == "__main__":
> +    sys.exit(main())
> diff --git a/tests/docker/docker_clean b/tests/docker/docker_clean
> new file mode 100755
> index 0000000..88cdba6
> --- /dev/null
> +++ b/tests/docker/docker_clean
> @@ -0,0 +1,22 @@
> +#!/usr/bin/env python2
> +#
> +# Clean up uselsee containers.
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import sys
> +import docker
> +
> +def main():
> +    docker.Docker().clean()
> +    return 0
> +
> +if __name__ == "__main__":
> +    sys.exit(main())
> diff --git a/tests/docker/docker_run b/tests/docker/docker_run
> new file mode 100755
> index 0000000..5cf9d04
> --- /dev/null
> +++ b/tests/docker/docker_run
> @@ -0,0 +1,28 @@
> +#!/usr/bin/env python2
> +#
> +# Wrapper for "docker run" with automatical clean up if the execution is
> +# iterrupted.
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <famz@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import os
> +import sys
> +import argparse
> +import docker
> +
> +def main():
> +    parser = argparse.ArgumentParser()
> +    parser.add_argument("--quiet", action="store_true",
> +                        help="Run quietly unless an error occured")
> +    args, argv = parser.parse_known_args()
> +    return docker.Docker().run(argv, quiet=args.quiet)
> +
> +if __name__ == "__main__":
> +    sys.exit(main())
>

On Mon, 02/08 16:49, John Snow wrote:
> > +    def _guess_command(self):
> > +        for c in [["docker"], ["sudo", "-n", "docker"]]:
> 
> If the sudo version fails (Say, because a password prompt shows up) we
> get the unhelpful error "Cannot find working docker command."

If you have previously "sudo $something" and typed in the password, this will
work.

You can also specify passworless for docker only:

    fam ALL=(ALL) NOPASSWD: /usr/bin/docker

Fam

> 
> > +            if subprocess.call(c + ["images"],
> > +                               stdout=subprocess.PIPE,
> > +                               stderr=subprocess.PIPE) == 0:
> > +                return c
> > +        raise Exception("Cannot find working docker command")
> > +
> > +    def get_image_dockerfile_checksum(self, tag):
> > +        resp = self._output(["inspect", tag])
> > +        t = json.loads(resp)[0]
> > +        return t["Config"].get("Labels", {}).get("com.qemu.dockerfile-checksum", "")
> > +
> > +    def checksum(self, text):
> > +        return hashlib.sha1(text).hexdigest()
> > +
> > +    def build_image(self, tag, dockerfile, df, quiet=True):
> > +        tmp = dockerfile + "\n" + \
> > +              "LABEL com.qemu.dockerfile-checksum=%s" % self.checksum(dockerfile)
> > +        tmp_df = df + ".tmp"
> > +        f = open(tmp_df, "wb")
> > +        f.write(tmp)
> > +        f.close()
> > +        self._do(["build", "-t", tag, "-f", tmp_df, os.path.dirname(df)],
> > +                 quiet=quiet)
> > +        os.unlink(tmp_df)
> > +
> > +    def image_matches_dockerfile(self, tag, dockerfile):
> > +        try:
> > +            a = self.get_image_dockerfile_checksum(tag)
> > +        except:
> > +            return False
> > +        return a == self.checksum(dockerfile)
> > +
> > +    def run(self, cmd, quiet, **kwargs):
> > +        label = uuid.uuid1().hex
> > +        self._instances.append(label)
> > +        r = self._do(["run", "--label", "com.qemu.instance.uuid=" + label] + cmd, quiet=quiet)
> > +        self._instances.remove(label)
> > +        return r
> > +
> > diff --git a/tests/docker/docker_build b/tests/docker/docker_build
> > new file mode 100755
> > index 0000000..b4f0dec
> > --- /dev/null
> > +++ b/tests/docker/docker_build
> > @@ -0,0 +1,42 @@
> > +#!/usr/bin/env python2
> > +#
> > +# Compare to Dockerfile and rebuild a docker image if necessary.
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import sys
> > +import docker
> > +import argparse
> > +
> > +def main():
> > +    parser = argparse.ArgumentParser()
> > +    parser.add_argument("tag",
> > +                        help="Image Tag")
> > +    parser.add_argument("dockerfile",
> > +                        help="Dockerfile name")
> > +    parser.add_argument("--verbose", "-v", action="store_true",
> > +                        help="Print verbose information")
> > +    args = parser.parse_args()
> > +
> > +    dockerfile = open(args.dockerfile, "rb").read()
> > +    tag = args.tag
> > +
> > +    d = docker.Docker()
> > +    if d.image_matches_dockerfile(tag, dockerfile):
> > +        if args.verbose:
> > +            print "Image is up to date."
> > +        return 0
> > +
> > +    quiet = not args.verbose
> > +    d.build_image(tag, dockerfile, args.dockerfile, quiet=quiet)
> > +    return 0
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> > diff --git a/tests/docker/docker_clean b/tests/docker/docker_clean
> > new file mode 100755
> > index 0000000..88cdba6
> > --- /dev/null
> > +++ b/tests/docker/docker_clean
> > @@ -0,0 +1,22 @@
> > +#!/usr/bin/env python2
> > +#
> > +# Clean up uselsee containers.
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import sys
> > +import docker
> > +
> > +def main():
> > +    docker.Docker().clean()
> > +    return 0
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> > diff --git a/tests/docker/docker_run b/tests/docker/docker_run
> > new file mode 100755
> > index 0000000..5cf9d04
> > --- /dev/null
> > +++ b/tests/docker/docker_run
> > @@ -0,0 +1,28 @@
> > +#!/usr/bin/env python2
> > +#
> > +# Wrapper for "docker run" with automatical clean up if the execution is
> > +# iterrupted.
> > +#
> > +# Copyright (c) 2016 Red Hat Inc.
> > +#
> > +# Authors:
> > +#  Fam Zheng <famz@redhat.com>
> > +#
> > +# This work is licensed under the terms of the GNU GPL, version 2
> > +# or (at your option) any later version. See the COPYING file in
> > +# the top-level directory.
> > +
> > +import os
> > +import sys
> > +import argparse
> > +import docker
> > +
> > +def main():
> > +    parser = argparse.ArgumentParser()
> > +    parser.add_argument("--quiet", action="store_true",
> > +                        help="Run quietly unless an error occured")
> > +    args, argv = parser.parse_known_args()
> > +    return docker.Docker().run(argv, quiet=args.quiet)
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> > 
> 
> -- 
> —js

On 02/08/2016 09:01 PM, Fam Zheng wrote:
> On Mon, 02/08 16:49, John Snow wrote:
>>> +    def _guess_command(self):
>>> +        for c in [["docker"], ["sudo", "-n", "docker"]]:
>>
>> If the sudo version fails (Say, because a password prompt shows up) we
>> get the unhelpful error "Cannot find working docker command."
> 
> If you have previously "sudo $something" and typed in the password, this will
> work.
> 
> You can also specify passworless for docker only:
> 
>     fam ALL=(ALL) NOPASSWD: /usr/bin/docker
> 
> Fam
> 

Sure, but the failure here is not particularly obvious, because it makes
it seem as if docker has failed instead of a sudo privilege situation.

--js

>>
>>> +            if subprocess.call(c + ["images"],
>>> +                               stdout=subprocess.PIPE,
>>> +                               stderr=subprocess.PIPE) == 0:
>>> +                return c
>>> +        raise Exception("Cannot find working docker command")
>>> +
>>> +    def get_image_dockerfile_checksum(self, tag):
>>> +        resp = self._output(["inspect", tag])
>>> +        t = json.loads(resp)[0]
>>> +        return t["Config"].get("Labels", {}).get("com.qemu.dockerfile-checksum", "")
>>> +
>>> +    def checksum(self, text):
>>> +        return hashlib.sha1(text).hexdigest()
>>> +
>>> +    def build_image(self, tag, dockerfile, df, quiet=True):
>>> +        tmp = dockerfile + "\n" + \
>>> +              "LABEL com.qemu.dockerfile-checksum=%s" % self.checksum(dockerfile)
>>> +        tmp_df = df + ".tmp"
>>> +        f = open(tmp_df, "wb")
>>> +        f.write(tmp)
>>> +        f.close()
>>> +        self._do(["build", "-t", tag, "-f", tmp_df, os.path.dirname(df)],
>>> +                 quiet=quiet)
>>> +        os.unlink(tmp_df)
>>> +
>>> +    def image_matches_dockerfile(self, tag, dockerfile):
>>> +        try:
>>> +            a = self.get_image_dockerfile_checksum(tag)
>>> +        except:
>>> +            return False
>>> +        return a == self.checksum(dockerfile)
>>> +
>>> +    def run(self, cmd, quiet, **kwargs):
>>> +        label = uuid.uuid1().hex
>>> +        self._instances.append(label)
>>> +        r = self._do(["run", "--label", "com.qemu.instance.uuid=" + label] + cmd, quiet=quiet)
>>> +        self._instances.remove(label)
>>> +        return r
>>> +
>>> diff --git a/tests/docker/docker_build b/tests/docker/docker_build
>>> new file mode 100755
>>> index 0000000..b4f0dec
>>> --- /dev/null
>>> +++ b/tests/docker/docker_build
>>> @@ -0,0 +1,42 @@
>>> +#!/usr/bin/env python2
>>> +#
>>> +# Compare to Dockerfile and rebuild a docker image if necessary.
>>> +#
>>> +# Copyright (c) 2016 Red Hat Inc.
>>> +#
>>> +# Authors:
>>> +#  Fam Zheng <famz@redhat.com>
>>> +#
>>> +# This work is licensed under the terms of the GNU GPL, version 2
>>> +# or (at your option) any later version. See the COPYING file in
>>> +# the top-level directory.
>>> +
>>> +import sys
>>> +import docker
>>> +import argparse
>>> +
>>> +def main():
>>> +    parser = argparse.ArgumentParser()
>>> +    parser.add_argument("tag",
>>> +                        help="Image Tag")
>>> +    parser.add_argument("dockerfile",
>>> +                        help="Dockerfile name")
>>> +    parser.add_argument("--verbose", "-v", action="store_true",
>>> +                        help="Print verbose information")
>>> +    args = parser.parse_args()
>>> +
>>> +    dockerfile = open(args.dockerfile, "rb").read()
>>> +    tag = args.tag
>>> +
>>> +    d = docker.Docker()
>>> +    if d.image_matches_dockerfile(tag, dockerfile):
>>> +        if args.verbose:
>>> +            print "Image is up to date."
>>> +        return 0
>>> +
>>> +    quiet = not args.verbose
>>> +    d.build_image(tag, dockerfile, args.dockerfile, quiet=quiet)
>>> +    return 0
>>> +
>>> +if __name__ == "__main__":
>>> +    sys.exit(main())
>>> diff --git a/tests/docker/docker_clean b/tests/docker/docker_clean
>>> new file mode 100755
>>> index 0000000..88cdba6
>>> --- /dev/null
>>> +++ b/tests/docker/docker_clean
>>> @@ -0,0 +1,22 @@
>>> +#!/usr/bin/env python2
>>> +#
>>> +# Clean up uselsee containers.
>>> +#
>>> +# Copyright (c) 2016 Red Hat Inc.
>>> +#
>>> +# Authors:
>>> +#  Fam Zheng <famz@redhat.com>
>>> +#
>>> +# This work is licensed under the terms of the GNU GPL, version 2
>>> +# or (at your option) any later version. See the COPYING file in
>>> +# the top-level directory.
>>> +
>>> +import sys
>>> +import docker
>>> +
>>> +def main():
>>> +    docker.Docker().clean()
>>> +    return 0
>>> +
>>> +if __name__ == "__main__":
>>> +    sys.exit(main())
>>> diff --git a/tests/docker/docker_run b/tests/docker/docker_run
>>> new file mode 100755
>>> index 0000000..5cf9d04
>>> --- /dev/null
>>> +++ b/tests/docker/docker_run
>>> @@ -0,0 +1,28 @@
>>> +#!/usr/bin/env python2
>>> +#
>>> +# Wrapper for "docker run" with automatical clean up if the execution is
>>> +# iterrupted.
>>> +#
>>> +# Copyright (c) 2016 Red Hat Inc.
>>> +#
>>> +# Authors:
>>> +#  Fam Zheng <famz@redhat.com>
>>> +#
>>> +# This work is licensed under the terms of the GNU GPL, version 2
>>> +# or (at your option) any later version. See the COPYING file in
>>> +# the top-level directory.
>>> +
>>> +import os
>>> +import sys
>>> +import argparse
>>> +import docker
>>> +
>>> +def main():
>>> +    parser = argparse.ArgumentParser()
>>> +    parser.add_argument("--quiet", action="store_true",
>>> +                        help="Run quietly unless an error occured")
>>> +    args, argv = parser.parse_known_args()
>>> +    return docker.Docker().run(argv, quiet=args.quiet)
>>> +
>>> +if __name__ == "__main__":
>>> +    sys.exit(main())
>>>
>>
>> -- 
>> —js

On Tue, 02/09 18:16, John Snow wrote:
> Sure, but the failure here is not particularly obvious, because it makes
> it seem as if docker has failed instead of a sudo privilege situation.

Of course you're right, I'll improve the error message.  Thanks!

Fam