From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:38911) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1aaYrr-0004KD-K3 for qemu-devel@nongnu.org; Mon, 29 Feb 2016 20:12:53 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1aaYro-0007x3-4M for qemu-devel@nongnu.org; Mon, 29 Feb 2016 20:12:51 -0500 Received: from mx1.redhat.com ([209.132.183.28]:39950) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1aaYrn-0007wm-S7 for qemu-devel@nongnu.org; Mon, 29 Feb 2016 20:12:48 -0500 Date: Tue, 1 Mar 2016 09:12:41 +0800 From: Fam Zheng Message-ID: <20160301011241.GC15213@ad.usersys.redhat.com> References: <1455626399-7111-1-git-send-email-famz@redhat.com> <1455626399-7111-2-git-send-email-famz@redhat.com> <877fhnjwh9.fsf@linaro.org> MIME-Version: 1.0 Content-Type: text/plain; charset=iso-8859-1 Content-Disposition: inline In-Reply-To: <877fhnjwh9.fsf@linaro.org> Content-Transfer-Encoding: quoted-printable Subject: Re: [Qemu-devel] [PATCH v2 01/15] tests: Add utilities for docker testing List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Alex =?iso-8859-1?Q?Benn=E9e?= Cc: kwolf@redhat.com, peter.maydell@linaro.org, sw@weilnetz.de, qemu-devel@nongnu.org, stefanha@redhat.com, Paolo Bonzini , jsnow@redhat.com, david@gibson.dropbear.id.au On Mon, 02/29 16:46, Alex Benn=E9e wrote: >=20 > Fam Zheng writes: >=20 > > docker_run: A wrapper for "docker run" (or "sudo -n docker run" if > > necessary), which takes care of killing and removing the running > > container at SIGINT. > > > > docker_clean: A tool to tear down all the containers including inacti= ve > > ones that are started by docker_run. > > > > docker_build: A tool to compare an image from given dockerfile and > > rebuild it if they're different. > > > > Signed-off-by: Fam Zheng > > --- > > tests/docker/docker.py | 113 ++++++++++++++++++++++++++++++++++++= ++++++++++ > > tests/docker/docker_build | 42 +++++++++++++++++ > > tests/docker/docker_clean | 22 +++++++++ > > tests/docker/docker_run | 29 ++++++++++++ > > 4 files changed, 206 insertions(+) > > create mode 100755 tests/docker/docker.py > > create mode 100755 tests/docker/docker_build > > create mode 100755 tests/docker/docker_clean > > create mode 100755 tests/docker/docker_run > > > > diff --git a/tests/docker/docker.py b/tests/docker/docker.py > > new file mode 100755 > > index 0000000..d175a86 > > --- /dev/null > > +++ b/tests/docker/docker.py > > @@ -0,0 +1,113 @@ > > +#!/usr/bin/env python2 -B > > +# > > +# Docker controlling module > > +# > > +# Copyright (c) 2016 Red Hat Inc. > > +# > > +# Authors: > > +# Fam Zheng > > +# > > +# This work is licensed under the terms of the GNU GPL, version 2 > > +# or (at your option) any later version. See the COPYING file in > > +# the top-level directory. > > + > > +import os > > +import subprocess > > +import json > > +import hashlib > > +import atexit > > +import uuid > > + > > +class ContainerTerminated(Exception): > > + """ Raised if the container has already existed """ > > + pass > > + > > +class Docker(object): > > + """ Running Docker commands """ > > + def __init__(self): > > + self._command =3D self._guess_command() > > + self._instances =3D [] > > + atexit.register(self._kill_instances) > > + > > + def _do(self, cmd, quiet=3DTrue, **kwargs): > > + if quiet: > > + kwargs["stdout"] =3D subprocess.PIPE > > + return subprocess.call(self._command + cmd, **kwargs) > > + > > + def _do_kill_instances(self, only_known, only_active=3DTrue): > > + cmd =3D ["ps", "-q"] >=20 > Hmm ps -q barfs on my command line: >=20 > 16:04 alex@zen/x86_64 [qemu.git/mttcg/multi_tcg_v8_ajb-r2] >ps -q > error: unsupported SysV option >=20 > Is there not a more portable way of doing this, even if it is a standar= d > library? Down the road this is "sudo docker ps" command. :) >=20 > > + if not only_active: > > + cmd.append("-a") > > + for i in self._output(cmd).split(): > > + resp =3D self._output(["inspect", i]) > > + labels =3D json.loads(resp)[0]["Config"]["Labels"] > > + active =3D json.loads(resp)[0]["State"]["Running"] > > + if not labels: > > + continue > > + instance_uuid =3D labels.get("com.qemu.instance.uuid", N= one) > > + if not instance_uuid: > > + continue > > + if only_known and instance_uuid not in self._instances: > > + continue > > + print "Terminating", i > > + if active: > > + self._do(["kill", i]) > > + self._do(["rm", i]) > > + > > + def clean(self): > > + self._do_kill_instances(False, False) > > + return 0 > > + > > + def _kill_instances(self): > > + return self._do_kill_instances(True) > > + > > + def _output(self, cmd, **kwargs): > > + return subprocess.check_output(self._command + cmd, > > + stderr=3Dsubprocess.STDOUT, > > + **kwargs) > > + > > + def _guess_command(self): > > + commands =3D [["docker"], ["sudo", "-n", "docker"]] > > + for cmd in commands: > > + if subprocess.call(cmd + ["images"], > > + stdout=3Dsubprocess.PIPE, > > + stderr=3Dsubprocess.PIPE) =3D=3D 0: > > + return cmd > > + commands_txt =3D "\n".join([" " + " ".join(x) for x in comm= ands]) > > + raise Exception("Cannot find working docker command. Tried:\= n%s" % commands_txt) > > + > > + def get_image_dockerfile_checksum(self, tag): > > + resp =3D self._output(["inspect", tag]) > > + labels =3D json.loads(resp)[0]["Config"].get("Labels", {}) > > + return labels.get("com.qemu.dockerfile-checksum", "") > > + > > + def checksum(self, text): > > + return hashlib.sha1(text).hexdigest() > > + > > + def build_image(self, tag, dockerfile, df, quiet=3DTrue): > > + tmp =3D dockerfile + "\n" + \ > > + "LABEL com.qemu.dockerfile-checksum=3D%s" % self.check= sum(dockerfile) > > + tmp_df =3D df + ".tmp" > > + tmp_file =3D open(tmp_df, "wb") > > + tmp_file.write(tmp) > > + tmp_file.close() > > + self._do(["build", "-t", tag, "-f", tmp_df, os.path.dirname(= df)], > > + quiet=3Dquiet) > > + os.unlink(tmp_df) > > + > > + def image_matches_dockerfile(self, tag, dockerfile): > > + try: > > + checksum =3D self.get_image_dockerfile_checksum(tag) > > + except: > > + return False > > + return checksum =3D=3D self.checksum(dockerfile) > > + > > + def run(self, cmd, keep, quiet): > > + label =3D uuid.uuid1().hex > > + if not keep: > > + self._instances.append(label) > > + ret =3D self._do(["run", "--label", "com.qemu.instance.uuid=3D= " + label] + cmd, quiet=3Dquiet) > > + if not keep: > > + self._instances.remove(label) > > + return ret >=20 > I think it might be useful to catch some arguments here for testing > things. It is likely to be the first script someone runs while poking > around so some help text would be useful even if it just points at the > other commands. Sure, I can do that. >=20 > In fact I'm not sure why all the various commands aren't in one script > for now given this does most of the heavy lifting. OK, I can merge them into one script. >=20 > > + > > diff --git a/tests/docker/docker_build b/tests/docker/docker_build > > new file mode 100755 > > index 0000000..6948e2c > > --- /dev/null > > +++ b/tests/docker/docker_build > > @@ -0,0 +1,42 @@ > > +#!/usr/bin/env python2 > > +# > > +# Compare to Dockerfile and rebuild a docker image if necessary. > > +# > > +# Copyright (c) 2016 Red Hat Inc. > > +# > > +# Authors: > > +# Fam Zheng > > +# > > +# This work is licensed under the terms of the GNU GPL, version 2 > > +# or (at your option) any later version. See the COPYING file in > > +# the top-level directory. > > + > > +import sys > > +import docker > > +import argparse > > + > > +def main(): > > + parser =3D argparse.ArgumentParser() > > + parser.add_argument("tag", > > + help=3D"Image Tag") > > + parser.add_argument("dockerfile", > > + help=3D"Dockerfile name") > > + parser.add_argument("--verbose", "-v", action=3D"store_true", > > + help=3D"Print verbose information") > > + args =3D parser.parse_args() > > + > > + dockerfile =3D open(args.dockerfile, "rb").read() > > + tag =3D args.tag > > + > > + dkr =3D docker.Docker() > > + if dkr.image_matches_dockerfile(tag, dockerfile): > > + if args.verbose: > > + print "Image is up to date." > > + return 0 > > + > > + quiet =3D not args.verbose > > + dkr.build_image(tag, dockerfile, args.dockerfile, quiet=3Dquiet) > > + return 0 > > + > > +if __name__ =3D=3D "__main__": > > + sys.exit(main()) > > diff --git a/tests/docker/docker_clean b/tests/docker/docker_clean > > new file mode 100755 > > index 0000000..88cdba6 > > --- /dev/null > > +++ b/tests/docker/docker_clean > > @@ -0,0 +1,22 @@ > > +#!/usr/bin/env python2 > > +# > > +# Clean up uselsee containers. > > +# > > +# Copyright (c) 2016 Red Hat Inc. > > +# > > +# Authors: > > +# Fam Zheng > > +# > > +# This work is licensed under the terms of the GNU GPL, version 2 > > +# or (at your option) any later version. See the COPYING file in > > +# the top-level directory. > > + > > +import sys > > +import docker > > + > > +def main(): > > + docker.Docker().clean() > > + return 0 > > + > > +if __name__ =3D=3D "__main__": > > + sys.exit(main()) >=20 > Of all the scripts run if you call with --help this just does something > straight away. It should at least attempt a usage() text to prevent > accidents. Yes. Will address. Fam >=20 > > diff --git a/tests/docker/docker_run b/tests/docker/docker_run > > new file mode 100755 > > index 0000000..4c46d90 > > --- /dev/null > > +++ b/tests/docker/docker_run > > @@ -0,0 +1,29 @@ > > +#!/usr/bin/env python2 > > +# > > +# Wrapper for "docker run" with automatical clean up if the executio= n is > > +# iterrupted. > > +# > > +# Copyright (c) 2016 Red Hat Inc. > > +# > > +# Authors: > > +# Fam Zheng > > +# > > +# This work is licensed under the terms of the GNU GPL, version 2 > > +# or (at your option) any later version. See the COPYING file in > > +# the top-level directory. > > + > > +import sys > > +import argparse > > +import docker > > + > > +def main(): > > + parser =3D argparse.ArgumentParser() > > + parser.add_argument("--keep", action=3D"store_true", > > + help=3D"Don't remove image when the command = completes") > > + parser.add_argument("--quiet", action=3D"store_true", > > + help=3D"Run quietly unless an error occured"= ) > > + args, argv =3D parser.parse_known_args() > > + return docker.Docker().run(argv, args.keep, quiet=3Dargs.quiet) > > + > > +if __name__ =3D=3D "__main__": > > + sys.exit(main()) >=20 >=20 > -- > Alex Benn=E9e