docker-image-%: $(DOCKER_FILES_DIR)/%.docker
$(call quiet-command,\
$(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \
- $(if $V,,--quiet) $(if $(NOCACHE),--no-cache),\
+ $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \
+ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
" BUILD $*")
# Expand all the pre-requistes for each docker image and test combination
@echo ' DEBUG=1 Stop and drop to shell in the created container'
@echo ' before running the command.'
@echo ' NOCACHE=1 Ignore cache when build images.'
+ @echo ' EXECUTABLE=<path> Include executable in image.'
docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/')
docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/')
fi
$(if $(filter $(TESTS),$(CMD)),$(if $(filter $(IMAGES),$(IMAGE)), \
$(call quiet-command,\
- $(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
+ if $(SRC_PATH)/tests/docker/docker.py images \
+ --format={{.Repository}}:{{.Tag}} | \
+ grep -qx qemu:$(IMAGE); then \
+ $(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
-t \
$(if $(DEBUG),-i,--net=none) \
-e TARGET_LIST=$(TARGET_LIST) \
-e CCACHE_DIR=/var/tmp/ccache \
-v $$(realpath $(DOCKER_SRC_COPY)):/var/tmp/qemu:z$(COMMA)ro \
-v $(DOCKER_CCACHE_DIR):/var/tmp/ccache:z \
- -w /var/tmp/qemu \
qemu:$(IMAGE) \
- $(if $V,/bin/bash -x ,) \
- ./run \
+ /var/tmp/qemu/run \
$(CMD); \
+ fi \
, " RUN $(CMD) in $(IMAGE)")))
docker-clean:
import uuid
import argparse
import tempfile
-from shutil import copy
+import re
+from tarfile import TarFile, TarInfo
+from StringIO import StringIO
+from shutil import copy, rmtree
def _text_checksum(text):
"""Calculate a digest string unique to the text content"""
raise Exception("Cannot find working docker command. Tried:\n%s" % \
commands_txt)
+def _copy_with_mkdir(src, root_dir, sub_path):
+ """Copy src into root_dir, creating sub_path as needed."""
+ dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
+ try:
+ os.makedirs(dest_dir)
+ except OSError:
+ # we can safely ignore already created directories
+ pass
+
+ dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
+ copy(src, dest_file)
+
+
+def _get_so_libs(executable):
+ """Return a list of libraries associated with an executable.
+
+ The paths may be symbolic links which would need to be resolved to
+ ensure theright data is copied."""
+
+ libs = []
+ ldd_re = re.compile(r"(/.*/)(\S*)")
+ try:
+ ldd_output = subprocess.check_output(["ldd", executable])
+ for line in ldd_output.split("\n"):
+ search = ldd_re.search(line)
+ if search and len(search.groups()) == 2:
+ so_path = search.groups()[0]
+ so_lib = search.groups()[1]
+ libs.append("%s/%s" % (so_path, so_lib))
+ except subprocess.CalledProcessError:
+ print "%s had no associated libraries (static build?)" % (executable)
+
+ return libs
+
+def _copy_binary_with_libs(src, dest_dir):
+ """Copy a binary executable and all its dependant libraries.
+
+ This does rely on the host file-system being fairly multi-arch
+ aware so the file don't clash with the guests layout."""
+
+ _copy_with_mkdir(src, dest_dir, "/usr/bin")
+
+ libs = _get_so_libs(src)
+ if libs:
+ for l in libs:
+ so_path = os.path.dirname(l)
+ _copy_with_mkdir(l , dest_dir, so_path)
+
class Docker(object):
""" Running Docker commands """
def __init__(self):
self._instances = []
atexit.register(self._kill_instances)
- def _do(self, cmd, quiet=True, **kwargs):
+ def _do(self, cmd, quiet=True, infile=None, **kwargs):
if quiet:
kwargs["stdout"] = subprocess.PIPE
+ if infile:
+ kwargs["stdin"] = infile
return subprocess.call(self._command + cmd, **kwargs)
def _do_kill_instances(self, only_known, only_active=True):
labels = json.loads(resp)[0]["Config"].get("Labels", {})
return labels.get("com.qemu.dockerfile-checksum", "")
- def build_image(self, tag, dockerfile, df_path, quiet=True, argv=None):
+ def build_image(self, tag, docker_dir, dockerfile, quiet=True, argv=None):
if argv == None:
argv = []
- tmp_dir = tempfile.mkdtemp(prefix="docker_build")
- tmp_df = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".docker")
+ tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
tmp_df.write(dockerfile)
tmp_df.write("\n")
tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
_text_checksum(dockerfile))
tmp_df.flush()
+
self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
- [tmp_dir],
+ [docker_dir],
quiet=quiet)
+ def update_image(self, tag, tarball, quiet=True):
+ "Update a tagged image using "
+
+ self._do(["build", "-t", tag, "-"], quiet=quiet, infile=tarball)
+
def image_matches_dockerfile(self, tag, dockerfile):
try:
checksum = self.get_image_dockerfile_checksum(tag)
self._instances.remove(label)
return ret
+ def command(self, cmd, argv, quiet):
+ return self._do([cmd] + argv, quiet=quiet)
+
class SubCommand(object):
"""A SubCommand template base class"""
name = None # Subcommand name
""" Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
name = "build"
def args(self, parser):
+ parser.add_argument("--include-executable", "-e",
+ help="""Specify a binary that will be copied to the
+ container together with all its dependent
+ libraries""")
parser.add_argument("tag",
help="Image Tag")
parser.add_argument("dockerfile",
if dkr.image_matches_dockerfile(tag, dockerfile):
if not args.quiet:
print "Image is up to date."
- return 0
+ else:
+ # Create a docker context directory for the build
+ docker_dir = tempfile.mkdtemp(prefix="docker_build")
+
+ # Is there a .pre file to run in the build context?
+ docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
+ if os.path.exists(docker_pre):
+ rc = subprocess.call(os.path.realpath(docker_pre),
+ cwd=docker_dir)
+ if rc == 3:
+ print "Skip"
+ return 0
+ elif rc != 0:
+ print "%s exited with code %d" % (docker_pre, rc)
+ return 1
+
+ # Do we include a extra binary?
+ if args.include_executable:
+ _copy_binary_with_libs(args.include_executable,
+ docker_dir)
+
+ dkr.build_image(tag, docker_dir, dockerfile,
+ quiet=args.quiet, argv=argv)
+
+ rmtree(docker_dir)
+
+ return 0
+
+class UpdateCommand(SubCommand):
+ """ Update a docker image with new executables. Arguments: <tag> <executable>"""
+ name = "update"
+ def args(self, parser):
+ parser.add_argument("tag",
+ help="Image Tag")
+ parser.add_argument("executable",
+ help="Executable to copy")
+
+ def run(self, args, argv):
+ # Create a temporary tarball with our whole build context and
+ # dockerfile for the update
+ tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
+ tmp_tar = TarFile(fileobj=tmp, mode='w')
+
+ # Add the executable to the tarball
+ bn = os.path.basename(args.executable)
+ ff = "/usr/bin/%s" % bn
+ tmp_tar.add(args.executable, arcname=ff)
+
+ # Add any associated libraries
+ libs = _get_so_libs(args.executable)
+ if libs:
+ for l in libs:
+ tmp_tar.add(os.path.realpath(l), arcname=l)
+
+ # Create a Docker buildfile
+ df = StringIO()
+ df.write("FROM %s\n" % args.tag)
+ df.write("ADD . /\n")
+ df.seek(0)
+
+ df_tar = TarInfo(name="Dockerfile")
+ df_tar.size = len(df.buf)
+ tmp_tar.addfile(df_tar, fileobj=df)
+
+ tmp_tar.close()
+
+ # reset the file pointers
+ tmp.flush()
+ tmp.seek(0)
+
+ # Run the build with our tarball context
+ dkr = Docker()
+ dkr.update_image(args.tag, tmp, quiet=args.quiet)
- dkr.build_image(tag, dockerfile, args.dockerfile,
- quiet=args.quiet, argv=argv)
return 0
class CleanCommand(SubCommand):
Docker().clean()
return 0
+class ImagesCommand(SubCommand):
+ """Run "docker images" command"""
+ name = "images"
+ def run(self, args, argv):
+ return Docker().command("images", argv, args.quiet)
+
def main():
parser = argparse.ArgumentParser(description="A Docker helper",
usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
--- /dev/null
+#!/bin/sh
+#
+# Simple wrapper for debootstrap, run in the docker build context
+#
+FAKEROOT=`which fakeroot 2> /dev/null`
+
+exit_and_skip()
+{
+ exit 3
+}
+
+#
+# fakeroot is needed to run the bootstrap stage
+#
+if [ -z $FAKEROOT ]; then
+ echo "Please install fakeroot to enable bootstraping"
+ exit_and_skip
+fi
+
+# We check in order for
+#
+# - DEBOOTSTRAP_DIR pointing at a development checkout
+# - PATH for the debootstrap script (installed)
+#
+# If neither option works then we checkout debootstrap from its
+# upstream SCM and run it from there.
+#
+
+if [ -z $DEBOOTSTRAP_DIR ]; then
+ DEBOOTSTRAP=`which debootstrap 2> /dev/null`
+ if [ -z $DEBOOTSTRAP ]; then
+ echo "No debootstrap installed, attempting to install from SCM"
+ DEBOOTSTRAP_SOURCE=https://anonscm.debian.org/git/d-i/debootstrap.git
+ git clone ${DEBOOTSTRAP_SOURCE} ./debootstrap.git
+ export DEBOOTSTRAP_DIR=./debootstrap.git
+ DEBOOTSTRAP=./debootstrap.git/debootstrap
+ fi
+else
+ DEBOOTSTRAP=${DEBOOTSTRAP_DIR}/debootstrap
+ if [ ! -f $DEBOOTSTRAP ]; then
+ echo "Couldn't find script at ${DEBOOTSTRAP}"
+ exit_and_skip
+ fi
+fi
+
+#
+# Finally check to see if any qemu's are installed
+#
+BINFMT_DIR=/proc/sys/fs/binfmt_misc
+if [ ! -e $BINFMT_DIR ]; then
+ echo "binfmt_misc needs enabling for a QEMU bootstrap to work"
+ exit_and_skip
+else
+ # DEB_ARCH and QEMU arch names are not totally aligned
+ case "${DEB_ARCH}" in
+ amd64)
+ QEMU=qemu-i386
+ ;;
+ armel|armhf)
+ QEMU=qemu-arm
+ ;;
+ arm64)
+ QEMU=qemu-aarch64
+ ;;
+ powerpc)
+ QEMU=qemu-ppc
+ ;;
+ ppc64el)
+ QEMU=qemu-ppc64le
+ ;;
+ s390)
+ QEMU=qemu-s390x
+ ;;
+ *)
+ QEMU=qemu-${DEB_ARCH}
+ ;;
+ esac
+ if [ ! -e "${BINFMT_DIR}/$QEMU" ]; then
+ echo "No binfmt_misc rule to run $QEMU, can't bootstrap"
+ exit_and_skip
+ fi
+fi
+
+echo "Building a rootfs using ${FAKEROOT} and ${DEBOOTSTRAP} ${DEB_ARCH}/${DEB_TYPE}"
+
+${FAKEROOT} ${DEBOOTSTRAP} --variant=buildd --foreign --arch=$DEB_ARCH $DEB_TYPE . http://httpredir.debian.org/debian || exit 1
+exit 0
projects / mirror_qemu.git / commitdiff