tests/docker/docker.py

   1 #!/usr/bin/env python2
   2 #
   3 # Docker controlling module
   4 #
   5 # Copyright (c) 2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Fam Zheng <famz@redhat.com>
   9 #
  10 # This work is licensed under the terms of the GNU GPL, version 2
  11 # or (at your option) any later version. See the COPYING file in
  12 # the top-level directory.
  13
  14 import os
  15 import sys
  16 sys.path.append(os.path.join(os.path.dirname(__file__),
  17                              '..', '..', 'scripts'))
  18 import argparse
  19 import subprocess
  20 import json
  21 import hashlib
  22 import atexit
  23 import uuid
  24 import tempfile
  25 import re
  26 import signal
  27 from tarfile import TarFile, TarInfo
  28 from StringIO import StringIO
  29 from shutil import copy, rmtree
  30 from pwd import getpwuid
  31
  32
  33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
  34
  35
  36 DEVNULL = open(os.devnull, 'wb')
  37
  38
  39 def _text_checksum(text):
  40     """Calculate a digest string unique to the text content"""
  41     return hashlib.sha1(text).hexdigest()
  42
  43 def _file_checksum(filename):
  44     return _text_checksum(open(filename, 'rb').read())
  45
  46 def _guess_docker_command():
  47     """ Guess a working docker command or raise exception if not found"""
  48     commands = [["docker"], ["sudo", "-n", "docker"]]
  49     for cmd in commands:
  50         try:
  51             if subprocess.call(cmd + ["images"],
  52                                stdout=DEVNULL, stderr=DEVNULL) == 0:
  53                 return cmd
  54         except OSError:
  55             pass
  56     commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
  57     raise Exception("Cannot find working docker command. Tried:\n%s" % \
  58                     commands_txt)
  59
  60 def _copy_with_mkdir(src, root_dir, sub_path='.'):
  61     """Copy src into root_dir, creating sub_path as needed."""
  62     dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
  63     try:
  64         os.makedirs(dest_dir)
  65     except OSError:
  66         # we can safely ignore already created directories
  67         pass
  68
  69     dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
  70     copy(src, dest_file)
  71
  72
  73 def _get_so_libs(executable):
  74     """Return a list of libraries associated with an executable.
  75
  76     The paths may be symbolic links which would need to be resolved to
  77     ensure theright data is copied."""
  78
  79     libs = []
  80     ldd_re = re.compile(r"(/.*/)(\S*)")
  81     try:
  82         ldd_output = subprocess.check_output(["ldd", executable])
  83         for line in ldd_output.split("\n"):
  84             search = ldd_re.search(line)
  85             if search and len(search.groups()) == 2:
  86                 so_path = search.groups()[0]
  87                 so_lib = search.groups()[1]
  88                 libs.append("%s/%s" % (so_path, so_lib))
  89     except subprocess.CalledProcessError:
  90         print "%s had no associated libraries (static build?)" % (executable)
  91
  92     return libs
  93
  94 def _copy_binary_with_libs(src, dest_dir):
  95     """Copy a binary executable and all its dependant libraries.
  96
  97     This does rely on the host file-system being fairly multi-arch
  98     aware so the file don't clash with the guests layout."""
  99
 100     _copy_with_mkdir(src, dest_dir, "/usr/bin")
 101
 102     libs = _get_so_libs(src)
 103     if libs:
 104         for l in libs:
 105             so_path = os.path.dirname(l)
 106             _copy_with_mkdir(l , dest_dir, so_path)
 107
 108 def _read_qemu_dockerfile(img_name):
 109     df = os.path.join(os.path.dirname(__file__), "dockerfiles",
 110                       img_name + ".docker")
 111     return open(df, "r").read()
 112
 113 def _dockerfile_preprocess(df):
 114     out = ""
 115     for l in df.splitlines():
 116         if len(l.strip()) == 0 or l.startswith("#"):
 117             continue
 118         from_pref = "FROM qemu:"
 119         if l.startswith(from_pref):
 120             # TODO: Alternatively we could replace this line with "FROM $ID"
 121             # where $ID is the image's hex id obtained with
 122             #    $ docker images $IMAGE --format="{{.Id}}"
 123             # but unfortunately that's not supported by RHEL 7.
 124             inlining = _read_qemu_dockerfile(l[len(from_pref):])
 125             out += _dockerfile_preprocess(inlining)
 126             continue
 127         out += l + "\n"
 128     return out
 129
 130 class Docker(object):
 131     """ Running Docker commands """
 132     def __init__(self):
 133         self._command = _guess_docker_command()
 134         self._instances = []
 135         atexit.register(self._kill_instances)
 136         signal.signal(signal.SIGTERM, self._kill_instances)
 137         signal.signal(signal.SIGHUP, self._kill_instances)
 138
 139     def _do(self, cmd, quiet=True, **kwargs):
 140         if quiet:
 141             kwargs["stdout"] = DEVNULL
 142         return subprocess.call(self._command + cmd, **kwargs)
 143
 144     def _do_check(self, cmd, quiet=True, **kwargs):
 145         if quiet:
 146             kwargs["stdout"] = DEVNULL
 147         return subprocess.check_call(self._command + cmd, **kwargs)
 148
 149     def _do_kill_instances(self, only_known, only_active=True):
 150         cmd = ["ps", "-q"]
 151         if not only_active:
 152             cmd.append("-a")
 153         for i in self._output(cmd).split():
 154             resp = self._output(["inspect", i])
 155             labels = json.loads(resp)[0]["Config"]["Labels"]
 156             active = json.loads(resp)[0]["State"]["Running"]
 157             if not labels:
 158                 continue
 159             instance_uuid = labels.get("com.qemu.instance.uuid", None)
 160             if not instance_uuid:
 161                 continue
 162             if only_known and instance_uuid not in self._instances:
 163                 continue
 164             print "Terminating", i
 165             if active:
 166                 self._do(["kill", i])
 167             self._do(["rm", i])
 168
 169     def clean(self):
 170         self._do_kill_instances(False, False)
 171         return 0
 172
 173     def _kill_instances(self, *args, **kwargs):
 174         return self._do_kill_instances(True)
 175
 176     def _output(self, cmd, **kwargs):
 177         return subprocess.check_output(self._command + cmd,
 178                                        stderr=subprocess.STDOUT,
 179                                        **kwargs)
 180
 181     def get_image_dockerfile_checksum(self, tag):
 182         resp = self._output(["inspect", tag])
 183         labels = json.loads(resp)[0]["Config"].get("Labels", {})
 184         return labels.get("com.qemu.dockerfile-checksum", "")
 185
 186     def build_image(self, tag, docker_dir, dockerfile,
 187                     quiet=True, user=False, argv=None, extra_files_cksum=[]):
 188         if argv == None:
 189             argv = []
 190
 191         tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
 192         tmp_df.write(dockerfile)
 193
 194         if user:
 195             uid = os.getuid()
 196             uname = getpwuid(uid).pw_name
 197             tmp_df.write("\n")
 198             tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
 199                          (uname, uid, uname))
 200
 201         tmp_df.write("\n")
 202         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
 203                      _text_checksum("\n".join([dockerfile] +
 204                                     extra_files_cksum)))
 205         tmp_df.flush()
 206
 207         self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
 208                        [docker_dir],
 209                        quiet=quiet)
 210
 211     def update_image(self, tag, tarball, quiet=True):
 212         "Update a tagged image using "
 213
 214         self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
 215
 216     def image_matches_dockerfile(self, tag, dockerfile):
 217         try:
 218             checksum = self.get_image_dockerfile_checksum(tag)
 219         except Exception:
 220             return False
 221         return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
 222
 223     def run(self, cmd, keep, quiet):
 224         label = uuid.uuid1().hex
 225         if not keep:
 226             self._instances.append(label)
 227         ret = self._do_check(["run", "--label",
 228                              "com.qemu.instance.uuid=" + label] + cmd,
 229                              quiet=quiet)
 230         if not keep:
 231             self._instances.remove(label)
 232         return ret
 233
 234     def command(self, cmd, argv, quiet):
 235         return self._do([cmd] + argv, quiet=quiet)
 236
 237 class SubCommand(object):
 238     """A SubCommand template base class"""
 239     name = None # Subcommand name
 240     def shared_args(self, parser):
 241         parser.add_argument("--quiet", action="store_true",
 242                             help="Run quietly unless an error occured")
 243
 244     def args(self, parser):
 245         """Setup argument parser"""
 246         pass
 247     def run(self, args, argv):
 248         """Run command.
 249         args: parsed argument by argument parser.
 250         argv: remaining arguments from sys.argv.
 251         """
 252         pass
 253
 254 class RunCommand(SubCommand):
 255     """Invoke docker run and take care of cleaning up"""
 256     name = "run"
 257     def args(self, parser):
 258         parser.add_argument("--keep", action="store_true",
 259                             help="Don't remove image when command completes")
 260     def run(self, args, argv):
 261         return Docker().run(argv, args.keep, quiet=args.quiet)
 262
 263 class BuildCommand(SubCommand):
 264     """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
 265     name = "build"
 266     def args(self, parser):
 267         parser.add_argument("--include-executable", "-e",
 268                             help="""Specify a binary that will be copied to the
 269                             container together with all its dependent
 270                             libraries""")
 271         parser.add_argument("--extra-files", "-f", nargs='*',
 272                             help="""Specify files that will be copied in the
 273                             Docker image, fulfilling the ADD directive from the
 274                             Dockerfile""")
 275         parser.add_argument("--add-current-user", "-u", dest="user",
 276                             action="store_true",
 277                             help="Add the current user to image's passwd")
 278         parser.add_argument("tag",
 279                             help="Image Tag")
 280         parser.add_argument("dockerfile",
 281                             help="Dockerfile name")
 282
 283     def run(self, args, argv):
 284         dockerfile = open(args.dockerfile, "rb").read()
 285         tag = args.tag
 286
 287         dkr = Docker()
 288         if "--no-cache" not in argv and \
 289            dkr.image_matches_dockerfile(tag, dockerfile):
 290             if not args.quiet:
 291                 print "Image is up to date."
 292         else:
 293             # Create a docker context directory for the build
 294             docker_dir = tempfile.mkdtemp(prefix="docker_build")
 295
 296             # Is there a .pre file to run in the build context?
 297             docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
 298             if os.path.exists(docker_pre):
 299                 stdout = DEVNULL if args.quiet else None
 300                 rc = subprocess.call(os.path.realpath(docker_pre),
 301                                      cwd=docker_dir, stdout=stdout)
 302                 if rc == 3:
 303                     print "Skip"
 304                     return 0
 305                 elif rc != 0:
 306                     print "%s exited with code %d" % (docker_pre, rc)
 307                     return 1
 308
 309             # Copy any extra files into the Docker context. These can be
 310             # included by the use of the ADD directive in the Dockerfile.
 311             cksum = []
 312             if args.include_executable:
 313                 # FIXME: there is no checksum of this executable and the linked
 314                 # libraries, once the image built any change of this executable
 315                 # or any library won't trigger another build.
 316                 _copy_binary_with_libs(args.include_executable, docker_dir)
 317             for filename in args.extra_files or []:
 318                 _copy_with_mkdir(filename, docker_dir)
 319                 cksum += [_file_checksum(filename)]
 320
 321             argv += ["--build-arg=" + k.lower() + "=" + v
 322                         for k, v in os.environ.iteritems()
 323                         if k.lower() in FILTERED_ENV_NAMES]
 324             dkr.build_image(tag, docker_dir, dockerfile,
 325                             quiet=args.quiet, user=args.user, argv=argv,
 326                             extra_files_cksum=cksum)
 327
 328             rmtree(docker_dir)
 329
 330         return 0
 331
 332 class UpdateCommand(SubCommand):
 333     """ Update a docker image with new executables. Arguments: <tag> <executable>"""
 334     name = "update"
 335     def args(self, parser):
 336         parser.add_argument("tag",
 337                             help="Image Tag")
 338         parser.add_argument("executable",
 339                             help="Executable to copy")
 340
 341     def run(self, args, argv):
 342         # Create a temporary tarball with our whole build context and
 343         # dockerfile for the update
 344         tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
 345         tmp_tar = TarFile(fileobj=tmp, mode='w')
 346
 347         # Add the executable to the tarball
 348         bn = os.path.basename(args.executable)
 349         ff = "/usr/bin/%s" % bn
 350         tmp_tar.add(args.executable, arcname=ff)
 351
 352         # Add any associated libraries
 353         libs = _get_so_libs(args.executable)
 354         if libs:
 355             for l in libs:
 356                 tmp_tar.add(os.path.realpath(l), arcname=l)
 357
 358         # Create a Docker buildfile
 359         df = StringIO()
 360         df.write("FROM %s\n" % args.tag)
 361         df.write("ADD . /\n")
 362         df.seek(0)
 363
 364         df_tar = TarInfo(name="Dockerfile")
 365         df_tar.size = len(df.buf)
 366         tmp_tar.addfile(df_tar, fileobj=df)
 367
 368         tmp_tar.close()
 369
 370         # reset the file pointers
 371         tmp.flush()
 372         tmp.seek(0)
 373
 374         # Run the build with our tarball context
 375         dkr = Docker()
 376         dkr.update_image(args.tag, tmp, quiet=args.quiet)
 377
 378         return 0
 379
 380 class CleanCommand(SubCommand):
 381     """Clean up docker instances"""
 382     name = "clean"
 383     def run(self, args, argv):
 384         Docker().clean()
 385         return 0
 386
 387 class ImagesCommand(SubCommand):
 388     """Run "docker images" command"""
 389     name = "images"
 390     def run(self, args, argv):
 391         return Docker().command("images", argv, args.quiet)
 392
 393 def main():
 394     parser = argparse.ArgumentParser(description="A Docker helper",
 395             usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
 396     subparsers = parser.add_subparsers(title="subcommands", help=None)
 397     for cls in SubCommand.__subclasses__():
 398         cmd = cls()
 399         subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
 400         cmd.shared_args(subp)
 401         cmd.args(subp)
 402         subp.set_defaults(cmdobj=cmd)
 403     args, argv = parser.parse_known_args()
 404     return args.cmdobj.run(args, argv)
 405
 406 if __name__ == "__main__":
 407     sys.exit(main())