]>
git.proxmox.com Git - mirror_qemu.git/blob - tests/docker/docker.py
3 # Docker controlling module
5 # Copyright (c) 2016 Red Hat Inc.
8 # Fam Zheng <famz@redhat.com>
10 # This work is licensed under the terms of the GNU GPL, version 2
11 # or (at your option) any later version. See the COPYING file in
12 # the top-level directory.
14 from __future__
import print_function
27 from tarfile
import TarFile
, TarInfo
29 from StringIO
import StringIO
31 from io
import StringIO
32 from shutil
import copy
, rmtree
33 from pwd
import getpwuid
34 from datetime
import datetime
, timedelta
37 FILTERED_ENV_NAMES
= ['ftp_proxy', 'http_proxy', 'https_proxy']
40 DEVNULL
= open(os
.devnull
, 'wb')
42 class EngineEnum(enum
.IntEnum
):
48 return self
.name
.lower()
56 return EngineEnum
[s
.upper()]
61 USE_ENGINE
= EngineEnum
.AUTO
63 def _text_checksum(text
):
64 """Calculate a digest string unique to the text content"""
65 return hashlib
.sha1(text
).hexdigest()
68 def _file_checksum(filename
):
69 return _text_checksum(open(filename
, 'rb').read())
72 def _guess_engine_command():
73 """ Guess a working engine command or raise exception if not found"""
76 if USE_ENGINE
in [EngineEnum
.AUTO
, EngineEnum
.PODMAN
]:
77 commands
+= [["podman"]]
78 if USE_ENGINE
in [EngineEnum
.AUTO
, EngineEnum
.DOCKER
]:
79 commands
+= [["docker"], ["sudo", "-n", "docker"]]
82 # docker version will return the client details in stdout
83 # but still report a status of 1 if it can't contact the daemon
84 if subprocess
.call(cmd
+ ["version"],
85 stdout
=DEVNULL
, stderr
=DEVNULL
) == 0:
89 commands_txt
= "\n".join([" " + " ".join(x
) for x
in commands
])
90 raise Exception("Cannot find working engine command. Tried:\n%s" %
94 def _copy_with_mkdir(src
, root_dir
, sub_path
='.'):
95 """Copy src into root_dir, creating sub_path as needed."""
96 dest_dir
= os
.path
.normpath("%s/%s" % (root_dir
, sub_path
))
100 # we can safely ignore already created directories
103 dest_file
= "%s/%s" % (dest_dir
, os
.path
.basename(src
))
107 def _get_so_libs(executable
):
108 """Return a list of libraries associated with an executable.
110 The paths may be symbolic links which would need to be resolved to
111 ensure theright data is copied."""
114 ldd_re
= re
.compile(r
"(/.*/)(\S*)")
116 ldd_output
= subprocess
.check_output(["ldd", executable
])
117 for line
in ldd_output
.split("\n"):
118 search
= ldd_re
.search(line
)
119 if search
and len(search
.groups()) == 2:
120 so_path
= search
.groups()[0]
121 so_lib
= search
.groups()[1]
122 libs
.append("%s/%s" % (so_path
, so_lib
))
123 except subprocess
.CalledProcessError
:
124 print("%s had no associated libraries (static build?)" % (executable
))
129 def _copy_binary_with_libs(src
, bin_dest
, dest_dir
):
130 """Maybe copy a binary and all its dependent libraries.
132 If bin_dest isn't set we only copy the support libraries because
133 we don't need qemu in the docker path to run (due to persistent
134 mapping). Indeed users may get confused if we aren't running what
137 This does rely on the host file-system being fairly multi-arch
138 aware so the file don't clash with the guests layout.
142 _copy_with_mkdir(src
, dest_dir
, os
.path
.dirname(bin_dest
))
144 print("only copying support libraries for %s" % (src
))
146 libs
= _get_so_libs(src
)
149 so_path
= os
.path
.dirname(l
)
150 _copy_with_mkdir(l
, dest_dir
, so_path
)
153 def _check_binfmt_misc(executable
):
154 """Check binfmt_misc has entry for executable in the right place.
156 The details of setting up binfmt_misc are outside the scope of
157 this script but we should at least fail early with a useful
158 message if it won't work.
160 Returns the configured binfmt path and a valid flag. For
161 persistent configurations we will still want to copy and dependent
165 binary
= os
.path
.basename(executable
)
166 binfmt_entry
= "/proc/sys/fs/binfmt_misc/%s" % (binary
)
168 if not os
.path
.exists(binfmt_entry
):
169 print ("No binfmt_misc entry for %s" % (binary
))
172 with
open(binfmt_entry
) as x
: entry
= x
.read()
174 if re
.search("flags:.*F.*\n", entry
):
175 print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
179 m
= re
.search("interpreter (\S+)\n", entry
)
181 if interp
and interp
!= executable
:
182 print("binfmt_misc for %s does not point to %s, using %s" %
183 (binary
, executable
, interp
))
188 def _read_qemu_dockerfile(img_name
):
189 # special case for Debian linux-user images
190 if img_name
.startswith("debian") and img_name
.endswith("user"):
191 img_name
= "debian-bootstrap"
193 df
= os
.path
.join(os
.path
.dirname(__file__
), "dockerfiles",
194 img_name
+ ".docker")
195 return open(df
, "r").read()
198 def _dockerfile_preprocess(df
):
200 for l
in df
.splitlines():
201 if len(l
.strip()) == 0 or l
.startswith("#"):
203 from_pref
= "FROM qemu:"
204 if l
.startswith(from_pref
):
205 # TODO: Alternatively we could replace this line with "FROM $ID"
206 # where $ID is the image's hex id obtained with
207 # $ docker images $IMAGE --format="{{.Id}}"
208 # but unfortunately that's not supported by RHEL 7.
209 inlining
= _read_qemu_dockerfile(l
[len(from_pref
):])
210 out
+= _dockerfile_preprocess(inlining
)
216 class Docker(object):
217 """ Running Docker commands """
219 self
._command
= _guess_engine_command()
221 atexit
.register(self
._kill
_instances
)
222 signal
.signal(signal
.SIGTERM
, self
._kill
_instances
)
223 signal
.signal(signal
.SIGHUP
, self
._kill
_instances
)
225 def _do(self
, cmd
, quiet
=True, **kwargs
):
227 kwargs
["stdout"] = DEVNULL
228 return subprocess
.call(self
._command
+ cmd
, **kwargs
)
230 def _do_check(self
, cmd
, quiet
=True, **kwargs
):
232 kwargs
["stdout"] = DEVNULL
233 return subprocess
.check_call(self
._command
+ cmd
, **kwargs
)
235 def _do_kill_instances(self
, only_known
, only_active
=True):
239 for i
in self
._output
(cmd
).split():
240 resp
= self
._output
(["inspect", i
])
241 labels
= json
.loads(resp
)[0]["Config"]["Labels"]
242 active
= json
.loads(resp
)[0]["State"]["Running"]
245 instance_uuid
= labels
.get("com.qemu.instance.uuid", None)
246 if not instance_uuid
:
248 if only_known
and instance_uuid
not in self
._instances
:
250 print("Terminating", i
)
252 self
._do
(["kill", i
])
256 self
._do
_kill
_instances
(False, False)
259 def _kill_instances(self
, *args
, **kwargs
):
260 return self
._do
_kill
_instances
(True)
262 def _output(self
, cmd
, **kwargs
):
263 return subprocess
.check_output(self
._command
+ cmd
,
264 stderr
=subprocess
.STDOUT
,
267 def inspect_tag(self
, tag
):
269 return self
._output
(["inspect", tag
])
270 except subprocess
.CalledProcessError
:
273 def get_image_creation_time(self
, info
):
274 return json
.loads(info
)[0]["Created"]
276 def get_image_dockerfile_checksum(self
, tag
):
277 resp
= self
.inspect_tag(tag
)
278 labels
= json
.loads(resp
)[0]["Config"].get("Labels", {})
279 return labels
.get("com.qemu.dockerfile-checksum", "")
281 def build_image(self
, tag
, docker_dir
, dockerfile
,
282 quiet
=True, user
=False, argv
=None, extra_files_cksum
=[]):
286 tmp_df
= tempfile
.NamedTemporaryFile(dir=docker_dir
, suffix
=".docker")
287 tmp_df
.write(dockerfile
)
291 uname
= getpwuid(uid
).pw_name
293 tmp_df
.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
297 tmp_df
.write("LABEL com.qemu.dockerfile-checksum=%s" %
298 _text_checksum(_dockerfile_preprocess(dockerfile
)))
299 for f
, c
in extra_files_cksum
:
300 tmp_df
.write("LABEL com.qemu.%s-checksum=%s" % (f
, c
))
304 self
._do
_check
(["build", "-t", tag
, "-f", tmp_df
.name
] + argv
+
308 def update_image(self
, tag
, tarball
, quiet
=True):
309 "Update a tagged image using "
311 self
._do
_check
(["build", "-t", tag
, "-"], quiet
=quiet
, stdin
=tarball
)
313 def image_matches_dockerfile(self
, tag
, dockerfile
):
315 checksum
= self
.get_image_dockerfile_checksum(tag
)
318 return checksum
== _text_checksum(_dockerfile_preprocess(dockerfile
))
320 def run(self
, cmd
, keep
, quiet
):
321 label
= uuid
.uuid1().hex
323 self
._instances
.append(label
)
324 ret
= self
._do
_check
(["run", "--label",
325 "com.qemu.instance.uuid=" + label
] + cmd
,
328 self
._instances
.remove(label
)
331 def command(self
, cmd
, argv
, quiet
):
332 return self
._do
([cmd
] + argv
, quiet
=quiet
)
335 class SubCommand(object):
336 """A SubCommand template base class"""
337 name
= None # Subcommand name
339 def shared_args(self
, parser
):
340 parser
.add_argument("--quiet", action
="store_true",
341 help="Run quietly unless an error occurred")
343 def args(self
, parser
):
344 """Setup argument parser"""
347 def run(self
, args
, argv
):
349 args: parsed argument by argument parser.
350 argv: remaining arguments from sys.argv.
355 class RunCommand(SubCommand
):
356 """Invoke docker run and take care of cleaning up"""
359 def args(self
, parser
):
360 parser
.add_argument("--keep", action
="store_true",
361 help="Don't remove image when command completes")
362 parser
.add_argument("--run-as-current-user", action
="store_true",
363 help="Run container using the current user's uid")
365 def run(self
, args
, argv
):
366 if args
.run_as_current_user
:
368 argv
= [ "-u", str(uid
) ] + argv
370 if docker
._command
[0] == "podman":
371 argv
= [ "--uidmap", "%d:0:1" % uid
,
372 "--uidmap", "0:1:%d" % uid
,
373 "--uidmap", "%d:%d:64536" % (uid
+ 1, uid
+ 1)] + argv
374 return Docker().run(argv
, args
.keep
, quiet
=args
.quiet
)
377 class BuildCommand(SubCommand
):
378 """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
381 def args(self
, parser
):
382 parser
.add_argument("--include-executable", "-e",
383 help="""Specify a binary that will be copied to the
384 container together with all its dependent
386 parser
.add_argument("--extra-files", "-f", nargs
='*',
387 help="""Specify files that will be copied in the
388 Docker image, fulfilling the ADD directive from the
390 parser
.add_argument("--add-current-user", "-u", dest
="user",
392 help="Add the current user to image's passwd")
393 parser
.add_argument("tag",
395 parser
.add_argument("dockerfile",
396 help="Dockerfile name")
398 def run(self
, args
, argv
):
399 dockerfile
= open(args
.dockerfile
, "rb").read()
403 if "--no-cache" not in argv
and \
404 dkr
.image_matches_dockerfile(tag
, dockerfile
):
406 print("Image is up to date.")
408 # Create a docker context directory for the build
409 docker_dir
= tempfile
.mkdtemp(prefix
="docker_build")
411 # Validate binfmt_misc will work
412 if args
.include_executable
:
413 qpath
, enabled
= _check_binfmt_misc(args
.include_executable
)
417 # Is there a .pre file to run in the build context?
418 docker_pre
= os
.path
.splitext(args
.dockerfile
)[0]+".pre"
419 if os
.path
.exists(docker_pre
):
420 stdout
= DEVNULL
if args
.quiet
else None
421 rc
= subprocess
.call(os
.path
.realpath(docker_pre
),
422 cwd
=docker_dir
, stdout
=stdout
)
427 print("%s exited with code %d" % (docker_pre
, rc
))
430 # Copy any extra files into the Docker context. These can be
431 # included by the use of the ADD directive in the Dockerfile.
433 if args
.include_executable
:
434 # FIXME: there is no checksum of this executable and the linked
435 # libraries, once the image built any change of this executable
436 # or any library won't trigger another build.
437 _copy_binary_with_libs(args
.include_executable
,
440 for filename
in args
.extra_files
or []:
441 _copy_with_mkdir(filename
, docker_dir
)
442 cksum
+= [(filename
, _file_checksum(filename
))]
444 argv
+= ["--build-arg=" + k
.lower() + "=" + v
445 for k
, v
in os
.environ
.iteritems()
446 if k
.lower() in FILTERED_ENV_NAMES
]
447 dkr
.build_image(tag
, docker_dir
, dockerfile
,
448 quiet
=args
.quiet
, user
=args
.user
, argv
=argv
,
449 extra_files_cksum
=cksum
)
456 class UpdateCommand(SubCommand
):
457 """ Update a docker image with new executables. Args: <tag> <executable>"""
460 def args(self
, parser
):
461 parser
.add_argument("tag",
463 parser
.add_argument("executable",
464 help="Executable to copy")
466 def run(self
, args
, argv
):
467 # Create a temporary tarball with our whole build context and
468 # dockerfile for the update
469 tmp
= tempfile
.NamedTemporaryFile(suffix
="dckr.tar.gz")
470 tmp_tar
= TarFile(fileobj
=tmp
, mode
='w')
472 # Add the executable to the tarball, using the current
473 # configured binfmt_misc path. If we don't get a path then we
474 # only need the support libraries copied
475 ff
, enabled
= _check_binfmt_misc(args
.executable
)
478 print("binfmt_misc not enabled, update disabled")
482 tmp_tar
.add(args
.executable
, arcname
=ff
)
484 # Add any associated libraries
485 libs
= _get_so_libs(args
.executable
)
488 tmp_tar
.add(os
.path
.realpath(l
), arcname
=l
)
490 # Create a Docker buildfile
492 df
.write("FROM %s\n" % args
.tag
)
493 df
.write("ADD . /\n")
496 df_tar
= TarInfo(name
="Dockerfile")
497 df_tar
.size
= len(df
.buf
)
498 tmp_tar
.addfile(df_tar
, fileobj
=df
)
502 # reset the file pointers
506 # Run the build with our tarball context
508 dkr
.update_image(args
.tag
, tmp
, quiet
=args
.quiet
)
513 class CleanCommand(SubCommand
):
514 """Clean up docker instances"""
517 def run(self
, args
, argv
):
522 class ImagesCommand(SubCommand
):
523 """Run "docker images" command"""
526 def run(self
, args
, argv
):
527 return Docker().command("images", argv
, args
.quiet
)
530 class ProbeCommand(SubCommand
):
531 """Probe if we can run docker automatically"""
534 def run(self
, args
, argv
):
537 if docker
._command
[0] == "docker":
539 elif docker
._command
[0] == "sudo":
541 elif docker
._command
[0] == "podman":
549 class CcCommand(SubCommand
):
550 """Compile sources with cc in images"""
553 def args(self
, parser
):
554 parser
.add_argument("--image", "-i", required
=True,
555 help="The docker image in which to run cc")
556 parser
.add_argument("--cc", default
="cc",
557 help="The compiler executable to call")
558 parser
.add_argument("--user",
559 help="The user-id to run under")
560 parser
.add_argument("--source-path", "-s", nargs
="*", dest
="paths",
561 help="""Extra paths to (ro) mount into container for
564 def run(self
, args
, argv
):
565 if argv
and argv
[0] == "--":
568 cmd
= ["--rm", "-w", cwd
,
569 "-v", "%s:%s:rw" % (cwd
, cwd
)]
572 cmd
+= ["-v", "%s:%s:ro,z" % (p
, p
)]
574 cmd
+= ["-u", args
.user
]
575 cmd
+= [args
.image
, args
.cc
]
577 return Docker().command("run", cmd
, args
.quiet
)
580 class CheckCommand(SubCommand
):
581 """Check if we need to re-build a docker image out of a dockerfile.
582 Arguments: <tag> <dockerfile>"""
585 def args(self
, parser
):
586 parser
.add_argument("tag",
588 parser
.add_argument("dockerfile", default
=None,
589 help="Dockerfile name", nargs
='?')
590 parser
.add_argument("--checktype", choices
=["checksum", "age"],
591 default
="checksum", help="check type")
592 parser
.add_argument("--olderthan", default
=60, type=int,
593 help="number of minutes")
595 def run(self
, args
, argv
):
600 except subprocess
.CalledProcessError
:
601 print("Docker not set up")
604 info
= dkr
.inspect_tag(tag
)
606 print("Image does not exist")
609 if args
.checktype
== "checksum":
610 if not args
.dockerfile
:
611 print("Need a dockerfile for tag:%s" % (tag
))
614 dockerfile
= open(args
.dockerfile
, "rb").read()
616 if dkr
.image_matches_dockerfile(tag
, dockerfile
):
618 print("Image is up to date")
621 print("Image needs updating")
623 elif args
.checktype
== "age":
624 timestr
= dkr
.get_image_creation_time(info
).split(".")[0]
625 created
= datetime
.strptime(timestr
, "%Y-%m-%dT%H:%M:%S")
626 past
= datetime
.now() - timedelta(minutes
=args
.olderthan
)
628 print ("Image created @ %s more than %d minutes old" %
629 (timestr
, args
.olderthan
))
633 print ("Image less than %d minutes old" % (args
.olderthan
))
640 parser
= argparse
.ArgumentParser(description
="A Docker helper",
641 usage
="%s <subcommand> ..." %
642 os
.path
.basename(sys
.argv
[0]))
643 parser
.add_argument("--engine", type=EngineEnum
.argparse
, choices
=list(EngineEnum
),
644 help="specify which container engine to use")
645 subparsers
= parser
.add_subparsers(title
="subcommands", help=None)
646 for cls
in SubCommand
.__subclasses
__():
648 subp
= subparsers
.add_parser(cmd
.name
, help=cmd
.__doc
__)
649 cmd
.shared_args(subp
)
651 subp
.set_defaults(cmdobj
=cmd
)
652 args
, argv
= parser
.parse_known_args()
653 USE_ENGINE
= args
.engine
654 return args
.cmdobj
.run(args
, argv
)
657 if __name__
== "__main__":