]>
Commit | Line | Data |
---|---|---|
4485b04b FZ |
1 | #!/usr/bin/env python2 |
2 | # | |
3 | # Docker controlling module | |
4 | # | |
5 | # Copyright (c) 2016 Red Hat Inc. | |
6 | # | |
7 | # Authors: | |
8 | # Fam Zheng <famz@redhat.com> | |
9 | # | |
10 | # This work is licensed under the terms of the GNU GPL, version 2 | |
11 | # or (at your option) any later version. See the COPYING file in | |
12 | # the top-level directory. | |
13 | ||
f03868bd | 14 | from __future__ import print_function |
4485b04b FZ |
15 | import os |
16 | import sys | |
c2d31896 SH |
17 | sys.path.append(os.path.join(os.path.dirname(__file__), |
18 | '..', '..', 'scripts')) | |
19 | import argparse | |
4485b04b FZ |
20 | import subprocess |
21 | import json | |
22 | import hashlib | |
23 | import atexit | |
24 | import uuid | |
4485b04b | 25 | import tempfile |
504ca3c2 | 26 | import re |
97cba1a1 | 27 | import signal |
6e733da6 AB |
28 | from tarfile import TarFile, TarInfo |
29 | from StringIO import StringIO | |
a9f8d038 | 30 | from shutil import copy, rmtree |
414a8ce5 | 31 | from pwd import getpwuid |
4485b04b | 32 | |
c9772570 | 33 | |
06cc3551 PMD |
34 | FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] |
35 | ||
36 | ||
c9772570 SS |
37 | DEVNULL = open(os.devnull, 'wb') |
38 | ||
39 | ||
4485b04b FZ |
40 | def _text_checksum(text): |
41 | """Calculate a digest string unique to the text content""" | |
42 | return hashlib.sha1(text).hexdigest() | |
43 | ||
438d1168 PMD |
44 | def _file_checksum(filename): |
45 | return _text_checksum(open(filename, 'rb').read()) | |
46 | ||
4485b04b FZ |
47 | def _guess_docker_command(): |
48 | """ Guess a working docker command or raise exception if not found""" | |
49 | commands = [["docker"], ["sudo", "-n", "docker"]] | |
50 | for cmd in commands: | |
0679f98b EH |
51 | try: |
52 | if subprocess.call(cmd + ["images"], | |
53 | stdout=DEVNULL, stderr=DEVNULL) == 0: | |
54 | return cmd | |
55 | except OSError: | |
56 | pass | |
4485b04b FZ |
57 | commands_txt = "\n".join([" " + " ".join(x) for x in commands]) |
58 | raise Exception("Cannot find working docker command. Tried:\n%s" % \ | |
59 | commands_txt) | |
60 | ||
2499ee9f | 61 | def _copy_with_mkdir(src, root_dir, sub_path='.'): |
504ca3c2 AB |
62 | """Copy src into root_dir, creating sub_path as needed.""" |
63 | dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path)) | |
64 | try: | |
65 | os.makedirs(dest_dir) | |
66 | except OSError: | |
67 | # we can safely ignore already created directories | |
68 | pass | |
69 | ||
70 | dest_file = "%s/%s" % (dest_dir, os.path.basename(src)) | |
71 | copy(src, dest_file) | |
72 | ||
73 | ||
74 | def _get_so_libs(executable): | |
75 | """Return a list of libraries associated with an executable. | |
76 | ||
77 | The paths may be symbolic links which would need to be resolved to | |
78 | ensure theright data is copied.""" | |
79 | ||
80 | libs = [] | |
81 | ldd_re = re.compile(r"(/.*/)(\S*)") | |
82 | try: | |
83 | ldd_output = subprocess.check_output(["ldd", executable]) | |
84 | for line in ldd_output.split("\n"): | |
85 | search = ldd_re.search(line) | |
86 | if search and len(search.groups()) == 2: | |
87 | so_path = search.groups()[0] | |
88 | so_lib = search.groups()[1] | |
89 | libs.append("%s/%s" % (so_path, so_lib)) | |
90 | except subprocess.CalledProcessError: | |
f03868bd | 91 | print("%s had no associated libraries (static build?)" % (executable)) |
504ca3c2 AB |
92 | |
93 | return libs | |
94 | ||
95 | def _copy_binary_with_libs(src, dest_dir): | |
96 | """Copy a binary executable and all its dependant libraries. | |
97 | ||
98 | This does rely on the host file-system being fairly multi-arch | |
99 | aware so the file don't clash with the guests layout.""" | |
100 | ||
101 | _copy_with_mkdir(src, dest_dir, "/usr/bin") | |
102 | ||
103 | libs = _get_so_libs(src) | |
104 | if libs: | |
105 | for l in libs: | |
106 | so_path = os.path.dirname(l) | |
107 | _copy_with_mkdir(l , dest_dir, so_path) | |
108 | ||
c1958e9d FZ |
109 | def _read_qemu_dockerfile(img_name): |
110 | df = os.path.join(os.path.dirname(__file__), "dockerfiles", | |
111 | img_name + ".docker") | |
112 | return open(df, "r").read() | |
113 | ||
114 | def _dockerfile_preprocess(df): | |
115 | out = "" | |
116 | for l in df.splitlines(): | |
117 | if len(l.strip()) == 0 or l.startswith("#"): | |
118 | continue | |
119 | from_pref = "FROM qemu:" | |
120 | if l.startswith(from_pref): | |
121 | # TODO: Alternatively we could replace this line with "FROM $ID" | |
122 | # where $ID is the image's hex id obtained with | |
123 | # $ docker images $IMAGE --format="{{.Id}}" | |
124 | # but unfortunately that's not supported by RHEL 7. | |
125 | inlining = _read_qemu_dockerfile(l[len(from_pref):]) | |
126 | out += _dockerfile_preprocess(inlining) | |
127 | continue | |
128 | out += l + "\n" | |
129 | return out | |
130 | ||
4485b04b FZ |
131 | class Docker(object): |
132 | """ Running Docker commands """ | |
133 | def __init__(self): | |
134 | self._command = _guess_docker_command() | |
135 | self._instances = [] | |
136 | atexit.register(self._kill_instances) | |
97cba1a1 FZ |
137 | signal.signal(signal.SIGTERM, self._kill_instances) |
138 | signal.signal(signal.SIGHUP, self._kill_instances) | |
4485b04b | 139 | |
58bf7b6d | 140 | def _do(self, cmd, quiet=True, **kwargs): |
4485b04b | 141 | if quiet: |
c9772570 | 142 | kwargs["stdout"] = DEVNULL |
4485b04b FZ |
143 | return subprocess.call(self._command + cmd, **kwargs) |
144 | ||
0b95ff72 FZ |
145 | def _do_check(self, cmd, quiet=True, **kwargs): |
146 | if quiet: | |
147 | kwargs["stdout"] = DEVNULL | |
148 | return subprocess.check_call(self._command + cmd, **kwargs) | |
149 | ||
4485b04b FZ |
150 | def _do_kill_instances(self, only_known, only_active=True): |
151 | cmd = ["ps", "-q"] | |
152 | if not only_active: | |
153 | cmd.append("-a") | |
154 | for i in self._output(cmd).split(): | |
155 | resp = self._output(["inspect", i]) | |
156 | labels = json.loads(resp)[0]["Config"]["Labels"] | |
157 | active = json.loads(resp)[0]["State"]["Running"] | |
158 | if not labels: | |
159 | continue | |
160 | instance_uuid = labels.get("com.qemu.instance.uuid", None) | |
161 | if not instance_uuid: | |
162 | continue | |
163 | if only_known and instance_uuid not in self._instances: | |
164 | continue | |
f03868bd | 165 | print("Terminating", i) |
4485b04b FZ |
166 | if active: |
167 | self._do(["kill", i]) | |
168 | self._do(["rm", i]) | |
169 | ||
170 | def clean(self): | |
171 | self._do_kill_instances(False, False) | |
172 | return 0 | |
173 | ||
97cba1a1 | 174 | def _kill_instances(self, *args, **kwargs): |
4485b04b FZ |
175 | return self._do_kill_instances(True) |
176 | ||
177 | def _output(self, cmd, **kwargs): | |
178 | return subprocess.check_output(self._command + cmd, | |
179 | stderr=subprocess.STDOUT, | |
180 | **kwargs) | |
181 | ||
182 | def get_image_dockerfile_checksum(self, tag): | |
183 | resp = self._output(["inspect", tag]) | |
184 | labels = json.loads(resp)[0]["Config"].get("Labels", {}) | |
185 | return labels.get("com.qemu.dockerfile-checksum", "") | |
186 | ||
414a8ce5 | 187 | def build_image(self, tag, docker_dir, dockerfile, |
438d1168 | 188 | quiet=True, user=False, argv=None, extra_files_cksum=[]): |
4485b04b FZ |
189 | if argv == None: |
190 | argv = [] | |
4485b04b | 191 | |
a9f8d038 | 192 | tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker") |
4485b04b FZ |
193 | tmp_df.write(dockerfile) |
194 | ||
414a8ce5 AB |
195 | if user: |
196 | uid = os.getuid() | |
197 | uname = getpwuid(uid).pw_name | |
198 | tmp_df.write("\n") | |
199 | tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" % | |
200 | (uname, uid, uname)) | |
201 | ||
4485b04b FZ |
202 | tmp_df.write("\n") |
203 | tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % | |
438d1168 PMD |
204 | _text_checksum("\n".join([dockerfile] + |
205 | extra_files_cksum))) | |
4485b04b | 206 | tmp_df.flush() |
a9f8d038 | 207 | |
0b95ff72 FZ |
208 | self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \ |
209 | [docker_dir], | |
210 | quiet=quiet) | |
4485b04b | 211 | |
6e733da6 AB |
212 | def update_image(self, tag, tarball, quiet=True): |
213 | "Update a tagged image using " | |
214 | ||
0b95ff72 | 215 | self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball) |
6e733da6 | 216 | |
4485b04b FZ |
217 | def image_matches_dockerfile(self, tag, dockerfile): |
218 | try: | |
219 | checksum = self.get_image_dockerfile_checksum(tag) | |
220 | except Exception: | |
221 | return False | |
c1958e9d | 222 | return checksum == _text_checksum(_dockerfile_preprocess(dockerfile)) |
4485b04b FZ |
223 | |
224 | def run(self, cmd, keep, quiet): | |
225 | label = uuid.uuid1().hex | |
226 | if not keep: | |
227 | self._instances.append(label) | |
0b95ff72 FZ |
228 | ret = self._do_check(["run", "--label", |
229 | "com.qemu.instance.uuid=" + label] + cmd, | |
230 | quiet=quiet) | |
4485b04b FZ |
231 | if not keep: |
232 | self._instances.remove(label) | |
233 | return ret | |
234 | ||
4b08af60 FZ |
235 | def command(self, cmd, argv, quiet): |
236 | return self._do([cmd] + argv, quiet=quiet) | |
237 | ||
4485b04b FZ |
238 | class SubCommand(object): |
239 | """A SubCommand template base class""" | |
240 | name = None # Subcommand name | |
241 | def shared_args(self, parser): | |
242 | parser.add_argument("--quiet", action="store_true", | |
243 | help="Run quietly unless an error occured") | |
244 | ||
245 | def args(self, parser): | |
246 | """Setup argument parser""" | |
247 | pass | |
248 | def run(self, args, argv): | |
249 | """Run command. | |
250 | args: parsed argument by argument parser. | |
251 | argv: remaining arguments from sys.argv. | |
252 | """ | |
253 | pass | |
254 | ||
255 | class RunCommand(SubCommand): | |
256 | """Invoke docker run and take care of cleaning up""" | |
257 | name = "run" | |
258 | def args(self, parser): | |
259 | parser.add_argument("--keep", action="store_true", | |
260 | help="Don't remove image when command completes") | |
261 | def run(self, args, argv): | |
262 | return Docker().run(argv, args.keep, quiet=args.quiet) | |
263 | ||
264 | class BuildCommand(SubCommand): | |
265 | """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>""" | |
266 | name = "build" | |
267 | def args(self, parser): | |
504ca3c2 AB |
268 | parser.add_argument("--include-executable", "-e", |
269 | help="""Specify a binary that will be copied to the | |
270 | container together with all its dependent | |
271 | libraries""") | |
4c84f662 PMD |
272 | parser.add_argument("--extra-files", "-f", nargs='*', |
273 | help="""Specify files that will be copied in the | |
274 | Docker image, fulfilling the ADD directive from the | |
275 | Dockerfile""") | |
414a8ce5 AB |
276 | parser.add_argument("--add-current-user", "-u", dest="user", |
277 | action="store_true", | |
278 | help="Add the current user to image's passwd") | |
4485b04b FZ |
279 | parser.add_argument("tag", |
280 | help="Image Tag") | |
281 | parser.add_argument("dockerfile", | |
282 | help="Dockerfile name") | |
283 | ||
284 | def run(self, args, argv): | |
285 | dockerfile = open(args.dockerfile, "rb").read() | |
286 | tag = args.tag | |
287 | ||
288 | dkr = Docker() | |
6fe3ae3f AB |
289 | if "--no-cache" not in argv and \ |
290 | dkr.image_matches_dockerfile(tag, dockerfile): | |
4485b04b | 291 | if not args.quiet: |
f03868bd | 292 | print("Image is up to date.") |
a9f8d038 AB |
293 | else: |
294 | # Create a docker context directory for the build | |
295 | docker_dir = tempfile.mkdtemp(prefix="docker_build") | |
296 | ||
920776ea AB |
297 | # Is there a .pre file to run in the build context? |
298 | docker_pre = os.path.splitext(args.dockerfile)[0]+".pre" | |
299 | if os.path.exists(docker_pre): | |
f8042dea | 300 | stdout = DEVNULL if args.quiet else None |
920776ea | 301 | rc = subprocess.call(os.path.realpath(docker_pre), |
f8042dea | 302 | cwd=docker_dir, stdout=stdout) |
920776ea | 303 | if rc == 3: |
f03868bd | 304 | print("Skip") |
920776ea AB |
305 | return 0 |
306 | elif rc != 0: | |
f03868bd | 307 | print("%s exited with code %d" % (docker_pre, rc)) |
920776ea AB |
308 | return 1 |
309 | ||
4c84f662 PMD |
310 | # Copy any extra files into the Docker context. These can be |
311 | # included by the use of the ADD directive in the Dockerfile. | |
438d1168 | 312 | cksum = [] |
504ca3c2 | 313 | if args.include_executable: |
438d1168 PMD |
314 | # FIXME: there is no checksum of this executable and the linked |
315 | # libraries, once the image built any change of this executable | |
316 | # or any library won't trigger another build. | |
4c84f662 PMD |
317 | _copy_binary_with_libs(args.include_executable, docker_dir) |
318 | for filename in args.extra_files or []: | |
319 | _copy_with_mkdir(filename, docker_dir) | |
438d1168 | 320 | cksum += [_file_checksum(filename)] |
504ca3c2 | 321 | |
06cc3551 PMD |
322 | argv += ["--build-arg=" + k.lower() + "=" + v |
323 | for k, v in os.environ.iteritems() | |
324 | if k.lower() in FILTERED_ENV_NAMES] | |
a9f8d038 | 325 | dkr.build_image(tag, docker_dir, dockerfile, |
438d1168 PMD |
326 | quiet=args.quiet, user=args.user, argv=argv, |
327 | extra_files_cksum=cksum) | |
a9f8d038 AB |
328 | |
329 | rmtree(docker_dir) | |
4485b04b | 330 | |
4485b04b FZ |
331 | return 0 |
332 | ||
6e733da6 AB |
333 | class UpdateCommand(SubCommand): |
334 | """ Update a docker image with new executables. Arguments: <tag> <executable>""" | |
335 | name = "update" | |
336 | def args(self, parser): | |
337 | parser.add_argument("tag", | |
338 | help="Image Tag") | |
339 | parser.add_argument("executable", | |
340 | help="Executable to copy") | |
341 | ||
342 | def run(self, args, argv): | |
343 | # Create a temporary tarball with our whole build context and | |
344 | # dockerfile for the update | |
345 | tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") | |
346 | tmp_tar = TarFile(fileobj=tmp, mode='w') | |
347 | ||
348 | # Add the executable to the tarball | |
349 | bn = os.path.basename(args.executable) | |
350 | ff = "/usr/bin/%s" % bn | |
351 | tmp_tar.add(args.executable, arcname=ff) | |
352 | ||
353 | # Add any associated libraries | |
354 | libs = _get_so_libs(args.executable) | |
355 | if libs: | |
356 | for l in libs: | |
357 | tmp_tar.add(os.path.realpath(l), arcname=l) | |
358 | ||
359 | # Create a Docker buildfile | |
360 | df = StringIO() | |
361 | df.write("FROM %s\n" % args.tag) | |
362 | df.write("ADD . /\n") | |
363 | df.seek(0) | |
364 | ||
365 | df_tar = TarInfo(name="Dockerfile") | |
366 | df_tar.size = len(df.buf) | |
367 | tmp_tar.addfile(df_tar, fileobj=df) | |
368 | ||
369 | tmp_tar.close() | |
370 | ||
371 | # reset the file pointers | |
372 | tmp.flush() | |
373 | tmp.seek(0) | |
374 | ||
375 | # Run the build with our tarball context | |
376 | dkr = Docker() | |
377 | dkr.update_image(args.tag, tmp, quiet=args.quiet) | |
378 | ||
379 | return 0 | |
380 | ||
4485b04b FZ |
381 | class CleanCommand(SubCommand): |
382 | """Clean up docker instances""" | |
383 | name = "clean" | |
384 | def run(self, args, argv): | |
385 | Docker().clean() | |
386 | return 0 | |
387 | ||
4b08af60 FZ |
388 | class ImagesCommand(SubCommand): |
389 | """Run "docker images" command""" | |
390 | name = "images" | |
391 | def run(self, args, argv): | |
392 | return Docker().command("images", argv, args.quiet) | |
393 | ||
15df9d37 AB |
394 | |
395 | class ProbeCommand(SubCommand): | |
396 | """Probe if we can run docker automatically""" | |
397 | name = "probe" | |
398 | ||
399 | def run(self, args, argv): | |
400 | try: | |
401 | docker = Docker() | |
402 | if docker._command[0] == "docker": | |
f03868bd | 403 | print("yes") |
15df9d37 | 404 | elif docker._command[0] == "sudo": |
f03868bd | 405 | print("sudo") |
15df9d37 | 406 | except Exception: |
f03868bd | 407 | print("no") |
15df9d37 AB |
408 | |
409 | return | |
410 | ||
411 | ||
4485b04b FZ |
412 | def main(): |
413 | parser = argparse.ArgumentParser(description="A Docker helper", | |
414 | usage="%s <subcommand> ..." % os.path.basename(sys.argv[0])) | |
415 | subparsers = parser.add_subparsers(title="subcommands", help=None) | |
416 | for cls in SubCommand.__subclasses__(): | |
417 | cmd = cls() | |
418 | subp = subparsers.add_parser(cmd.name, help=cmd.__doc__) | |
419 | cmd.shared_args(subp) | |
420 | cmd.args(subp) | |
421 | subp.set_defaults(cmdobj=cmd) | |
422 | args, argv = parser.parse_known_args() | |
423 | return args.cmdobj.run(args, argv) | |
424 | ||
425 | if __name__ == "__main__": | |
426 | sys.exit(main()) |