import tempfile
import re
import signal
+import getpass
from tarfile import TarFile, TarInfo
-from io import StringIO
+from io import StringIO, BytesIO
from shutil import copy, rmtree
-from pwd import getpwuid
from datetime import datetime, timedelta
USE_ENGINE = EngineEnum.AUTO
+def _bytes_checksum(bytes):
+ """Calculate a digest string unique to the text content"""
+ return hashlib.sha1(bytes).hexdigest()
+
def _text_checksum(text):
"""Calculate a digest string unique to the text content"""
- return hashlib.sha1(text.encode('utf-8')).hexdigest()
+ return _bytes_checksum(text.encode('utf-8'))
def _read_dockerfile(path):
return open(path, 'rt', encoding='utf-8').read()
def _file_checksum(filename):
- return _text_checksum(_read_dockerfile(filename))
+ return _bytes_checksum(open(filename, 'rb').read())
def _guess_engine_command():
commands_txt)
-def _copy_with_mkdir(src, root_dir, sub_path='.'):
+def _copy_with_mkdir(src, root_dir, sub_path='.', name=None):
"""Copy src into root_dir, creating sub_path as needed."""
dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
try:
# we can safely ignore already created directories
pass
- dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
- copy(src, dest_file)
+ dest_file = "%s/%s" % (dest_dir, name if name else os.path.basename(src))
+
+ try:
+ copy(src, dest_file)
+ except FileNotFoundError:
+ print("Couldn't copy %s to %s" % (src, dest_file))
+ pass
def _get_so_libs(executable):
"""Return a list of libraries associated with an executable.
The paths may be symbolic links which would need to be resolved to
- ensure theright data is copied."""
+ ensure the right data is copied."""
libs = []
- ldd_re = re.compile(r"(/.*/)(\S*)")
+ ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
try:
- ldd_output = subprocess.check_output(["ldd", executable])
+ ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
for line in ldd_output.split("\n"):
search = ldd_re.search(line)
- if search and len(search.groups()) == 2:
- so_path = search.groups()[0]
- so_lib = search.groups()[1]
- libs.append("%s/%s" % (so_path, so_lib))
+ if search:
+ try:
+ libs.append(search.group(1))
+ except IndexError:
+ pass
except subprocess.CalledProcessError:
print("%s had no associated libraries (static build?)" % (executable))
if libs:
for l in libs:
so_path = os.path.dirname(l)
- _copy_with_mkdir(l, dest_dir, so_path)
+ name = os.path.basename(l)
+ real_l = os.path.realpath(l)
+ _copy_with_mkdir(real_l, dest_dir, so_path, name)
def _check_binfmt_misc(executable):
return _read_dockerfile(df)
-def _dockerfile_preprocess(df):
- out = ""
+def _dockerfile_verify_flat(df):
+ "Verify we do not include other qemu/ layers"
for l in df.splitlines():
if len(l.strip()) == 0 or l.startswith("#"):
continue
- from_pref = "FROM qemu:"
+ from_pref = "FROM qemu/"
if l.startswith(from_pref):
- # TODO: Alternatively we could replace this line with "FROM $ID"
- # where $ID is the image's hex id obtained with
- # $ docker images $IMAGE --format="{{.Id}}"
- # but unfortunately that's not supported by RHEL 7.
- inlining = _read_qemu_dockerfile(l[len(from_pref):])
- out += _dockerfile_preprocess(inlining)
- continue
- out += l + "\n"
- return out
+ print("We no longer support multiple QEMU layers.")
+ print("Dockerfiles should be flat, ideally created by lcitool")
+ return False
+ return True
class Docker(object):
""" Running Docker commands """
def __init__(self):
self._command = _guess_engine_command()
- self._instances = []
+
+ if ("docker" in self._command and
+ "TRAVIS" not in os.environ and
+ "GITLAB_CI" not in os.environ):
+ os.environ["DOCKER_BUILDKIT"] = "1"
+ self._buildkit = True
+ else:
+ self._buildkit = False
+
+ self._instance = None
atexit.register(self._kill_instances)
signal.signal(signal.SIGTERM, self._kill_instances)
signal.signal(signal.SIGHUP, self._kill_instances)
cmd = ["ps", "-q"]
if not only_active:
cmd.append("-a")
+
+ filter = "--filter=label=com.qemu.instance.uuid"
+ if only_known:
+ if self._instance:
+ filter += "=%s" % (self._instance)
+ else:
+ # no point trying to kill, we finished
+ return
+
+ print("filter=%s" % (filter))
+ cmd.append(filter)
for i in self._output(cmd).split():
- resp = self._output(["inspect", i])
- labels = json.loads(resp)[0]["Config"]["Labels"]
- active = json.loads(resp)[0]["State"]["Running"]
- if not labels:
- continue
- instance_uuid = labels.get("com.qemu.instance.uuid", None)
- if not instance_uuid:
- continue
- if only_known and instance_uuid not in self._instances:
- continue
- print("Terminating", i)
- if active:
- self._do(["kill", i])
- self._do(["rm", i])
+ self._do(["rm", "-f", i])
def clean(self):
self._do_kill_instances(False, False)
return self._do_kill_instances(True)
def _output(self, cmd, **kwargs):
- return subprocess.check_output(self._command + cmd,
- stderr=subprocess.STDOUT,
- encoding='utf-8',
- **kwargs)
+ try:
+ return subprocess.check_output(self._command + cmd,
+ stderr=subprocess.STDOUT,
+ encoding='utf-8',
+ **kwargs)
+ except TypeError:
+ # 'encoding' argument was added in 3.6+
+ return subprocess.check_output(self._command + cmd,
+ stderr=subprocess.STDOUT,
+ **kwargs).decode('utf-8')
+
def inspect_tag(self, tag):
try:
return labels.get("com.qemu.dockerfile-checksum", "")
def build_image(self, tag, docker_dir, dockerfile,
- quiet=True, user=False, argv=None, extra_files_cksum=[]):
+ quiet=True, user=False, argv=None, registry=None,
+ extra_files_cksum=[]):
if argv is None:
argv = []
+ if not _dockerfile_verify_flat(dockerfile):
+ return -1
+
+ checksum = _text_checksum(dockerfile)
+
tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
encoding='utf-8',
dir=docker_dir, suffix=".docker")
if user:
uid = os.getuid()
- uname = getpwuid(uid).pw_name
+ uname = getpass.getuser()
tmp_df.write("\n")
tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
(uname, uid, uname))
tmp_df.write("\n")
- tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
- _text_checksum(_dockerfile_preprocess(dockerfile)))
+ tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s\n" % (checksum))
for f, c in extra_files_cksum:
- tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
+ tmp_df.write("LABEL com.qemu.%s-checksum=%s\n" % (f, c))
tmp_df.flush()
- self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv +
- [docker_dir],
+ build_args = ["build", "-t", tag, "-f", tmp_df.name]
+ if self._buildkit:
+ build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
+
+ if registry is not None:
+ pull_args = ["pull", "%s/%s" % (registry, tag)]
+ self._do(pull_args, quiet=quiet)
+ cache = "%s/%s" % (registry, tag)
+ build_args += ["--cache-from", cache]
+ build_args += argv
+ build_args += [docker_dir]
+
+ self._do_check(build_args,
quiet=quiet)
def update_image(self, tag, tarball, quiet=True):
checksum = self.get_image_dockerfile_checksum(tag)
except Exception:
return False
- return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
+ return checksum == _text_checksum(dockerfile)
- def run(self, cmd, keep, quiet):
- label = uuid.uuid1().hex
+ def run(self, cmd, keep, quiet, as_user=False):
+ label = uuid.uuid4().hex
if not keep:
- self._instances.append(label)
- ret = self._do_check(["run", "--label",
+ self._instance = label
+
+ if as_user:
+ uid = os.getuid()
+ cmd = [ "-u", str(uid) ] + cmd
+ # podman requires a bit more fiddling
+ if self._command[0] == "podman":
+ cmd.insert(0, '--userns=keep-id')
+
+ ret = self._do_check(["run", "--rm", "--label",
"com.qemu.instance.uuid=" + label] + cmd,
quiet=quiet)
if not keep:
- self._instances.remove(label)
+ self._instance = None
return ret
def command(self, cmd, argv, quiet):
help="Run container using the current user's uid")
def run(self, args, argv):
- if args.run_as_current_user:
- uid = os.getuid()
- argv = [ "-u", str(uid) ] + argv
- docker = Docker()
- if docker._command[0] == "podman":
- argv = [ "--uidmap", "%d:0:1" % uid,
- "--uidmap", "0:1:%d" % uid,
- "--uidmap", "%d:%d:64536" % (uid + 1, uid + 1)] + argv
- return Docker().run(argv, args.keep, quiet=args.quiet)
+ return Docker().run(argv, args.keep, quiet=args.quiet,
+ as_user=args.run_as_current_user)
class BuildCommand(SubCommand):
help="""Specify a binary that will be copied to the
container together with all its dependent
libraries""")
- parser.add_argument("--extra-files", "-f", nargs='*',
+ parser.add_argument("--skip-binfmt",
+ action="store_true",
+ help="""Skip binfmt entry check (used for testing)""")
+ parser.add_argument("--extra-files", nargs='*',
help="""Specify files that will be copied in the
Docker image, fulfilling the ADD directive from the
Dockerfile""")
parser.add_argument("--add-current-user", "-u", dest="user",
action="store_true",
help="Add the current user to image's passwd")
- parser.add_argument("tag",
+ parser.add_argument("--registry", "-r",
+ help="cache from docker registry")
+ parser.add_argument("-t", dest="tag",
help="Image Tag")
- parser.add_argument("dockerfile",
+ parser.add_argument("-f", dest="dockerfile",
help="Dockerfile name")
def run(self, args, argv):
docker_dir = tempfile.mkdtemp(prefix="docker_build")
# Validate binfmt_misc will work
- if args.include_executable:
+ if args.skip_binfmt:
+ qpath = args.include_executable
+ elif args.include_executable:
qpath, enabled = _check_binfmt_misc(args.include_executable)
if not enabled:
return 1
for k, v in os.environ.items()
if k.lower() in FILTERED_ENV_NAMES]
dkr.build_image(tag, docker_dir, dockerfile,
- quiet=args.quiet, user=args.user, argv=argv,
+ quiet=args.quiet, user=args.user,
+ argv=argv, registry=args.registry,
extra_files_cksum=cksum)
rmtree(docker_dir)
return 0
+class FetchCommand(SubCommand):
+ """ Fetch a docker image from the registry. Args: <tag> <registry>"""
+ name = "fetch"
+
+ def args(self, parser):
+ parser.add_argument("tag",
+ help="Local tag for image")
+ parser.add_argument("registry",
+ help="Docker registry")
+
+ def run(self, args, argv):
+ dkr = Docker()
+ dkr.command(cmd="pull", quiet=args.quiet,
+ argv=["%s/%s" % (args.registry, args.tag)])
+ dkr.command(cmd="tag", quiet=args.quiet,
+ argv=["%s/%s" % (args.registry, args.tag), args.tag])
+
class UpdateCommand(SubCommand):
- """ Update a docker image with new executables. Args: <tag> <executable>"""
+ """ Update a docker image. Args: <tag> <actions>"""
name = "update"
def args(self, parser):
parser.add_argument("tag",
help="Image Tag")
- parser.add_argument("executable",
+ parser.add_argument("--executable",
help="Executable to copy")
+ parser.add_argument("--add-current-user", "-u", dest="user",
+ action="store_true",
+ help="Add the current user to image's passwd")
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
- # Add the executable to the tarball, using the current
- # configured binfmt_misc path. If we don't get a path then we
- # only need the support libraries copied
- ff, enabled = _check_binfmt_misc(args.executable)
+ # Create a Docker buildfile
+ df = StringIO()
+ df.write(u"FROM %s\n" % args.tag)
- if not enabled:
- print("binfmt_misc not enabled, update disabled")
- return 1
+ if args.executable:
+ # Add the executable to the tarball, using the current
+ # configured binfmt_misc path. If we don't get a path then we
+ # only need the support libraries copied
+ ff, enabled = _check_binfmt_misc(args.executable)
- if ff:
- tmp_tar.add(args.executable, arcname=ff)
+ if not enabled:
+ print("binfmt_misc not enabled, update disabled")
+ return 1
- # Add any associated libraries
- libs = _get_so_libs(args.executable)
- if libs:
- for l in libs:
- tmp_tar.add(os.path.realpath(l), arcname=l)
+ if ff:
+ tmp_tar.add(args.executable, arcname=ff)
- # Create a Docker buildfile
- df = StringIO()
- df.write("FROM %s\n" % args.tag)
- df.write("ADD . /\n")
- df.seek(0)
+ # Add any associated libraries
+ libs = _get_so_libs(args.executable)
+ if libs:
+ for l in libs:
+ so_path = os.path.dirname(l)
+ name = os.path.basename(l)
+ real_l = os.path.realpath(l)
+ try:
+ tmp_tar.add(real_l, arcname="%s/%s" % (so_path, name))
+ except FileNotFoundError:
+ print("Couldn't add %s/%s to archive" % (so_path, name))
+ pass
+
+ df.write(u"ADD . /\n")
+
+ if args.user:
+ uid = os.getuid()
+ uname = getpass.getuser()
+ df.write("\n")
+ df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
+ (uname, uid, uname))
+
+ df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
df_tar = TarInfo(name="Dockerfile")
- df_tar.size = len(df.buf)
- tmp_tar.addfile(df_tar, fileobj=df)
+ df_tar.size = df_bytes.getbuffer().nbytes
+ tmp_tar.addfile(df_tar, fileobj=df_bytes)
tmp_tar.close()
help="The docker image in which to run cc")
parser.add_argument("--cc", default="cc",
help="The compiler executable to call")
- parser.add_argument("--user",
- help="The user-id to run under")
parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
help="""Extra paths to (ro) mount into container for
reading sources""")
if argv and argv[0] == "--":
argv = argv[1:]
cwd = os.getcwd()
- cmd = ["--rm", "-w", cwd,
+ cmd = ["-w", cwd,
"-v", "%s:%s:rw" % (cwd, cwd)]
if args.paths:
for p in args.paths:
cmd += ["-v", "%s:%s:ro,z" % (p, p)]
- if args.user:
- cmd += ["-u", args.user]
cmd += [args.image, args.cc]
cmd += argv
- return Docker().command("run", cmd, args.quiet)
-
-
-class CheckCommand(SubCommand):
- """Check if we need to re-build a docker image out of a dockerfile.
- Arguments: <tag> <dockerfile>"""
- name = "check"
-
- def args(self, parser):
- parser.add_argument("tag",
- help="Image Tag")
- parser.add_argument("dockerfile", default=None,
- help="Dockerfile name", nargs='?')
- parser.add_argument("--checktype", choices=["checksum", "age"],
- default="checksum", help="check type")
- parser.add_argument("--olderthan", default=60, type=int,
- help="number of minutes")
-
- def run(self, args, argv):
- tag = args.tag
-
- try:
- dkr = Docker()
- except subprocess.CalledProcessError:
- print("Docker not set up")
- return 1
-
- info = dkr.inspect_tag(tag)
- if info is None:
- print("Image does not exist")
- return 1
-
- if args.checktype == "checksum":
- if not args.dockerfile:
- print("Need a dockerfile for tag:%s" % (tag))
- return 1
-
- dockerfile = _read_dockerfile(args.dockerfile)
-
- if dkr.image_matches_dockerfile(tag, dockerfile):
- if not args.quiet:
- print("Image is up to date")
- return 0
- else:
- print("Image needs updating")
- return 1
- elif args.checktype == "age":
- timestr = dkr.get_image_creation_time(info).split(".")[0]
- created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
- past = datetime.now() - timedelta(minutes=args.olderthan)
- if created < past:
- print ("Image created @ %s more than %d minutes old" %
- (timestr, args.olderthan))
- return 1
- else:
- if not args.quiet:
- print ("Image less than %d minutes old" % (args.olderthan))
- return 0
+ return Docker().run(cmd, False, quiet=args.quiet,
+ as_user=True)
def main():
projects / mirror_qemu.git / blobdiff