src/lxc/criu.c

   1 /*
   2  * lxc: linux Container library
   3  *
   4  * Copyright © 2014-2015 Canonical Ltd.
   5  *
   6  * Authors:
   7  * Tycho Andersen <tycho.andersen@canonical.com>
   8  *
   9  * This library is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License as published by the Free Software Foundation; either
  12  * version 2.1 of the License, or (at your option) any later version.
  13  *
  14  * This library is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with this library; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  22  */
  23 #define _GNU_SOURCE
  24 #include <inttypes.h>
  25 #include <linux/limits.h>
  26 #include <sched.h>
  27 #include <stdio.h>
  28 #include <stdlib.h>
  29 #include <string.h>
  30 #include <sys/mount.h>
  31 #include <sys/types.h>
  32 #include <sys/wait.h>
  33 #include <unistd.h>
  34
  35 #include "config.h"
  36
  37 #include "cgroup.h"
  38 #include "conf.h"
  39 #include "commands.h"
  40 #include "criu.h"
  41 #include "log.h"
  42 #include "lxc.h"
  43 #include "lxclock.h"
  44 #include "network.h"
  45 #include "storage.h"
  46 #include "utils.h"
  47
  48 #if IS_BIONIC
  49 #include <../include/lxcmntent.h>
  50 #else
  51 #include <mntent.h>
  52 #endif
  53
  54 #define CRIU_VERSION            "2.0"
  55
  56 #define CRIU_GITID_VERSION      "2.0"
  57 #define CRIU_GITID_PATCHLEVEL   0
  58
  59 #define CRIU_IN_FLIGHT_SUPPORT  "2.4"
  60 #define CRIU_EXTERNAL_NOT_VETH  "2.8"
  61
  62 lxc_log_define(lxc_criu, lxc);
  63
  64 struct criu_opts {
  65         /* the thing to hook to stdout and stderr for logging */
  66         int pipefd;
  67
  68         /* The type of criu invocation, one of "dump" or "restore" */
  69         char *action;
  70
  71         /* the user-provided migrate options relevant to this action */
  72         struct migrate_opts *user;
  73
  74         /* The container to dump */
  75         struct lxc_container *c;
  76
  77         /* dump: stop the container or not after dumping? */
  78         char tty_id[32]; /* the criu tty id for /dev/console, i.e. "tty[${rdev}:${dev}]" */
  79
  80         /* restore: the file to write the init process' pid into */
  81         struct lxc_handler *handler;
  82         int console_fd;
  83         /* The path that is bind mounted from /dev/console, if any. We don't
  84          * want to use `--ext-mount-map auto`'s result here because the pts
  85          * device may have a different path (e.g. if the pty number is
  86          * different) on the target host. NULL if lxc.console.path = "none".
  87          */
  88         char *console_name;
  89
  90         /* The detected version of criu */
  91         char *criu_version;
  92 };
  93
  94 static int load_tty_major_minor(char *directory, char *output, int len)
  95 {
  96         FILE *f;
  97         char path[PATH_MAX];
  98         int ret;
  99
 100         ret = snprintf(path, sizeof(path), "%s/tty.info", directory);
 101         if (ret < 0 || ret >= sizeof(path)) {
 102                 ERROR("snprintf'd too many chacters: %d", ret);
 103                 return -1;
 104         }
 105
 106         f = fopen(path, "r");
 107         if (!f) {
 108                 /* This means we're coming from a liblxc which didn't export
 109                  * the tty info. In this case they had to have lxc.console.path
 110                  * = * none, so there's no problem restoring.
 111                  */
 112                 if (errno == ENOENT)
 113                         return 0;
 114
 115                 SYSERROR("couldn't open %s", path);
 116                 return -1;
 117         }
 118
 119         if (!fgets(output, len, f)) {
 120                 fclose(f);
 121                 SYSERROR("couldn't read %s", path);
 122                 return -1;
 123         }
 124
 125         fclose(f);
 126         return 0;
 127 }
 128
 129 static int cmp_version(const char *v1, const char *v2)
 130 {
 131         int ret;
 132         int oct_v1[3], oct_v2[3];
 133
 134         memset(oct_v1, -1, sizeof(oct_v1));
 135         memset(oct_v2, -1, sizeof(oct_v2));
 136
 137         ret = sscanf(v1, "%d.%d.%d", &oct_v1[0], &oct_v1[1], &oct_v1[2]);
 138         if (ret < 1)
 139                 return -1;
 140
 141         ret = sscanf(v2, "%d.%d.%d", &oct_v2[0], &oct_v2[1], &oct_v2[2]);
 142         if (ret < 1)
 143                 return -1;
 144
 145         /* Major version is greater. */
 146         if (oct_v1[0] > oct_v2[0])
 147                 return 1;
 148
 149         if (oct_v1[0] < oct_v2[0])
 150                 return -1;
 151
 152         /* Minor number is greater.*/
 153         if (oct_v1[1] > oct_v2[1])
 154                 return 1;
 155
 156         if (oct_v1[1] < oct_v2[1])
 157                 return -1;
 158
 159         /* Patch number is greater. */
 160         if (oct_v1[2] > oct_v2[2])
 161                 return 1;
 162
 163         /* Patch numbers are equal. */
 164         if (oct_v1[2] == oct_v2[2])
 165                 return 0;
 166
 167         return -1;
 168 }
 169
 170 static void exec_criu(struct criu_opts *opts)
 171 {
 172         char **argv, log[PATH_MAX];
 173         int static_args = 23, argc = 0, i, ret;
 174         int netnr = 0;
 175         struct lxc_list *it;
 176         FILE *mnts;
 177         struct mntent mntent;
 178
 179         char buf[4096], tty_info[32];
 180         size_t pos;
 181
 182         /* If we are currently in a cgroup /foo/bar, and the container is in a
 183          * cgroup /lxc/foo, lxcfs will give us an ENOENT if some task in the
 184          * container has an open fd that points to one of the cgroup files
 185          * (systemd always opens its "root" cgroup). So, let's escape to the
 186          * /actual/ root cgroup so that lxcfs thinks criu has enough rights to
 187          * see all cgroups.
 188          */
 189         if (!cgroup_escape()) {
 190                 ERROR("failed to escape cgroups");
 191                 return;
 192         }
 193
 194         /* The command line always looks like:
 195          * criu $(action) --tcp-established --file-locks --link-remap \
 196          * --manage-cgroups=full --action-script foo.sh -D $(directory) \
 197          * -o $(directory)/$(action).log --ext-mount-map auto
 198          * --enable-external-sharing --enable-external-masters
 199          * --enable-fs hugetlbfs --enable-fs tracefs --ext-mount-map console:/dev/pts/n
 200          * +1 for final NULL */
 201
 202         if (strcmp(opts->action, "dump") == 0 || strcmp(opts->action, "pre-dump") == 0) {
 203                 /* -t pid --freeze-cgroup /lxc/ct */
 204                 static_args += 4;
 205
 206                 /* --prev-images-dir <path-to-directory-A-relative-to-B> */
 207                 if (opts->user->predump_dir)
 208                         static_args += 2;
 209
 210                 /* --page-server --address <address> --port <port> */
 211                 if (opts->user->pageserver_address && opts->user->pageserver_port)
 212                         static_args += 5;
 213
 214                 /* --leave-running (only for final dump) */
 215                 if (strcmp(opts->action, "dump") == 0 && !opts->user->stop)
 216                         static_args++;
 217
 218                 /* --external tty[88,4] */
 219                 if (opts->tty_id[0])
 220                         static_args += 2;
 221
 222                 /* --force-irmap */
 223                 if (!opts->user->preserves_inodes)
 224                         static_args++;
 225
 226                 /* --ghost-limit 1024 */
 227                 if (opts->user->ghost_limit)
 228                         static_args += 2;
 229         } else if (strcmp(opts->action, "restore") == 0) {
 230                 /* --root $(lxc_mount_point) --restore-detached
 231                  * --restore-sibling
 232                  * --lsm-profile apparmor:whatever
 233                  */
 234                 static_args += 6;
 235
 236                 tty_info[0] = 0;
 237                 if (load_tty_major_minor(opts->user->directory, tty_info, sizeof(tty_info)))
 238                         return;
 239
 240                 /* --inherit-fd fd[%d]:tty[%s] */
 241                 if (tty_info[0])
 242                         static_args += 2;
 243         } else {
 244                 return;
 245         }
 246
 247         if (cgroup_num_hierarchies() > 0)
 248                 static_args += 2 * cgroup_num_hierarchies();
 249
 250         if (opts->user->verbose)
 251                 static_args++;
 252
 253         if (opts->user->action_script)
 254                 static_args += 2;
 255
 256         static_args += 2 * lxc_list_len(&opts->c->lxc_conf->mount_list);
 257
 258         ret = snprintf(log, PATH_MAX, "%s/%s.log", opts->user->directory, opts->action);
 259         if (ret < 0 || ret >= PATH_MAX) {
 260                 ERROR("logfile name too long");
 261                 return;
 262         }
 263
 264         argv = malloc(static_args * sizeof(*argv));
 265         if (!argv)
 266                 return;
 267
 268         memset(argv, 0, static_args * sizeof(*argv));
 269
 270 #define DECLARE_ARG(arg)                                        \
 271         do {                                                    \
 272                 if (arg == NULL) {                              \
 273                         ERROR("Got NULL argument for criu");    \
 274                         goto err;                               \
 275                 }                                               \
 276                 argv[argc++] = strdup(arg);                     \
 277                 if (!argv[argc-1])                              \
 278                         goto err;                               \
 279         } while (0)
 280
 281         argv[argc++] = on_path("criu", NULL);
 282         if (!argv[argc-1]) {
 283                 ERROR("Couldn't find criu binary");
 284                 goto err;
 285         }
 286
 287         DECLARE_ARG(opts->action);
 288         DECLARE_ARG("--tcp-established");
 289         DECLARE_ARG("--file-locks");
 290         DECLARE_ARG("--link-remap");
 291         DECLARE_ARG("--manage-cgroups=full");
 292         DECLARE_ARG("--ext-mount-map");
 293         DECLARE_ARG("auto");
 294         DECLARE_ARG("--enable-external-sharing");
 295         DECLARE_ARG("--enable-external-masters");
 296         DECLARE_ARG("--enable-fs");
 297         DECLARE_ARG("hugetlbfs");
 298         DECLARE_ARG("--enable-fs");
 299         DECLARE_ARG("tracefs");
 300         DECLARE_ARG("-D");
 301         DECLARE_ARG(opts->user->directory);
 302         DECLARE_ARG("-o");
 303         DECLARE_ARG(log);
 304
 305         for (i = 0; i < cgroup_num_hierarchies(); i++) {
 306                 char **controllers = NULL, *fullname;
 307                 char *path, *tmp;
 308
 309                 if (!cgroup_get_hierarchies(i, &controllers)) {
 310                         ERROR("failed to get hierarchy %d", i);
 311                         goto err;
 312                 }
 313
 314                 /* if we are in a dump, we have to ask the monitor process what
 315                  * the right cgroup is. if this is a restore, we can just use
 316                  * the handler the restore task created.
 317                  */
 318                 if (!strcmp(opts->action, "dump") || !strcmp(opts->action, "pre-dump")) {
 319                         path = lxc_cmd_get_cgroup_path(opts->c->name, opts->c->config_path, controllers[0]);
 320                         if (!path) {
 321                                 ERROR("failed to get cgroup path for %s", controllers[0]);
 322                                 goto err;
 323                         }
 324                 } else {
 325                         const char *p;
 326
 327                         p = cgroup_get_cgroup(opts->handler, controllers[0]);
 328                         if (!p) {
 329                                 ERROR("failed to get cgroup path for %s", controllers[0]);
 330                                 goto err;
 331                         }
 332
 333                         path = strdup(p);
 334                         if (!path) {
 335                                 ERROR("strdup failed");
 336                                 goto err;
 337                         }
 338                 }
 339
 340                 tmp = lxc_deslashify(path);
 341                 if (!tmp) {
 342                         ERROR("Failed to remove extraneous slashes from \"%s\"",
 343                               path);
 344                         free(path);
 345                         goto err;
 346                 }
 347                 free(path);
 348                 path = tmp;
 349
 350                 fullname = lxc_string_join(",", (const char **) controllers, false);
 351                 if (!fullname) {
 352                         ERROR("failed to join controllers");
 353                         free(path);
 354                         goto err;
 355                 }
 356
 357                 ret = sprintf(buf, "%s:%s", fullname, path);
 358                 free(path);
 359                 free(fullname);
 360                 if (ret < 0 || ret >= sizeof(buf)) {
 361                         ERROR("sprintf of cgroup root arg failed");
 362                         goto err;
 363                 }
 364
 365                 DECLARE_ARG("--cgroup-root");
 366                 DECLARE_ARG(buf);
 367         }
 368
 369         if (opts->user->verbose)
 370                 DECLARE_ARG("-vvvvvv");
 371
 372         if (opts->user->action_script) {
 373                 DECLARE_ARG("--action-script");
 374                 DECLARE_ARG(opts->user->action_script);
 375         }
 376
 377         mnts = make_anonymous_mount_file(&opts->c->lxc_conf->mount_list);
 378         if (!mnts)
 379                 goto err;
 380
 381         while (getmntent_r(mnts, &mntent, buf, sizeof(buf))) {
 382                 char *fmt, *key, *val, *mntdata;
 383                 char arg[2 * PATH_MAX + 2];
 384                 unsigned long flags;
 385
 386                 if (parse_mntopts(mntent.mnt_opts, &flags, &mntdata) < 0)
 387                         goto err;
 388
 389                 free(mntdata);
 390
 391                 /* only add --ext-mount-map for actual bind mounts */
 392                 if (!(flags & MS_BIND))
 393                         continue;
 394
 395                 if (strcmp(opts->action, "dump") == 0) {
 396                         fmt = "/%s:%s";
 397                         key = mntent.mnt_dir;
 398                         val = mntent.mnt_dir;
 399                 } else {
 400                         fmt = "%s:%s";
 401                         key = mntent.mnt_dir;
 402                         val = mntent.mnt_fsname;
 403                 }
 404
 405                 ret = snprintf(arg, sizeof(arg), fmt, key, val);
 406                 if (ret < 0 || ret >= sizeof(arg)) {
 407                         fclose(mnts);
 408                         ERROR("snprintf failed");
 409                         goto err;
 410                 }
 411
 412                 DECLARE_ARG("--ext-mount-map");
 413                 DECLARE_ARG(arg);
 414         }
 415         fclose(mnts);
 416
 417         if (strcmp(opts->action, "dump") == 0 || strcmp(opts->action, "pre-dump") == 0) {
 418                 char pid[32], *freezer_relative;
 419
 420                 if (sprintf(pid, "%d", opts->c->init_pid(opts->c)) < 0)
 421                         goto err;
 422
 423                 DECLARE_ARG("-t");
 424                 DECLARE_ARG(pid);
 425
 426                 freezer_relative = lxc_cmd_get_cgroup_path(opts->c->name,
 427                                                            opts->c->config_path,
 428                                                            "freezer");
 429                 if (!freezer_relative) {
 430                         ERROR("failed getting freezer path");
 431                         goto err;
 432                 }
 433
 434                 ret = snprintf(log, sizeof(log), "/sys/fs/cgroup/freezer/%s", freezer_relative);
 435                 if (ret < 0 || ret >= sizeof(log))
 436                         goto err;
 437
 438                 if (!opts->user->disable_skip_in_flight &&
 439                                 strcmp(opts->criu_version, CRIU_IN_FLIGHT_SUPPORT) >= 0)
 440                         DECLARE_ARG("--skip-in-flight");
 441
 442                 DECLARE_ARG("--freeze-cgroup");
 443                 DECLARE_ARG(log);
 444
 445                 if (opts->tty_id[0]) {
 446                         DECLARE_ARG("--ext-mount-map");
 447                         DECLARE_ARG("/dev/console:console");
 448
 449                         DECLARE_ARG("--external");
 450                         DECLARE_ARG(opts->tty_id);
 451                 }
 452
 453                 if (opts->user->predump_dir) {
 454                         DECLARE_ARG("--prev-images-dir");
 455                         DECLARE_ARG(opts->user->predump_dir);
 456                         DECLARE_ARG("--track-mem");
 457                 }
 458
 459                 if (opts->user->pageserver_address && opts->user->pageserver_port) {
 460                         DECLARE_ARG("--page-server");
 461                         DECLARE_ARG("--address");
 462                         DECLARE_ARG(opts->user->pageserver_address);
 463                         DECLARE_ARG("--port");
 464                         DECLARE_ARG(opts->user->pageserver_port);
 465                 }
 466
 467                 if (!opts->user->preserves_inodes)
 468                         DECLARE_ARG("--force-irmap");
 469
 470                 if (opts->user->ghost_limit) {
 471                         char ghost_limit[32];
 472
 473                         ret = sprintf(ghost_limit, "%"PRIu64, opts->user->ghost_limit);
 474                         if (ret < 0 || ret >= sizeof(ghost_limit)) {
 475                                 ERROR("failed to print ghost limit %"PRIu64, opts->user->ghost_limit);
 476                                 goto err;
 477                         }
 478
 479                         DECLARE_ARG("--ghost-limit");
 480                         DECLARE_ARG(ghost_limit);
 481                 }
 482
 483                 /* only for final dump */
 484                 if (strcmp(opts->action, "dump") == 0 && !opts->user->stop)
 485                         DECLARE_ARG("--leave-running");
 486         } else if (strcmp(opts->action, "restore") == 0) {
 487                 void *m;
 488                 int additional;
 489                 struct lxc_conf *lxc_conf = opts->c->lxc_conf;
 490
 491                 DECLARE_ARG("--root");
 492                 DECLARE_ARG(opts->c->lxc_conf->rootfs.mount);
 493                 DECLARE_ARG("--restore-detached");
 494                 DECLARE_ARG("--restore-sibling");
 495
 496                 if (tty_info[0]) {
 497                         if (opts->console_fd < 0) {
 498                                 ERROR("lxc.console.path configured on source host but not target");
 499                                 goto err;
 500                         }
 501
 502                         ret = snprintf(buf, sizeof(buf), "fd[%d]:%s", opts->console_fd, tty_info);
 503                         if (ret < 0 || ret >= sizeof(buf))
 504                                 goto err;
 505
 506                         DECLARE_ARG("--inherit-fd");
 507                         DECLARE_ARG(buf);
 508                 }
 509                 if (opts->console_name) {
 510                         if (snprintf(buf, sizeof(buf), "console:%s", opts->console_name) < 0) {
 511                                 SYSERROR("sprintf'd too many bytes");
 512                         }
 513                         DECLARE_ARG("--ext-mount-map");
 514                         DECLARE_ARG(buf);
 515                 }
 516
 517                 if (lxc_conf->lsm_aa_profile || lxc_conf->lsm_se_context) {
 518
 519                         if (lxc_conf->lsm_aa_profile)
 520                                 ret = snprintf(buf, sizeof(buf), "apparmor:%s", lxc_conf->lsm_aa_profile);
 521                         else
 522                                 ret = snprintf(buf, sizeof(buf), "selinux:%s", lxc_conf->lsm_se_context);
 523
 524                         if (ret < 0 || ret >= sizeof(buf))
 525                                 goto err;
 526
 527                         DECLARE_ARG("--lsm-profile");
 528                         DECLARE_ARG(buf);
 529                 }
 530
 531                 additional = lxc_list_len(&opts->c->lxc_conf->network) * 2;
 532
 533                 m = realloc(argv, (argc + additional + 1) * sizeof(*argv));
 534                 if (!m)
 535                         goto err;
 536                 argv = m;
 537
 538                 lxc_list_for_each(it, &opts->c->lxc_conf->network) {
 539                         char eth[128], *veth;
 540                         char *fmt;
 541                         struct lxc_netdev *n = it->elem;
 542                         bool external_not_veth;
 543
 544                         if (cmp_version(opts->criu_version, CRIU_EXTERNAL_NOT_VETH) >= 0) {
 545                                 /* Since criu version 2.8 the usage of --veth-pair
 546                                  * has been deprecated:
 547                                  * git tag --contains f2037e6d3445fc400
 548                                  * v2.8 */
 549                                 external_not_veth = true;
 550                         } else {
 551                                 external_not_veth = false;
 552                         }
 553
 554                         if (n->name[0] != '\0') {
 555                                 if (strlen(n->name) >= sizeof(eth))
 556                                         goto err;
 557                                 strncpy(eth, n->name, sizeof(eth));
 558                         } else {
 559                                 ret = snprintf(eth, sizeof(eth), "eth%d", netnr);
 560                                 if (ret < 0 || ret >= sizeof(eth))
 561                                         goto err;
 562                         }
 563
 564                         switch (n->type) {
 565                         case LXC_NET_VETH:
 566                                 veth = n->priv.veth_attr.pair;
 567                                 if (veth[0] == '\0')
 568                                         veth = n->priv.veth_attr.veth1;
 569
 570                                 if (n->link[0] != '\0') {
 571                                         if (external_not_veth)
 572                                                 fmt = "veth[%s]:%s@%s";
 573                                         else
 574                                                 fmt = "%s=%s@%s";
 575
 576                                         ret = snprintf(buf, sizeof(buf), fmt, eth, veth, n->link);
 577                                 } else {
 578                                         if (external_not_veth)
 579                                                 fmt = "veth[%s]:%s";
 580                                         else
 581                                                 fmt = "%s=%s";
 582
 583                                         ret = snprintf(buf, sizeof(buf), fmt, eth, veth);
 584                                 }
 585                                 if (ret < 0 || ret >= sizeof(buf))
 586                                         goto err;
 587                                 break;
 588                         case LXC_NET_MACVLAN:
 589                                 if (n->link[0] == '\0') {
 590                                         ERROR("no host interface for macvlan %s", n->name);
 591                                         goto err;
 592                                 }
 593
 594                                 ret = snprintf(buf, sizeof(buf), "macvlan[%s]:%s", eth, n->link);
 595                                 if (ret < 0 || ret >= sizeof(buf))
 596                                         goto err;
 597                                 break;
 598                         case LXC_NET_NONE:
 599                         case LXC_NET_EMPTY:
 600                                 break;
 601                         default:
 602                                 /* we have screened for this earlier... */
 603                                 ERROR("unexpected network type %d", n->type);
 604                                 goto err;
 605                         }
 606
 607                         if (external_not_veth)
 608                                 DECLARE_ARG("--external");
 609                         else
 610                                 DECLARE_ARG("--veth-pair");
 611                         DECLARE_ARG(buf);
 612                         netnr++;
 613                 }
 614
 615         }
 616
 617         argv[argc] = NULL;
 618
 619         buf[0] = 0;
 620         pos = 0;
 621
 622         for (i = 0; argv[i]; i++) {
 623                 ret = snprintf(buf + pos, sizeof(buf) - pos, "%s ", argv[i]);
 624                 if (ret < 0 || ret >= sizeof(buf) - pos)
 625                         goto err;
 626                 else
 627                         pos += ret;
 628         }
 629
 630         INFO("execing: %s", buf);
 631
 632         /* before criu inits its log, it sometimes prints things to stdout/err;
 633          * let's be sure we capture that.
 634          */
 635         if (dup2(opts->pipefd, STDOUT_FILENO) < 0) {
 636                 SYSERROR("dup2 stdout failed");
 637                 goto err;
 638         }
 639
 640         if (dup2(opts->pipefd, STDERR_FILENO) < 0) {
 641                 SYSERROR("dup2 stderr failed");
 642                 goto err;
 643         }
 644
 645         close(opts->pipefd);
 646
 647 #undef DECLARE_ARG
 648         execv(argv[0], argv);
 649 err:
 650         for (i = 0; argv[i]; i++)
 651                 free(argv[i]);
 652         free(argv);
 653 }
 654
 655 /*
 656  * Function to check if the checks activated in 'features_to_check' are
 657  * available with the current architecture/kernel/criu combination.
 658  *
 659  * Parameter features_to_check is a bit mask of all features that should be
 660  * checked (see feature check defines in lxc/lxccontainer.h).
 661  *
 662  * If the return value is true, all requested features are supported. If
 663  * the return value is false the features_to_check parameter is updated
 664  * to reflect which features are available. '0' means no feature but
 665  * also that something went totally wrong.
 666  *
 667  * Some of the code flow of criu_version_ok() is duplicated and maybe it
 668  * is a good candidate for refactoring.
 669  */
 670 bool __criu_check_feature(uint64_t *features_to_check)
 671 {
 672         pid_t pid;
 673         uint64_t current_bit = 0;
 674         int ret;
 675         int features = *features_to_check;
 676         /* Feature checking is currently always like
 677          * criu check --feature <feature-name>
 678          */
 679         char *args[] = { "criu", "check", "--feature", NULL, NULL };
 680
 681         if ((features & ~FEATURE_MEM_TRACK & ~FEATURE_LAZY_PAGES) != 0) {
 682                 /* There are feature bits activated we do not understand.
 683                  * Refusing to answer at all */
 684                 *features_to_check = 0;
 685                 return false;
 686         }
 687
 688         while (current_bit < sizeof(uint64_t) * 8) {
 689                 /* only test requested features */
 690                 if (!(features & (1ULL << current_bit))) {
 691                         /* skip this */
 692                         current_bit++;
 693                         continue;
 694                 }
 695
 696                 pid = fork();
 697                 if (pid < 0) {
 698                         SYSERROR("fork() failed");
 699                         *features_to_check = 0;
 700                         return false;
 701                 }
 702
 703                 if (pid == 0) {
 704                         if ((1ULL << current_bit) == FEATURE_MEM_TRACK)
 705                                 /* This is needed for pre-dump support, which
 706                                  * enables pre-copy migration. */
 707                                 args[3] = "mem_dirty_track";
 708                         else if ((1ULL << current_bit) == FEATURE_LAZY_PAGES)
 709                                 /* CRIU has two checks for userfaultfd support.
 710                                  *
 711                                  * The simpler check is only for 'uffd'. If the
 712                                  * kernel supports userfaultfd without noncoop
 713                                  * then only process can be lazily restored
 714                                  * which do not fork. With 'uffd-noncoop'
 715                                  * it is also possible to lazily restore processes
 716                                  * which do fork. For a container runtime like
 717                                  * LXC checking only for 'uffd' makes not much sense. */
 718                                 args[3] = "uffd-noncoop";
 719                         else
 720                                 exit(1);
 721
 722                         null_stdfds();
 723
 724                         execvp("criu", args);
 725                         SYSERROR("Failed to exec \"criu\"");
 726                         exit(1);
 727                 }
 728
 729                 ret = wait_for_pid(pid);
 730
 731                 if (ret == -1) {
 732                         /* It is not known why CRIU failed. Either
 733                          * CRIU is not available, the feature check
 734                          * does not exist or the feature is not
 735                          * supported. */
 736                         INFO("feature not supported");
 737                         /* Clear not supported feature bit */
 738                         features &= ~(1ULL << current_bit);
 739                 }
 740
 741                 current_bit++;
 742                 /* no more checks requested; exit check loop */
 743                 if (!(features & ~((1ULL << current_bit)-1)))
 744                         break;
 745         }
 746         if (features != *features_to_check) {
 747                 *features_to_check = features;
 748                 return false;
 749         }
 750         return true;
 751 }
 752
 753 /*
 754  * Check to see if the criu version is recent enough for all the features we
 755  * use. This version allows either CRIU_VERSION or (CRIU_GITID_VERSION and
 756  * CRIU_GITID_PATCHLEVEL) to work, enabling users building from git to c/r
 757  * things potentially before a version is released with a particular feature.
 758  *
 759  * The intent is that when criu development slows down, we can drop this, but
 760  * for now we shouldn't attempt to c/r with versions that we know won't work.
 761  *
 762  * Note: If version != NULL criu_version() stores the detected criu version in
 763  * version. Allocates memory for version which must be freed by caller.
 764  */
 765 static bool criu_version_ok(char **version)
 766 {
 767         int pipes[2];
 768         pid_t pid;
 769
 770         if (pipe(pipes) < 0) {
 771                 SYSERROR("pipe() failed");
 772                 return false;
 773         }
 774
 775         pid = fork();
 776         if (pid < 0) {
 777                 SYSERROR("fork() failed");
 778                 return false;
 779         }
 780
 781         if (pid == 0) {
 782                 char *args[] = { "criu", "--version", NULL };
 783                 char *path;
 784                 close(pipes[0]);
 785
 786                 close(STDERR_FILENO);
 787                 if (dup2(pipes[1], STDOUT_FILENO) < 0)
 788                         exit(1);
 789
 790                 path = on_path("criu", NULL);
 791                 if (!path)
 792                         exit(1);
 793
 794                 execv(path, args);
 795                 exit(1);
 796         } else {
 797                 FILE *f;
 798                 char *tmp;
 799                 int patch;
 800
 801                 close(pipes[1]);
 802                 if (wait_for_pid(pid) < 0) {
 803                         close(pipes[0]);
 804                         SYSERROR("execing criu failed, is it installed?");
 805                         return false;
 806                 }
 807
 808                 f = fdopen(pipes[0], "r");
 809                 if (!f) {
 810                         close(pipes[0]);
 811                         return false;
 812                 }
 813
 814                 tmp = malloc(1024);
 815                 if (!tmp) {
 816                         fclose(f);
 817                         return false;
 818                 }
 819
 820                 if (fscanf(f, "Version: %1023[^\n]s", tmp) != 1)
 821                         goto version_error;
 822
 823                 if (fgetc(f) != '\n')
 824                         goto version_error;
 825
 826                 if (strcmp(tmp, CRIU_VERSION) >= 0)
 827                         goto version_match;
 828
 829                 if (fscanf(f, "GitID: v%1023[^-]s", tmp) != 1)
 830                         goto version_error;
 831
 832                 if (fgetc(f) != '-')
 833                         goto version_error;
 834
 835                 if (fscanf(f, "%d", &patch) != 1)
 836                         goto version_error;
 837
 838                 if (strcmp(tmp, CRIU_GITID_VERSION) < 0)
 839                         goto version_error;
 840
 841                 if (patch < CRIU_GITID_PATCHLEVEL)
 842                         goto version_error;
 843
 844 version_match:
 845                 fclose(f);
 846                 if (!version)
 847                         free(tmp);
 848                 else
 849                         *version = tmp;
 850                 return true;
 851
 852 version_error:
 853                 fclose(f);
 854                 free(tmp);
 855                 ERROR("must have criu " CRIU_VERSION " or greater to checkpoint/restore");
 856                 return false;
 857         }
 858 }
 859
 860 /* Check and make sure the container has a configuration that we know CRIU can
 861  * dump. */
 862 static bool criu_ok(struct lxc_container *c, char **criu_version)
 863 {
 864         struct lxc_list *it;
 865
 866         if (!criu_version_ok(criu_version))
 867                 return false;
 868
 869         if (geteuid()) {
 870                 ERROR("Must be root to checkpoint");
 871                 return false;
 872         }
 873
 874         /* We only know how to restore containers with veth networks. */
 875         lxc_list_for_each(it, &c->lxc_conf->network) {
 876                 struct lxc_netdev *n = it->elem;
 877                 switch(n->type) {
 878                 case LXC_NET_VETH:
 879                 case LXC_NET_NONE:
 880                 case LXC_NET_EMPTY:
 881                 case LXC_NET_MACVLAN:
 882                         break;
 883                 default:
 884                         ERROR("Found un-dumpable network: %s (%s)", lxc_net_type_to_str(n->type), n->name);
 885                         return false;
 886                 }
 887         }
 888
 889         return true;
 890 }
 891
 892 static bool restore_net_info(struct lxc_container *c)
 893 {
 894         struct lxc_list *it;
 895         bool has_error = true;
 896
 897         if (container_mem_lock(c))
 898                 return false;
 899
 900         lxc_list_for_each(it, &c->lxc_conf->network) {
 901                 struct lxc_netdev *netdev = it->elem;
 902                 char template[IFNAMSIZ];
 903
 904                 if (netdev->type != LXC_NET_VETH)
 905                         continue;
 906
 907                 snprintf(template, sizeof(template), "vethXXXXXX");
 908
 909                 if (netdev->priv.veth_attr.pair[0] == '\0' &&
 910                     netdev->priv.veth_attr.veth1[0] == '\0') {
 911                         if (!lxc_mkifname(template))
 912                                 goto out_unlock;
 913
 914                         strcpy(netdev->priv.veth_attr.veth1, template);
 915                 }
 916         }
 917
 918         has_error = false;
 919
 920 out_unlock:
 921         container_mem_unlock(c);
 922         return !has_error;
 923 }
 924
 925 /* do_restore never returns, the calling process is used as the monitor process.
 926  * do_restore calls exit() if it fails.
 927  */
 928 static void do_restore(struct lxc_container *c, int status_pipe, struct migrate_opts *opts, char *criu_version)
 929 {
 930         int fd, ret;
 931         pid_t pid;
 932         struct lxc_handler *handler;
 933         int status = 0;
 934         int pipes[2] = {-1, -1};
 935
 936         /* Try to detach from the current controlling tty if it exists.
 937          * Othwerise, lxc_init (via lxc_console) will attach the container's
 938          * console output to the current tty, which is probably not what any
 939          * library user wants, and if they do, they can just manually configure
 940          * it :)
 941          */
 942         fd = open("/dev/tty", O_RDWR);
 943         if (fd >= 0) {
 944                 if (ioctl(fd, TIOCNOTTY, NULL) < 0)
 945                         SYSERROR("couldn't detach from tty");
 946                 close(fd);
 947         }
 948
 949         handler = lxc_init_handler(c->name, c->lxc_conf, c->config_path, false);
 950         if (!handler)
 951                 goto out;
 952
 953         if (lxc_init(c->name, handler) < 0)
 954                 goto out;
 955
 956         if (!cgroup_init(handler)) {
 957                 ERROR("failed initing cgroups");
 958                 goto out_fini_handler;
 959         }
 960
 961         if (!cgroup_create(handler)) {
 962                 ERROR("failed creating groups");
 963                 goto out_fini_handler;
 964         }
 965
 966         if (!restore_net_info(c)) {
 967                 ERROR("failed restoring network info");
 968                 goto out_fini_handler;
 969         }
 970
 971         ret = resolve_clone_flags(handler);
 972         if (ret < 0) {
 973                 ERROR("%s - Unsupported clone flag specified", strerror(errno));
 974                 goto out_fini_handler;
 975         }
 976
 977         if (pipe(pipes) < 0) {
 978                 SYSERROR("pipe() failed");
 979                 goto out_fini_handler;
 980         }
 981
 982         pid = fork();
 983         if (pid < 0)
 984                 goto out_fini_handler;
 985
 986         if (pid == 0) {
 987                 struct criu_opts os;
 988                 struct lxc_rootfs *rootfs;
 989                 int flags;
 990
 991                 close(status_pipe);
 992                 status_pipe = -1;
 993
 994                 close(pipes[0]);
 995                 pipes[0] = -1;
 996
 997                 if (unshare(CLONE_NEWNS))
 998                         goto out_fini_handler;
 999
1000                 /* CRIU needs the lxc root bind mounted so that it is the root of some
1001                  * mount. */
1002                 rootfs = &c->lxc_conf->rootfs;
1003
1004                 if (rootfs_is_blockdev(c->lxc_conf)) {
1005                         if (do_rootfs_setup(c->lxc_conf, c->name, c->config_path) < 0)
1006                                 goto out_fini_handler;
1007                 } else {
1008                         if (mkdir(rootfs->mount, 0755) < 0 && errno != EEXIST)
1009                                 goto out_fini_handler;
1010
1011                         if (mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL) < 0) {
1012                                 SYSERROR("remount / to private failed");
1013                                 goto out_fini_handler;
1014                         }
1015
1016                         if (mount(rootfs->path, rootfs->mount, NULL, MS_BIND, NULL) < 0) {
1017                                 rmdir(rootfs->mount);
1018                                 goto out_fini_handler;
1019                         }
1020                 }
1021
1022                 os.pipefd = pipes[1];
1023                 os.action = "restore";
1024                 os.user = opts;
1025                 os.c = c;
1026                 os.console_fd = c->lxc_conf->console.slave;
1027                 os.criu_version = criu_version;
1028                 os.handler = handler;
1029
1030                 if (os.console_fd >= 0) {
1031                         /* Twiddle the FD_CLOEXEC bit. We want to pass this FD to criu
1032                          * via --inherit-fd, so we don't want it to close.
1033                          */
1034                         flags = fcntl(os.console_fd, F_GETFD);
1035                         if (flags < 0) {
1036                                 SYSERROR("F_GETFD failed: %d", os.console_fd);
1037                                 goto out_fini_handler;
1038                         }
1039
1040                         flags &= ~FD_CLOEXEC;
1041
1042                         if (fcntl(os.console_fd, F_SETFD, flags) < 0) {
1043                                 SYSERROR("F_SETFD failed");
1044                                 goto out_fini_handler;
1045                         }
1046                 }
1047                 os.console_name = c->lxc_conf->console.name;
1048
1049                 /* exec_criu() returning is an error */
1050                 exec_criu(&os);
1051                 umount(rootfs->mount);
1052                 rmdir(rootfs->mount);
1053                 goto out_fini_handler;
1054         } else {
1055                 int ret;
1056                 char title[2048];
1057
1058                 close(pipes[1]);
1059                 pipes[1] = -1;
1060
1061                 pid_t w = waitpid(pid, &status, 0);
1062                 if (w == -1) {
1063                         SYSERROR("waitpid");
1064                         goto out_fini_handler;
1065                 }
1066
1067                 if (WIFEXITED(status)) {
1068                         char buf[4096];
1069
1070                         if (WEXITSTATUS(status)) {
1071                                 int n;
1072
1073                                 n = read(pipes[0], buf, sizeof(buf));
1074                                 if (n < 0) {
1075                                         SYSERROR("failed reading from criu stderr");
1076                                         goto out_fini_handler;
1077                                 }
1078
1079                                 if (n == sizeof(buf))
1080                                         n--;
1081                                 buf[n] = 0;
1082
1083                                 ERROR("criu process exited %d, output:\n%s", WEXITSTATUS(status), buf);
1084                                 goto out_fini_handler;
1085                         } else {
1086                                 ret = snprintf(buf, sizeof(buf), "/proc/self/task/%lu/children", (unsigned long)syscall(__NR_gettid));
1087                                 if (ret < 0 || ret >= sizeof(buf)) {
1088                                         ERROR("snprintf'd too many characters: %d", ret);
1089                                         goto out_fini_handler;
1090                                 }
1091
1092                                 FILE *f = fopen(buf, "r");
1093                                 if (!f) {
1094                                         SYSERROR("couldn't read restore's children file %s", buf);
1095                                         goto out_fini_handler;
1096                                 }
1097
1098                                 ret = fscanf(f, "%d", (int*) &handler->pid);
1099                                 fclose(f);
1100                                 if (ret != 1) {
1101                                         ERROR("reading restore pid failed");
1102                                         goto out_fini_handler;
1103                                 }
1104
1105                                 if (lxc_set_state(c->name, handler, RUNNING)) {
1106                                         ERROR("error setting running state after restore");
1107                                         goto out_fini_handler;
1108                                 }
1109                         }
1110                 } else {
1111                         ERROR("CRIU was killed with signal %d", WTERMSIG(status));
1112                         goto out_fini_handler;
1113                 }
1114
1115                 close(pipes[0]);
1116
1117                 ret = write(status_pipe, &status, sizeof(status));
1118                 close(status_pipe);
1119                 status_pipe = -1;
1120
1121                 if (sizeof(status) != ret) {
1122                         SYSERROR("failed to write all of status");
1123                         goto out_fini_handler;
1124                 }
1125
1126                 /*
1127                  * See comment in lxcapi_start; we don't care if these
1128                  * fail because it's just a beauty thing. We just
1129                  * assign the return here to silence potential.
1130                  */
1131                 ret = snprintf(title, sizeof(title), "[lxc monitor] %s %s", c->config_path, c->name);
1132                 if (ret < 0 || (size_t)ret >= sizeof(title))
1133                         INFO("Setting truncated process name");
1134
1135                 ret = setproctitle(title);
1136                 if (ret < 0)
1137                         INFO("Failed to set process name");
1138
1139                 ret = lxc_poll(c->name, handler);
1140                 if (ret)
1141                         lxc_abort(c->name, handler);
1142                 lxc_fini(c->name, handler);
1143                 exit(ret);
1144         }
1145
1146 out_fini_handler:
1147         if (pipes[0] >= 0)
1148                 close(pipes[0]);
1149         if (pipes[1] >= 0)
1150                 close(pipes[1]);
1151
1152         lxc_fini(c->name, handler);
1153
1154 out:
1155         if (status_pipe >= 0) {
1156                 /* ensure getting here was a failure, e.g. if we failed to
1157                  * parse the child pid or something, even after a successful
1158                  * restore
1159                  */
1160                 if (!status)
1161                         status = 1;
1162
1163                 if (write(status_pipe, &status, sizeof(status)) != sizeof(status))
1164                         SYSERROR("writing status failed");
1165                 close(status_pipe);
1166         }
1167
1168         exit(1);
1169 }
1170
1171 static int save_tty_major_minor(char *directory, struct lxc_container *c, char *tty_id, int len)
1172 {
1173         FILE *f;
1174         char path[PATH_MAX];
1175         int ret;
1176         struct stat sb;
1177
1178         if (c->lxc_conf->console.path && !strcmp(c->lxc_conf->console.path, "none")) {
1179                 tty_id[0] = 0;
1180                 return 0;
1181         }
1182
1183         ret = snprintf(path, sizeof(path), "/proc/%d/root/dev/console", c->init_pid(c));
1184         if (ret < 0 || ret >= sizeof(path)) {
1185                 ERROR("snprintf'd too many chacters: %d", ret);
1186                 return -1;
1187         }
1188
1189         ret = stat(path, &sb);
1190         if (ret < 0) {
1191                 SYSERROR("stat of %s failed", path);
1192                 return -1;
1193         }
1194
1195         ret = snprintf(path, sizeof(path), "%s/tty.info", directory);
1196         if (ret < 0 || ret >= sizeof(path)) {
1197                 ERROR("snprintf'd too many characters: %d", ret);
1198                 return -1;
1199         }
1200
1201         ret = snprintf(tty_id, len, "tty[%llx:%llx]",
1202                                         (long long unsigned) sb.st_rdev,
1203                                         (long long unsigned) sb.st_dev);
1204         if (ret < 0 || ret >= sizeof(path)) {
1205                 ERROR("snprintf'd too many characters: %d", ret);
1206                 return -1;
1207         }
1208
1209         f = fopen(path, "w");
1210         if (!f) {
1211                 SYSERROR("failed to open %s", path);
1212                 return -1;
1213         }
1214
1215         ret = fprintf(f, "%s", tty_id);
1216         fclose(f);
1217         if (ret < 0)
1218                 SYSERROR("failed to write to %s", path);
1219         return ret;
1220 }
1221
1222 /* do one of either predump or a regular dump */
1223 static bool do_dump(struct lxc_container *c, char *mode, struct migrate_opts *opts)
1224 {
1225         int ret;
1226         pid_t pid;
1227         int criuout[2];
1228         char *criu_version = NULL;
1229
1230         if (!criu_ok(c, &criu_version))
1231                 return false;
1232
1233         ret = pipe(criuout);
1234         if (ret < 0) {
1235                 SYSERROR("pipe() failed");
1236                 return false;
1237         }
1238
1239         if (mkdir_p(opts->directory, 0700) < 0)
1240                 goto fail;
1241
1242         pid = fork();
1243         if (pid < 0) {
1244                 SYSERROR("fork failed");
1245                 goto fail;
1246         }
1247
1248         if (pid == 0) {
1249                 struct criu_opts os;
1250                 struct lxc_handler h;
1251
1252                 close(criuout[0]);
1253
1254                 lxc_zero_handler(&h);
1255
1256                 h.name = c->name;
1257                 if (!cgroup_init(&h)) {
1258                         ERROR("failed to cgroup_init()");
1259                         exit(1);
1260                 }
1261
1262                 os.pipefd = criuout[1];
1263                 os.action = mode;
1264                 os.user = opts;
1265                 os.c = c;
1266                 os.console_name = c->lxc_conf->console.path;
1267                 os.criu_version = criu_version;
1268
1269                 ret = save_tty_major_minor(opts->directory, c, os.tty_id, sizeof(os.tty_id));
1270                 if (ret < 0) {
1271                         free(criu_version);
1272                         exit(EXIT_FAILURE);
1273                 }
1274
1275                 /* exec_criu() returning is an error */
1276                 exec_criu(&os);
1277                 free(criu_version);
1278                 exit(EXIT_FAILURE);
1279         } else {
1280                 int status;
1281                 ssize_t n;
1282                 char buf[4096];
1283                 bool ret;
1284
1285                 close(criuout[1]);
1286
1287                 pid_t w = waitpid(pid, &status, 0);
1288                 if (w == -1) {
1289                         SYSERROR("waitpid");
1290                         close(criuout[0]);
1291                         return false;
1292                 }
1293
1294                 n = read(criuout[0], buf, sizeof(buf));
1295                 close(criuout[0]);
1296                 if (n < 0) {
1297                         SYSERROR("read");
1298                         n = 0;
1299                 }
1300                 buf[n] = 0;
1301
1302                 if (WIFEXITED(status)) {
1303                         if (WEXITSTATUS(status)) {
1304                                 ERROR("dump failed with %d", WEXITSTATUS(status));
1305                                 ret = false;
1306                         } else {
1307                                 ret = true;
1308                         }
1309                 } else if (WIFSIGNALED(status)) {
1310                         ERROR("dump signaled with %d", WTERMSIG(status));
1311                         ret = false;
1312                 } else {
1313                         ERROR("unknown dump exit %d", status);
1314                         ret = false;
1315                 }
1316
1317                 if (!ret)
1318                         ERROR("criu output: %s", buf);
1319                 return ret;
1320         }
1321 fail:
1322         close(criuout[0]);
1323         close(criuout[1]);
1324         rmdir(opts->directory);
1325         free(criu_version);
1326         return false;
1327 }
1328
1329 bool __criu_pre_dump(struct lxc_container *c, struct migrate_opts *opts)
1330 {
1331         return do_dump(c, "pre-dump", opts);
1332 }
1333
1334 bool __criu_dump(struct lxc_container *c, struct migrate_opts *opts)
1335 {
1336         char path[PATH_MAX];
1337         int ret;
1338
1339         ret = snprintf(path, sizeof(path), "%s/inventory.img", opts->directory);
1340         if (ret < 0 || ret >= sizeof(path))
1341                 return false;
1342
1343         if (access(path, F_OK) == 0) {
1344                 ERROR("please use a fresh directory for the dump directory");
1345                 return false;
1346         }
1347
1348         return do_dump(c, "dump", opts);
1349 }
1350
1351 bool __criu_restore(struct lxc_container *c, struct migrate_opts *opts)
1352 {
1353         pid_t pid;
1354         int status, nread;
1355         int pipefd[2];
1356         char *criu_version = NULL;
1357
1358         if (!criu_ok(c, &criu_version))
1359                 return false;
1360
1361         if (geteuid()) {
1362                 ERROR("Must be root to restore");
1363                 return false;
1364         }
1365
1366         if (pipe(pipefd)) {
1367                 ERROR("failed to create pipe");
1368                 return false;
1369         }
1370
1371         pid = fork();
1372         if (pid < 0) {
1373                 close(pipefd[0]);
1374                 close(pipefd[1]);
1375                 return false;
1376         }
1377
1378         if (pid == 0) {
1379                 close(pipefd[0]);
1380                 /* this never returns */
1381                 do_restore(c, pipefd[1], opts, criu_version);
1382         }
1383
1384         close(pipefd[1]);
1385
1386         nread = read(pipefd[0], &status, sizeof(status));
1387         close(pipefd[0]);
1388         if (sizeof(status) != nread) {
1389                 ERROR("reading status from pipe failed");
1390                 goto err_wait;
1391         }
1392
1393         /* If the criu process was killed or exited nonzero, wait() for the
1394          * handler, since the restore process died. Otherwise, we don't need to
1395          * wait, since the child becomes the monitor process.
1396          */
1397         if (!WIFEXITED(status) || WEXITSTATUS(status))
1398                 goto err_wait;
1399         return true;
1400
1401 err_wait:
1402         if (wait_for_pid(pid))
1403                 ERROR("restore process died");
1404         return false;
1405 }