src/lxc/criu.c

   1 /*
   2  * lxc: linux Container library
   3  *
   4  * Copyright © 2014-2015 Canonical Ltd.
   5  *
   6  * Authors:
   7  * Tycho Andersen <tycho.andersen@canonical.com>
   8  *
   9  * This library is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License as published by the Free Software Foundation; either
  12  * version 2.1 of the License, or (at your option) any later version.
  13  *
  14  * This library is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with this library; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  22  */
  23 #define _GNU_SOURCE
  24 #include <assert.h>
  25 #include <inttypes.h>
  26 #include <linux/limits.h>
  27 #include <sched.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/mount.h>
  32 #include <sys/types.h>
  33 #include <sys/wait.h>
  34 #include <unistd.h>
  35
  36 #include "config.h"
  37
  38 #include "bdev.h"
  39 #include "cgroup.h"
  40 #include "conf.h"
  41 #include "commands.h"
  42 #include "criu.h"
  43 #include "log.h"
  44 #include "lxc.h"
  45 #include "lxclock.h"
  46 #include "network.h"
  47 #include "utils.h"
  48
  49 #define CRIU_VERSION            "2.0"
  50
  51 #define CRIU_GITID_VERSION      "2.0"
  52 #define CRIU_GITID_PATCHLEVEL   0
  53
  54 #define CRIU_IN_FLIGHT_SUPPORT  "2.4"
  55
  56 lxc_log_define(lxc_criu, lxc);
  57
  58 struct criu_opts {
  59         /* The type of criu invocation, one of "dump" or "restore" */
  60         char *action;
  61
  62         /* the user-provided migrate options relevant to this action */
  63         struct migrate_opts *user;
  64
  65         /* The container to dump */
  66         struct lxc_container *c;
  67
  68         /* dump: stop the container or not after dumping? */
  69         char tty_id[32]; /* the criu tty id for /dev/console, i.e. "tty[${rdev}:${dev}]" */
  70
  71         /* restore: the file to write the init process' pid into */
  72         struct lxc_handler *handler;
  73         int console_fd;
  74         /* The path that is bind mounted from /dev/console, if any. We don't
  75          * want to use `--ext-mount-map auto`'s result here because the pts
  76          * device may have a different path (e.g. if the pty number is
  77          * different) on the target host. NULL if lxc.console = "none".
  78          */
  79         char *console_name;
  80
  81         /* The detected version of criu */
  82         char *criu_version;
  83 };
  84
  85 static int load_tty_major_minor(char *directory, char *output, int len)
  86 {
  87         FILE *f;
  88         char path[PATH_MAX];
  89         int ret;
  90
  91         ret = snprintf(path, sizeof(path), "%s/tty.info", directory);
  92         if (ret < 0 || ret >= sizeof(path)) {
  93                 ERROR("snprintf'd too many chacters: %d", ret);
  94                 return -1;
  95         }
  96
  97         f = fopen(path, "r");
  98         if (!f) {
  99                 /* This means we're coming from a liblxc which didn't export
 100                  * the tty info. In this case they had to have lxc.console =
 101                  * none, so there's no problem restoring.
 102                  */
 103                 if (errno == ENOENT)
 104                         return 0;
 105
 106                 SYSERROR("couldn't open %s", path);
 107                 return -1;
 108         }
 109
 110         if (!fgets(output, len, f)) {
 111                 fclose(f);
 112                 SYSERROR("couldn't read %s", path);
 113                 return -1;
 114         }
 115
 116         fclose(f);
 117         return 0;
 118 }
 119
 120 static void exec_criu(struct criu_opts *opts)
 121 {
 122         char **argv, log[PATH_MAX];
 123         int static_args = 23, argc = 0, i, ret;
 124         int netnr = 0;
 125         struct lxc_list *it;
 126
 127         char buf[4096], tty_info[32];
 128         size_t pos;
 129         /* If we are currently in a cgroup /foo/bar, and the container is in a
 130          * cgroup /lxc/foo, lxcfs will give us an ENOENT if some task in the
 131          * container has an open fd that points to one of the cgroup files
 132          * (systemd always opens its "root" cgroup). So, let's escape to the
 133          * /actual/ root cgroup so that lxcfs thinks criu has enough rights to
 134          * see all cgroups.
 135          */
 136         if (!cgroup_escape()) {
 137                 ERROR("failed to escape cgroups");
 138                 return;
 139         }
 140
 141         /* The command line always looks like:
 142          * criu $(action) --tcp-established --file-locks --link-remap \
 143          * --manage-cgroups=full --action-script foo.sh -D $(directory) \
 144          * -o $(directory)/$(action).log --ext-mount-map auto
 145          * --enable-external-sharing --enable-external-masters
 146          * --enable-fs hugetlbfs --enable-fs tracefs --ext-mount-map console:/dev/pts/n
 147          * +1 for final NULL */
 148
 149         if (strcmp(opts->action, "dump") == 0 || strcmp(opts->action, "pre-dump") == 0) {
 150                 /* -t pid --freeze-cgroup /lxc/ct */
 151                 static_args += 4;
 152
 153                 /* --prev-images-dir <path-to-directory-A-relative-to-B> */
 154                 if (opts->user->predump_dir)
 155                         static_args += 2;
 156
 157                 /* --page-server --address <address> --port <port> */
 158                 if (opts->user->pageserver_address && opts->user->pageserver_port)
 159                         static_args += 5;
 160
 161                 /* --leave-running (only for final dump) */
 162                 if (strcmp(opts->action, "dump") == 0 && !opts->user->stop)
 163                         static_args++;
 164
 165                 /* --external tty[88,4] */
 166                 if (opts->tty_id[0])
 167                         static_args += 2;
 168
 169                 /* --force-irmap */
 170                 if (!opts->user->preserves_inodes)
 171                         static_args++;
 172
 173                 /* --ghost-limit 1024 */
 174                 if (opts->user->ghost_limit)
 175                         static_args += 2;
 176         } else if (strcmp(opts->action, "restore") == 0) {
 177                 /* --root $(lxc_mount_point) --restore-detached
 178                  * --restore-sibling
 179                  * --lsm-profile apparmor:whatever
 180                  */
 181                 static_args += 6;
 182
 183                 tty_info[0] = 0;
 184                 if (load_tty_major_minor(opts->user->directory, tty_info, sizeof(tty_info)))
 185                         return;
 186
 187                 /* --inherit-fd fd[%d]:tty[%s] */
 188                 if (tty_info[0])
 189                         static_args += 2;
 190         } else {
 191                 return;
 192         }
 193
 194         if (cgroup_num_hierarchies() > 0)
 195                 static_args += 2 * cgroup_num_hierarchies();
 196
 197         if (opts->user->verbose)
 198                 static_args++;
 199
 200         if (opts->user->action_script)
 201                 static_args += 2;
 202
 203         ret = snprintf(log, PATH_MAX, "%s/%s.log", opts->user->directory, opts->action);
 204         if (ret < 0 || ret >= PATH_MAX) {
 205                 ERROR("logfile name too long\n");
 206                 return;
 207         }
 208
 209         argv = malloc(static_args * sizeof(*argv));
 210         if (!argv)
 211                 return;
 212
 213         memset(argv, 0, static_args * sizeof(*argv));
 214
 215 #define DECLARE_ARG(arg)                                        \
 216         do {                                                    \
 217                 if (arg == NULL) {                              \
 218                         ERROR("Got NULL argument for criu");    \
 219                         goto err;                               \
 220                 }                                               \
 221                 argv[argc++] = strdup(arg);                     \
 222                 if (!argv[argc-1])                              \
 223                         goto err;                               \
 224         } while (0)
 225
 226         argv[argc++] = on_path("criu", NULL);
 227         if (!argv[argc-1]) {
 228                 ERROR("Couldn't find criu binary\n");
 229                 goto err;
 230         }
 231
 232         DECLARE_ARG(opts->action);
 233         DECLARE_ARG("--tcp-established");
 234         DECLARE_ARG("--file-locks");
 235         DECLARE_ARG("--link-remap");
 236         DECLARE_ARG("--manage-cgroups=full");
 237         DECLARE_ARG("--ext-mount-map");
 238         DECLARE_ARG("auto");
 239         DECLARE_ARG("--enable-external-sharing");
 240         DECLARE_ARG("--enable-external-masters");
 241         DECLARE_ARG("--enable-fs");
 242         DECLARE_ARG("hugetlbfs");
 243         DECLARE_ARG("--enable-fs");
 244         DECLARE_ARG("tracefs");
 245         DECLARE_ARG("-D");
 246         DECLARE_ARG(opts->user->directory);
 247         DECLARE_ARG("-o");
 248         DECLARE_ARG(log);
 249
 250         for (i = 0; i < cgroup_num_hierarchies(); i++) {
 251                 char **controllers = NULL, *fullname;
 252                 char *path;
 253
 254                 if (!cgroup_get_hierarchies(i, &controllers)) {
 255                         ERROR("failed to get hierarchy %d", i);
 256                         goto err;
 257                 }
 258
 259                 /* if we are in a dump, we have to ask the monitor process what
 260                  * the right cgroup is. if this is a restore, we can just use
 261                  * the handler the restore task created.
 262                  */
 263                 if (!strcmp(opts->action, "dump") || !strcmp(opts->action, "pre-dump")) {
 264                         path = lxc_cmd_get_cgroup_path(opts->c->name, opts->c->config_path, controllers[0]);
 265                         if (!path) {
 266                                 ERROR("failed to get cgroup path for %s", controllers[0]);
 267                                 goto err;
 268                         }
 269                 } else {
 270                         const char *p;
 271
 272                         p = cgroup_get_cgroup(opts->handler, controllers[0]);
 273                         if (!p) {
 274                                 ERROR("failed to get cgroup path for %s", controllers[0]);
 275                                 goto err;
 276                         }
 277
 278                         path = strdup(p);
 279                         if (!path) {
 280                                 ERROR("strdup failed");
 281                                 goto err;
 282                         }
 283                 }
 284
 285                 if (!lxc_deslashify(&path)) {
 286                         ERROR("failed to deslashify %s", path);
 287                         free(path);
 288                         goto err;
 289                 }
 290
 291                 fullname = lxc_string_join(",", (const char **) controllers, false);
 292                 if (!fullname) {
 293                         ERROR("failed to join controllers");
 294                         free(path);
 295                         goto err;
 296                 }
 297
 298                 ret = sprintf(buf, "%s:%s", fullname, path);
 299                 free(path);
 300                 free(fullname);
 301                 if (ret < 0 || ret >= sizeof(buf)) {
 302                         ERROR("sprintf of cgroup root arg failed");
 303                         goto err;
 304                 }
 305
 306                 DECLARE_ARG("--cgroup-root");
 307                 DECLARE_ARG(buf);
 308         }
 309
 310         if (opts->user->verbose)
 311                 DECLARE_ARG("-vvvvvv");
 312
 313         if (opts->user->action_script) {
 314                 DECLARE_ARG("--action-script");
 315                 DECLARE_ARG(opts->user->action_script);
 316         }
 317
 318         if (strcmp(opts->action, "dump") == 0 || strcmp(opts->action, "pre-dump") == 0) {
 319                 char pid[32], *freezer_relative;
 320
 321                 if (sprintf(pid, "%d", opts->c->init_pid(opts->c)) < 0)
 322                         goto err;
 323
 324                 DECLARE_ARG("-t");
 325                 DECLARE_ARG(pid);
 326
 327                 freezer_relative = lxc_cmd_get_cgroup_path(opts->c->name,
 328                                                            opts->c->config_path,
 329                                                            "freezer");
 330                 if (!freezer_relative) {
 331                         ERROR("failed getting freezer path");
 332                         goto err;
 333                 }
 334
 335                 ret = snprintf(log, sizeof(log), "/sys/fs/cgroup/freezer/%s", freezer_relative);
 336                 if (ret < 0 || ret >= sizeof(log))
 337                         goto err;
 338
 339                 if (!opts->user->disable_skip_in_flight &&
 340                                 strcmp(opts->criu_version, CRIU_IN_FLIGHT_SUPPORT) >= 0)
 341                         DECLARE_ARG("--skip-in-flight");
 342
 343                 DECLARE_ARG("--freeze-cgroup");
 344                 DECLARE_ARG(log);
 345
 346                 if (opts->tty_id[0]) {
 347                         DECLARE_ARG("--ext-mount-map");
 348                         DECLARE_ARG("/dev/console:console");
 349
 350                         DECLARE_ARG("--external");
 351                         DECLARE_ARG(opts->tty_id);
 352                 }
 353
 354                 if (opts->user->predump_dir) {
 355                         DECLARE_ARG("--prev-images-dir");
 356                         DECLARE_ARG(opts->user->predump_dir);
 357                 }
 358
 359                 if (opts->user->pageserver_address && opts->user->pageserver_port) {
 360                         DECLARE_ARG("--page-server");
 361                         DECLARE_ARG("--address");
 362                         DECLARE_ARG(opts->user->pageserver_address);
 363                         DECLARE_ARG("--port");
 364                         DECLARE_ARG(opts->user->pageserver_port);
 365                 }
 366
 367                 if (!opts->user->preserves_inodes)
 368                         DECLARE_ARG("--force-irmap");
 369
 370                 if (opts->user->ghost_limit) {
 371                         char ghost_limit[32];
 372
 373                         ret = sprintf(ghost_limit, "%"PRIu64, opts->user->ghost_limit);
 374                         if (ret < 0 || ret >= sizeof(ghost_limit)) {
 375                                 ERROR("failed to print ghost limit %"PRIu64, opts->user->ghost_limit);
 376                                 goto err;
 377                         }
 378
 379                         DECLARE_ARG("--ghost-limit");
 380                         DECLARE_ARG(ghost_limit);
 381                 }
 382
 383                 /* only for final dump */
 384                 if (strcmp(opts->action, "dump") == 0 && !opts->user->stop)
 385                         DECLARE_ARG("--leave-running");
 386         } else if (strcmp(opts->action, "restore") == 0) {
 387                 void *m;
 388                 int additional;
 389                 struct lxc_conf *lxc_conf = opts->c->lxc_conf;
 390
 391                 DECLARE_ARG("--root");
 392                 DECLARE_ARG(opts->c->lxc_conf->rootfs.mount);
 393                 DECLARE_ARG("--restore-detached");
 394                 DECLARE_ARG("--restore-sibling");
 395
 396                 if (tty_info[0]) {
 397                         if (opts->console_fd < 0) {
 398                                 ERROR("lxc.console configured on source host but not target");
 399                                 goto err;
 400                         }
 401
 402                         ret = snprintf(buf, sizeof(buf), "fd[%d]:%s", opts->console_fd, tty_info);
 403                         if (ret < 0 || ret >= sizeof(buf))
 404                                 goto err;
 405
 406                         DECLARE_ARG("--inherit-fd");
 407                         DECLARE_ARG(buf);
 408                 }
 409                 if (opts->console_name) {
 410                         if (snprintf(buf, sizeof(buf), "console:%s", opts->console_name) < 0) {
 411                                 SYSERROR("sprintf'd too many bytes");
 412                         }
 413                         DECLARE_ARG("--ext-mount-map");
 414                         DECLARE_ARG(buf);
 415                 }
 416
 417                 if (lxc_conf->lsm_aa_profile || lxc_conf->lsm_se_context) {
 418
 419                         if (lxc_conf->lsm_aa_profile)
 420                                 ret = snprintf(buf, sizeof(buf), "apparmor:%s", lxc_conf->lsm_aa_profile);
 421                         else
 422                                 ret = snprintf(buf, sizeof(buf), "selinux:%s", lxc_conf->lsm_se_context);
 423
 424                         if (ret < 0 || ret >= sizeof(buf))
 425                                 goto err;
 426
 427                         DECLARE_ARG("--lsm-profile");
 428                         DECLARE_ARG(buf);
 429                 }
 430
 431                 additional = lxc_list_len(&opts->c->lxc_conf->network) * 2;
 432
 433                 m = realloc(argv, (argc + additional + 1) * sizeof(*argv));
 434                 if (!m)
 435                         goto err;
 436                 argv = m;
 437
 438                 lxc_list_for_each(it, &opts->c->lxc_conf->network) {
 439                         char eth[128], *veth;
 440                         struct lxc_netdev *n = it->elem;
 441
 442                         if (n->name) {
 443                                 if (strlen(n->name) >= sizeof(eth))
 444                                         goto err;
 445                                 strncpy(eth, n->name, sizeof(eth));
 446                         } else
 447                                 sprintf(eth, "eth%d", netnr);
 448
 449                         switch (n->type) {
 450                         case LXC_NET_VETH:
 451                                 veth = n->priv.veth_attr.pair;
 452
 453                                 if (n->link)
 454                                         ret = snprintf(buf, sizeof(buf), "veth[%s]:%s@%s", eth, veth, n->link);
 455                                 else
 456                                         ret = snprintf(buf, sizeof(buf), "veth[%s]:%s", eth, veth);
 457                                 if (ret < 0 || ret >= sizeof(buf))
 458                                         goto err;
 459                                 break;
 460                         case LXC_NET_MACVLAN:
 461                                 if (n->name) {
 462                                         if (strlen(n->name) >= sizeof(eth))
 463                                                 goto err;
 464                                         strncpy(eth, n->name, sizeof(eth));
 465                                 } else
 466                                         sprintf(eth, "eth%d", netnr);
 467
 468                                 if (!n->link) {
 469                                         ERROR("no host interface for macvlan %s\n", n->name);
 470                                         goto err;
 471                                 }
 472
 473                                 ret = snprintf(buf, sizeof(buf), "macvlan[%s]:%s", eth, n->link);
 474                                 if (ret < 0 || ret >= sizeof(buf))
 475                                         goto err;
 476                                 break;
 477                         case LXC_NET_NONE:
 478                         case LXC_NET_EMPTY:
 479                                 break;
 480                         default:
 481                                 /* we have screened for this earlier... */
 482                                 ERROR("unexpected network type %d\n", n->type);
 483                                 goto err;
 484                         }
 485
 486                         DECLARE_ARG("--external");
 487                         DECLARE_ARG(buf);
 488                         netnr++;
 489                 }
 490
 491         }
 492
 493         argv[argc] = NULL;
 494
 495         buf[0] = 0;
 496         pos = 0;
 497
 498         for (i = 0; argv[i]; i++) {
 499                 ret = snprintf(buf + pos, sizeof(buf) - pos, "%s ", argv[i]);
 500                 if (ret < 0 || ret >= sizeof(buf) - pos)
 501                         goto err;
 502                 else
 503                         pos += ret;
 504         }
 505
 506         INFO("execing: %s", buf);
 507
 508 #undef DECLARE_ARG
 509         execv(argv[0], argv);
 510 err:
 511         for (i = 0; argv[i]; i++)
 512                 free(argv[i]);
 513         free(argv);
 514 }
 515
 516 /*
 517  * Check to see if the criu version is recent enough for all the features we
 518  * use. This version allows either CRIU_VERSION or (CRIU_GITID_VERSION and
 519  * CRIU_GITID_PATCHLEVEL) to work, enabling users building from git to c/r
 520  * things potentially before a version is released with a particular feature.
 521  *
 522  * The intent is that when criu development slows down, we can drop this, but
 523  * for now we shouldn't attempt to c/r with versions that we know won't work.
 524  *
 525  * Note: If version != NULL criu_version() stores the detected criu version in
 526  * version. Allocates memory for version which must be freed by caller.
 527  */
 528 static bool criu_version_ok(char **version)
 529 {
 530         int pipes[2];
 531         pid_t pid;
 532
 533         if (pipe(pipes) < 0) {
 534                 SYSERROR("pipe() failed");
 535                 return false;
 536         }
 537
 538         pid = fork();
 539         if (pid < 0) {
 540                 SYSERROR("fork() failed");
 541                 return false;
 542         }
 543
 544         if (pid == 0) {
 545                 char *args[] = { "criu", "--version", NULL };
 546                 char *path;
 547                 close(pipes[0]);
 548
 549                 close(STDERR_FILENO);
 550                 if (dup2(pipes[1], STDOUT_FILENO) < 0)
 551                         exit(1);
 552
 553                 path = on_path("criu", NULL);
 554                 if (!path)
 555                         exit(1);
 556
 557                 execv(path, args);
 558                 exit(1);
 559         } else {
 560                 FILE *f;
 561                 char *tmp;
 562                 int patch;
 563
 564                 close(pipes[1]);
 565                 if (wait_for_pid(pid) < 0) {
 566                         close(pipes[0]);
 567                         SYSERROR("execing criu failed, is it installed?");
 568                         return false;
 569                 }
 570
 571                 f = fdopen(pipes[0], "r");
 572                 if (!f) {
 573                         close(pipes[0]);
 574                         return false;
 575                 }
 576
 577                 tmp = malloc(1024);
 578                 if (!tmp) {
 579                         fclose(f);
 580                         return false;
 581                 }
 582
 583                 if (fscanf(f, "Version: %1023[^\n]s", tmp) != 1)
 584                         goto version_error;
 585
 586                 if (fgetc(f) != '\n')
 587                         goto version_error;
 588
 589                 if (strcmp(tmp, CRIU_VERSION) >= 0)
 590                         goto version_match;
 591
 592                 if (fscanf(f, "GitID: v%1023[^-]s", tmp) != 1)
 593                         goto version_error;
 594
 595                 if (fgetc(f) != '-')
 596                         goto version_error;
 597
 598                 if (fscanf(f, "%d", &patch) != 1)
 599                         goto version_error;
 600
 601                 if (strcmp(tmp, CRIU_GITID_VERSION) < 0)
 602                         goto version_error;
 603
 604                 if (patch < CRIU_GITID_PATCHLEVEL)
 605                         goto version_error;
 606
 607 version_match:
 608                 fclose(f);
 609                 if (!version)
 610                         free(tmp);
 611                 else
 612                         *version = tmp;
 613                 return true;
 614
 615 version_error:
 616                 fclose(f);
 617                 free(tmp);
 618                 ERROR("must have criu " CRIU_VERSION " or greater to checkpoint/restore\n");
 619                 return false;
 620         }
 621 }
 622
 623 /* Check and make sure the container has a configuration that we know CRIU can
 624  * dump. */
 625 static bool criu_ok(struct lxc_container *c, char **criu_version)
 626 {
 627         struct lxc_list *it;
 628
 629         if (!criu_version_ok(criu_version))
 630                 return false;
 631
 632         if (geteuid()) {
 633                 ERROR("Must be root to checkpoint\n");
 634                 return false;
 635         }
 636
 637         /* We only know how to restore containers with veth networks. */
 638         lxc_list_for_each(it, &c->lxc_conf->network) {
 639                 struct lxc_netdev *n = it->elem;
 640                 switch(n->type) {
 641                 case LXC_NET_VETH:
 642                 case LXC_NET_NONE:
 643                 case LXC_NET_EMPTY:
 644                 case LXC_NET_MACVLAN:
 645                         break;
 646                 default:
 647                         ERROR("Found un-dumpable network: %s (%s)\n", lxc_net_type_to_str(n->type), n->name);
 648                         return false;
 649                 }
 650         }
 651
 652         return true;
 653 }
 654
 655 static bool restore_net_info(struct lxc_container *c)
 656 {
 657         struct lxc_list *it;
 658         bool has_error = true;
 659
 660         if (container_mem_lock(c))
 661                 return false;
 662
 663         lxc_list_for_each(it, &c->lxc_conf->network) {
 664                 struct lxc_netdev *netdev = it->elem;
 665                 char template[IFNAMSIZ];
 666
 667                 if (netdev->type != LXC_NET_VETH)
 668                         continue;
 669
 670                 snprintf(template, sizeof(template), "vethXXXXXX");
 671
 672                 if (!netdev->priv.veth_attr.pair)
 673                         netdev->priv.veth_attr.pair = lxc_mkifname(template);
 674
 675                 if (!netdev->priv.veth_attr.pair)
 676                         goto out_unlock;
 677         }
 678
 679         has_error = false;
 680
 681 out_unlock:
 682         container_mem_unlock(c);
 683         return !has_error;
 684 }
 685
 686 // do_restore never returns, the calling process is used as the
 687 // monitor process. do_restore calls exit() if it fails.
 688 static void do_restore(struct lxc_container *c, int status_pipe, struct migrate_opts *opts, char *criu_version)
 689 {
 690         pid_t pid;
 691         struct lxc_handler *handler;
 692         int status, fd;
 693         int pipes[2] = {-1, -1};
 694
 695         /* Try to detach from the current controlling tty if it exists.
 696          * Othwerise, lxc_init (via lxc_console) will attach the container's
 697          * console output to the current tty, which is probably not what any
 698          * library user wants, and if they do, they can just manually configure
 699          * it :)
 700          */
 701         fd = open("/dev/tty", O_RDWR);
 702         if (fd >= 0) {
 703                 if (ioctl(fd, TIOCNOTTY, NULL) < 0)
 704                         SYSERROR("couldn't detach from tty");
 705                 close(fd);
 706         }
 707
 708         handler = lxc_init(c->name, c->lxc_conf, c->config_path);
 709         if (!handler)
 710                 goto out;
 711
 712         if (!cgroup_init(handler)) {
 713                 ERROR("failed initing cgroups");
 714                 goto out_fini_handler;
 715         }
 716
 717         if (!cgroup_create(handler)) {
 718                 ERROR("failed creating groups");
 719                 goto out_fini_handler;
 720         }
 721
 722         if (!restore_net_info(c)) {
 723                 ERROR("failed restoring network info");
 724                 goto out_fini_handler;
 725         }
 726
 727         resolve_clone_flags(handler);
 728
 729         if (pipe(pipes) < 0) {
 730                 SYSERROR("pipe() failed");
 731                 goto out_fini_handler;
 732         }
 733
 734         pid = fork();
 735         if (pid < 0)
 736                 goto out_fini_handler;
 737
 738         if (pid == 0) {
 739                 struct criu_opts os;
 740                 struct lxc_rootfs *rootfs;
 741                 int flags;
 742
 743                 close(status_pipe);
 744                 status_pipe = -1;
 745
 746                 close(pipes[0]);
 747                 pipes[0] = -1;
 748                 if (dup2(pipes[1], STDERR_FILENO) < 0) {
 749                         SYSERROR("dup2 failed");
 750                         goto out_fini_handler;
 751                 }
 752
 753                 if (dup2(pipes[1], STDOUT_FILENO) < 0) {
 754                         SYSERROR("dup2 failed");
 755                         goto out_fini_handler;
 756                 }
 757
 758                 if (unshare(CLONE_NEWNS))
 759                         goto out_fini_handler;
 760
 761                 /* CRIU needs the lxc root bind mounted so that it is the root of some
 762                  * mount. */
 763                 rootfs = &c->lxc_conf->rootfs;
 764
 765                 if (rootfs_is_blockdev(c->lxc_conf)) {
 766                         if (do_rootfs_setup(c->lxc_conf, c->name, c->config_path) < 0)
 767                                 goto out_fini_handler;
 768                 } else {
 769                         if (mkdir(rootfs->mount, 0755) < 0 && errno != EEXIST)
 770                                 goto out_fini_handler;
 771
 772                         if (mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL) < 0) {
 773                                 SYSERROR("remount / to private failed");
 774                                 goto out_fini_handler;
 775                         }
 776
 777                         if (mount(rootfs->path, rootfs->mount, NULL, MS_BIND, NULL) < 0) {
 778                                 rmdir(rootfs->mount);
 779                                 goto out_fini_handler;
 780                         }
 781                 }
 782
 783                 os.action = "restore";
 784                 os.user = opts;
 785                 os.c = c;
 786                 os.console_fd = c->lxc_conf->console.slave;
 787                 os.criu_version = criu_version;
 788                 os.handler = handler;
 789
 790                 if (os.console_fd >= 0) {
 791                         /* Twiddle the FD_CLOEXEC bit. We want to pass this FD to criu
 792                          * via --inherit-fd, so we don't want it to close.
 793                          */
 794                         flags = fcntl(os.console_fd, F_GETFD);
 795                         if (flags < 0) {
 796                                 SYSERROR("F_GETFD failed: %d", os.console_fd);
 797                                 goto out_fini_handler;
 798                         }
 799
 800                         flags &= ~FD_CLOEXEC;
 801
 802                         if (fcntl(os.console_fd, F_SETFD, flags) < 0) {
 803                                 SYSERROR("F_SETFD failed");
 804                                 goto out_fini_handler;
 805                         }
 806                 }
 807                 os.console_name = c->lxc_conf->console.name;
 808
 809                 /* exec_criu() returning is an error */
 810                 exec_criu(&os);
 811                 umount(rootfs->mount);
 812                 rmdir(rootfs->mount);
 813                 goto out_fini_handler;
 814         } else {
 815                 int ret;
 816                 char title[2048];
 817
 818                 close(pipes[1]);
 819                 pipes[1] = -1;
 820
 821                 pid_t w = waitpid(pid, &status, 0);
 822                 if (w == -1) {
 823                         SYSERROR("waitpid");
 824                         goto out_fini_handler;
 825                 }
 826
 827                 if (WIFEXITED(status)) {
 828                         char buf[4096];
 829
 830                         if (WEXITSTATUS(status)) {
 831                                 int n;
 832
 833                                 n = read(pipes[0], buf, sizeof(buf));
 834                                 if (n < 0) {
 835                                         SYSERROR("failed reading from criu stderr");
 836                                         goto out_fini_handler;
 837                                 }
 838
 839                                 buf[n] = 0;
 840
 841                                 ERROR("criu process exited %d, output:\n%s\n", WEXITSTATUS(status), buf);
 842                                 goto out_fini_handler;
 843                         } else {
 844                                 ret = snprintf(buf, sizeof(buf), "/proc/self/task/%lu/children", (unsigned long)syscall(__NR_gettid));
 845                                 if (ret < 0 || ret >= sizeof(buf)) {
 846                                         ERROR("snprintf'd too many characters: %d", ret);
 847                                         goto out_fini_handler;
 848                                 }
 849
 850                                 FILE *f = fopen(buf, "r");
 851                                 if (!f) {
 852                                         SYSERROR("couldn't read restore's children file %s\n", buf);
 853                                         goto out_fini_handler;
 854                                 }
 855
 856                                 ret = fscanf(f, "%d", (int*) &handler->pid);
 857                                 fclose(f);
 858                                 if (ret != 1) {
 859                                         ERROR("reading restore pid failed");
 860                                         goto out_fini_handler;
 861                                 }
 862
 863                                 if (lxc_set_state(c->name, handler, RUNNING)) {
 864                                         ERROR("error setting running state after restore");
 865                                         goto out_fini_handler;
 866                                 }
 867                         }
 868                 } else {
 869                         ERROR("CRIU was killed with signal %d\n", WTERMSIG(status));
 870                         goto out_fini_handler;
 871                 }
 872
 873                 close(pipes[0]);
 874
 875                 ret = write(status_pipe, &status, sizeof(status));
 876                 close(status_pipe);
 877                 status_pipe = -1;
 878
 879                 if (sizeof(status) != ret) {
 880                         SYSERROR("failed to write all of status");
 881                         goto out_fini_handler;
 882                 }
 883
 884                 /*
 885                  * See comment in lxcapi_start; we don't care if these
 886                  * fail because it's just a beauty thing. We just
 887                  * assign the return here to silence potential.
 888                  */
 889                 ret = snprintf(title, sizeof(title), "[lxc monitor] %s %s", c->config_path, c->name);
 890                 ret = setproctitle(title);
 891
 892                 ret = lxc_poll(c->name, handler);
 893                 if (ret)
 894                         lxc_abort(c->name, handler);
 895                 lxc_fini(c->name, handler);
 896                 exit(ret);
 897         }
 898
 899 out_fini_handler:
 900         if (pipes[0] >= 0)
 901                 close(pipes[0]);
 902         if (pipes[1] >= 0)
 903                 close(pipes[1]);
 904
 905         lxc_fini(c->name, handler);
 906
 907 out:
 908         if (status_pipe >= 0) {
 909                 /* ensure getting here was a failure, e.g. if we failed to
 910                  * parse the child pid or something, even after a successful
 911                  * restore
 912                  */
 913                 if (!status)
 914                         status = 1;
 915                 if (write(status_pipe, &status, sizeof(status)) != sizeof(status)) {
 916                         SYSERROR("writing status failed");
 917                 }
 918                 close(status_pipe);
 919         }
 920
 921         exit(1);
 922 }
 923
 924 static int save_tty_major_minor(char *directory, struct lxc_container *c, char *tty_id, int len)
 925 {
 926         FILE *f;
 927         char path[PATH_MAX];
 928         int ret;
 929         struct stat sb;
 930
 931         if (c->lxc_conf->console.path && !strcmp(c->lxc_conf->console.path, "none")) {
 932                 tty_id[0] = 0;
 933                 return 0;
 934         }
 935
 936         ret = snprintf(path, sizeof(path), "/proc/%d/root/dev/console", c->init_pid(c));
 937         if (ret < 0 || ret >= sizeof(path)) {
 938                 ERROR("snprintf'd too many chacters: %d", ret);
 939                 return -1;
 940         }
 941
 942         ret = stat(path, &sb);
 943         if (ret < 0) {
 944                 SYSERROR("stat of %s failed", path);
 945                 return -1;
 946         }
 947
 948         ret = snprintf(path, sizeof(path), "%s/tty.info", directory);
 949         if (ret < 0 || ret >= sizeof(path)) {
 950                 ERROR("snprintf'd too many characters: %d", ret);
 951                 return -1;
 952         }
 953
 954         ret = snprintf(tty_id, len, "tty[%llx:%llx]",
 955                                         (long long unsigned) sb.st_rdev,
 956                                         (long long unsigned) sb.st_dev);
 957         if (ret < 0 || ret >= sizeof(path)) {
 958                 ERROR("snprintf'd too many characters: %d", ret);
 959                 return -1;
 960         }
 961
 962         f = fopen(path, "w");
 963         if (!f) {
 964                 SYSERROR("failed to open %s", path);
 965                 return -1;
 966         }
 967
 968         ret = fprintf(f, "%s", tty_id);
 969         fclose(f);
 970         if (ret < 0)
 971                 SYSERROR("failed to write to %s", path);
 972         return ret;
 973 }
 974
 975 /* do one of either predump or a regular dump */
 976 static bool do_dump(struct lxc_container *c, char *mode, struct migrate_opts *opts)
 977 {
 978         pid_t pid;
 979         char *criu_version = NULL;
 980
 981         if (!criu_ok(c, &criu_version))
 982                 return false;
 983
 984         if (mkdir_p(opts->directory, 0700) < 0)
 985                 return false;
 986
 987         pid = fork();
 988         if (pid < 0) {
 989                 SYSERROR("fork failed");
 990                 return false;
 991         }
 992
 993         if (pid == 0) {
 994                 struct criu_opts os;
 995                 struct lxc_handler h;
 996
 997                 h.name = c->name;
 998                 if (!cgroup_init(&h)) {
 999                         ERROR("failed to cgroup_init()");
1000                         exit(1);
1001                 }
1002
1003                 os.action = mode;
1004                 os.user = opts;
1005                 os.c = c;
1006                 os.console_name = c->lxc_conf->console.path;
1007                 os.criu_version = criu_version;
1008
1009                 if (save_tty_major_minor(opts->directory, c, os.tty_id, sizeof(os.tty_id)) < 0)
1010                         exit(1);
1011
1012                 /* exec_criu() returning is an error */
1013                 exec_criu(&os);
1014                 exit(1);
1015         } else {
1016                 int status;
1017                 pid_t w = waitpid(pid, &status, 0);
1018                 if (w == -1) {
1019                         SYSERROR("waitpid");
1020                         return false;
1021                 }
1022
1023                 if (WIFEXITED(status)) {
1024                         if (WEXITSTATUS(status)) {
1025                                 ERROR("dump failed with %d\n", WEXITSTATUS(status));
1026                                 return false;
1027                         }
1028
1029                         return true;
1030                 } else if (WIFSIGNALED(status)) {
1031                         ERROR("dump signaled with %d\n", WTERMSIG(status));
1032                         return false;
1033                 } else {
1034                         ERROR("unknown dump exit %d\n", status);
1035                         return false;
1036                 }
1037         }
1038 }
1039
1040 bool __criu_pre_dump(struct lxc_container *c, struct migrate_opts *opts)
1041 {
1042         return do_dump(c, "pre-dump", opts);
1043 }
1044
1045 bool __criu_dump(struct lxc_container *c, struct migrate_opts *opts)
1046 {
1047         char path[PATH_MAX];
1048         int ret;
1049
1050         ret = snprintf(path, sizeof(path), "%s/inventory.img", opts->directory);
1051         if (ret < 0 || ret >= sizeof(path))
1052                 return false;
1053
1054         if (access(path, F_OK) == 0) {
1055                 ERROR("please use a fresh directory for the dump directory\n");
1056                 return false;
1057         }
1058
1059         return do_dump(c, "dump", opts);
1060 }
1061
1062 bool __criu_restore(struct lxc_container *c, struct migrate_opts *opts)
1063 {
1064         pid_t pid;
1065         int status, nread;
1066         int pipefd[2];
1067         char *criu_version = NULL;
1068
1069         if (!criu_ok(c, &criu_version))
1070                 return false;
1071
1072         if (geteuid()) {
1073                 ERROR("Must be root to restore\n");
1074                 return false;
1075         }
1076
1077         if (pipe(pipefd)) {
1078                 ERROR("failed to create pipe");
1079                 return false;
1080         }
1081
1082         pid = fork();
1083         if (pid < 0) {
1084                 close(pipefd[0]);
1085                 close(pipefd[1]);
1086                 return false;
1087         }
1088
1089         if (pid == 0) {
1090                 close(pipefd[0]);
1091                 // this never returns
1092                 do_restore(c, pipefd[1], opts, criu_version);
1093         }
1094
1095         close(pipefd[1]);
1096
1097         nread = read(pipefd[0], &status, sizeof(status));
1098         close(pipefd[0]);
1099         if (sizeof(status) != nread) {
1100                 ERROR("reading status from pipe failed");
1101                 goto err_wait;
1102         }
1103
1104         // If the criu process was killed or exited nonzero, wait() for the
1105         // handler, since the restore process died. Otherwise, we don't need to
1106         // wait, since the child becomes the monitor process.
1107         if (!WIFEXITED(status) || WEXITSTATUS(status))
1108                 goto err_wait;
1109         return true;
1110
1111 err_wait:
1112         if (wait_for_pid(pid))
1113                 ERROR("restore process died");
1114         return false;
1115 }