src/lxc/start.c

   1 /*
   2  * lxc: linux Container library
   3  *
   4  * (C) Copyright IBM Corp. 2007, 2008
   5  *
   6  * Authors:
   7  * Daniel Lezcano <daniel.lezcano at free.fr>
   8  * Serge Hallyn <serge@hallyn.com>
   9  * Christian Brauner <christian.brauner@ubuntu.com>
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2.1 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  24  */
  25
  26 #define _GNU_SOURCE
  27 #include "config.h"
  28
  29 #include <alloca.h>
  30 #include <dirent.h>
  31 #include <errno.h>
  32 #include <fcntl.h>
  33 #include <grp.h>
  34 #include <poll.h>
  35 #include <signal.h>
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <sys/file.h>
  41 #include <sys/mount.h>
  42 #include <sys/param.h>
  43 #include <sys/prctl.h>
  44 #include <sys/socket.h>
  45 #include <sys/stat.h>
  46 #include <sys/syscall.h>
  47 #include <sys/types.h>
  48 #include <sys/un.h>
  49 #include <sys/wait.h>
  50
  51 #if HAVE_LIBCAP
  52 #include <sys/capability.h>
  53 #endif
  54
  55 #if !HAVE_DECL_PR_CAPBSET_DROP
  56 #define PR_CAPBSET_DROP 24
  57 #endif
  58
  59 #if !HAVE_DECL_PR_SET_NO_NEW_PRIVS
  60 #define PR_SET_NO_NEW_PRIVS 38
  61 #endif
  62
  63 #if !HAVE_DECL_PR_GET_NO_NEW_PRIVS
  64 #define PR_GET_NO_NEW_PRIVS 39
  65 #endif
  66
  67 #include "af_unix.h"
  68 #include "caps.h"
  69 #include "cgroup.h"
  70 #include "commands.h"
  71 #include "commands_utils.h"
  72 #include "conf.h"
  73 #include "confile_utils.h"
  74 #include "console.h"
  75 #include "error.h"
  76 #include "log.h"
  77 #include "lxccontainer.h"
  78 #include "lxclock.h"
  79 #include "lxcseccomp.h"
  80 #include "mainloop.h"
  81 #include "monitor.h"
  82 #include "namespace.h"
  83 #include "network.h"
  84 #include "start.h"
  85 #include "storage.h"
  86 #include "storage_utils.h"
  87 #include "sync.h"
  88 #include "utils.h"
  89 #include "lsm/lsm.h"
  90
  91 lxc_log_define(lxc_start, lxc);
  92
  93 extern void mod_all_rdeps(struct lxc_container *c, bool inc);
  94 static bool do_destroy_container(struct lxc_handler *handler);
  95 static int lxc_rmdir_onedev_wrapper(void *data);
  96 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
  97                                             const char *name);
  98
  99 static void print_top_failing_dir(const char *path)
 100 {
 101         size_t len = strlen(path);
 102         char *copy = alloca(len + 1), *p, *e, saved;
 103         strcpy(copy, path);
 104
 105         p = copy;
 106         e = copy + len;
 107         while (p < e) {
 108                 while (p < e && *p == '/')
 109                         p++;
 110                 while (p < e && *p != '/')
 111                         p++;
 112                 saved = *p;
 113                 *p = '\0';
 114                 if (access(copy, X_OK)) {
 115                         SYSERROR("Could not access %s. Please grant it x "
 116                                  "access, or add an ACL for the container "
 117                                  "root.", copy);
 118                         return;
 119                 }
 120                 *p = saved;
 121         }
 122 }
 123
 124 static void close_ns(int ns_fd[LXC_NS_MAX])
 125 {
 126         int i;
 127
 128         for (i = 0; i < LXC_NS_MAX; i++) {
 129                 if (ns_fd[i] > -1) {
 130                         close(ns_fd[i]);
 131                         ns_fd[i] = -1;
 132                 }
 133         }
 134 }
 135
 136 /* preserve_ns: open /proc/@pid/ns/@ns for each namespace specified
 137  * in clone_flags.
 138  * Return true on success, false on failure.
 139  */
 140 static bool preserve_ns(int ns_fd[LXC_NS_MAX], int clone_flags, pid_t pid)
 141 {
 142         int i, ret;
 143
 144         for (i = 0; i < LXC_NS_MAX; i++)
 145                 ns_fd[i] = -1;
 146
 147         ret = lxc_preserve_ns(pid, "");
 148         if (ret < 0) {
 149                 SYSERROR("Kernel does not support attaching to namespaces.");
 150                 return false;
 151         } else {
 152                 close(ret);
 153         }
 154
 155         for (i = 0; i < LXC_NS_MAX; i++) {
 156                 if ((clone_flags & ns_info[i].clone_flag) == 0)
 157                         continue;
 158
 159                 ns_fd[i] = lxc_preserve_ns(pid, ns_info[i].proc_name);
 160                 if (ns_fd[i] < 0)
 161                         goto error;
 162
 163                 DEBUG("Preserved %s namespace via fd %d", ns_info[i].proc_name, ns_fd[i]);
 164         }
 165
 166         return true;
 167
 168 error:
 169         if (errno == ENOENT)
 170                 SYSERROR("Kernel does not support attaching to %s namespaces.", ns_info[i].proc_name);
 171         else
 172                 SYSERROR("Failed to open file descriptor for %s namespace: %s.", ns_info[i].proc_name, strerror(errno));
 173         close_ns(ns_fd);
 174         return false;
 175 }
 176
 177 static int match_fd(int fd)
 178 {
 179         return (fd == 0 || fd == 1 || fd == 2);
 180 }
 181
 182 int lxc_check_inherited(struct lxc_conf *conf, bool closeall,
 183                         int *fds_to_ignore, size_t len_fds)
 184 {
 185         struct dirent *direntp;
 186         int fd, fddir;
 187         size_t i;
 188         DIR *dir;
 189
 190         if (conf && conf->close_all_fds)
 191                 closeall = true;
 192
 193 restart:
 194         dir = opendir("/proc/self/fd");
 195         if (!dir) {
 196                 WARN("Failed to open directory: %s.", strerror(errno));
 197                 return -1;
 198         }
 199
 200         fddir = dirfd(dir);
 201
 202         while ((direntp = readdir(dir))) {
 203                 struct lxc_list *cur;
 204                 bool matched = false;
 205
 206                 if (!strcmp(direntp->d_name, "."))
 207                         continue;
 208
 209                 if (!strcmp(direntp->d_name, ".."))
 210                         continue;
 211
 212                 if (lxc_safe_int(direntp->d_name, &fd) < 0) {
 213                         INFO("Could not parse file descriptor for: %s", direntp->d_name);
 214                         continue;
 215                 }
 216
 217                 for (i = 0; i < len_fds; i++)
 218                         if (fds_to_ignore[i] == fd)
 219                                 break;
 220
 221                 if (fd == fddir || fd == lxc_log_fd ||
 222                     (i < len_fds && fd == fds_to_ignore[i]))
 223                         continue;
 224
 225                 /* Keep state clients that wait on reboots. */
 226                 if (conf) {
 227                         lxc_list_for_each(cur, &conf->state_clients) {
 228                                 struct lxc_state_client *client = cur->elem;
 229
 230                                 if (client->clientfd != fd)
 231                                         continue;
 232
 233                                 matched = true;
 234                                 break;
 235                         }
 236                 }
 237
 238                 if (matched)
 239                         continue;
 240
 241                 if (current_config && fd == current_config->logfd)
 242                         continue;
 243
 244                 if (match_fd(fd))
 245                         continue;
 246
 247                 if (closeall) {
 248                         close(fd);
 249                         closedir(dir);
 250                         INFO("Closed inherited fd %d", fd);
 251                         goto restart;
 252                 }
 253                 WARN("Inherited fd %d", fd);
 254         }
 255
 256         /* Only enable syslog at this point to avoid the above logging function
 257          * to open a new fd and make the check_inherited function enter an
 258          * infinite loop.
 259          */
 260         lxc_log_enable_syslog();
 261
 262         closedir(dir); /* cannot fail */
 263         return 0;
 264 }
 265
 266 static int setup_signal_fd(sigset_t *oldmask)
 267 {
 268         sigset_t mask;
 269         int fd;
 270
 271         /* Block everything except serious error signals. */
 272         if (sigfillset(&mask) ||
 273             sigdelset(&mask, SIGILL) ||
 274             sigdelset(&mask, SIGSEGV) ||
 275             sigdelset(&mask, SIGBUS) ||
 276             sigdelset(&mask, SIGWINCH) ||
 277             sigprocmask(SIG_BLOCK, &mask, oldmask)) {
 278                 SYSERROR("Failed to set signal mask.");
 279                 return -1;
 280         }
 281
 282         fd = signalfd(-1, &mask, 0);
 283         if (fd < 0) {
 284                 SYSERROR("Failed to create signal file descriptor.");
 285                 return -1;
 286         }
 287
 288         if (fcntl(fd, F_SETFD, FD_CLOEXEC)) {
 289                 SYSERROR("Failed to set FD_CLOEXEC on the signal file descriptor: %d.", fd);
 290                 close(fd);
 291                 return -1;
 292         }
 293
 294         DEBUG("Set SIGCHLD handler with file descriptor: %d.", fd);
 295
 296         return fd;
 297 }
 298
 299 static int signal_handler(int fd, uint32_t events, void *data,
 300                           struct lxc_epoll_descr *descr)
 301 {
 302         struct signalfd_siginfo siginfo;
 303         siginfo_t info;
 304         int ret;
 305         pid_t *pid = data;
 306         bool init_died = false;
 307
 308         ret = read(fd, &siginfo, sizeof(siginfo));
 309         if (ret < 0) {
 310                 ERROR("Failed to read signal info from signal file descriptor: %d.", fd);
 311                 return -1;
 312         }
 313
 314         if (ret != sizeof(siginfo)) {
 315                 ERROR("Unexpected size for siginfo struct.");
 316                 return -1;
 317         }
 318
 319         /* Check whether init is running. */
 320         info.si_pid = 0;
 321         ret = waitid(P_PID, *pid, &info, WEXITED | WNOWAIT | WNOHANG);
 322         if (ret == 0 && info.si_pid == *pid)
 323                 init_died = true;
 324
 325         if (siginfo.ssi_signo != SIGCHLD) {
 326                 kill(*pid, siginfo.ssi_signo);
 327                 INFO("Forwarded signal %d to pid %d.", siginfo.ssi_signo, *pid);
 328                 return init_died ? 1 : 0;
 329         }
 330
 331         if (siginfo.ssi_code == CLD_STOPPED) {
 332                 INFO("Container init process was stopped.");
 333                 return init_died ? 1 : 0;
 334         } else if (siginfo.ssi_code == CLD_CONTINUED) {
 335                 INFO("Container init process was continued.");
 336                 return init_died ? 1 : 0;
 337         }
 338
 339         /* More robustness, protect ourself from a SIGCHLD sent
 340          * by a process different from the container init.
 341          */
 342         if (siginfo.ssi_pid != *pid) {
 343                 NOTICE("Received SIGCHLD from pid %d instead of container init %d.", siginfo.ssi_pid, *pid);
 344                 return init_died ? 1 : 0;
 345         }
 346
 347         DEBUG("Container init process %d exited.", *pid);
 348         return 1;
 349 }
 350
 351 static int lxc_serve_state_clients(const char *name,
 352                                    struct lxc_handler *handler,
 353                                    lxc_state_t state)
 354 {
 355         ssize_t ret;
 356         struct lxc_list *cur, *next;
 357         struct lxc_state_client *client;
 358         struct lxc_msg msg = {.type = lxc_msg_state, .value = state};
 359
 360         process_lock();
 361         handler->state = state;
 362         TRACE("Set container state to %s", lxc_state2str(state));
 363
 364         if (lxc_list_empty(&handler->conf->state_clients)) {
 365                 TRACE("No state clients registered");
 366                 process_unlock();
 367                 lxc_monitor_send_state(name, state, handler->lxcpath);
 368                 return 0;
 369         }
 370
 371         strncpy(msg.name, name, sizeof(msg.name));
 372         msg.name[sizeof(msg.name) - 1] = 0;
 373
 374         lxc_list_for_each_safe(cur, &handler->conf->state_clients, next) {
 375                 client = cur->elem;
 376
 377                 if (client->states[state] == 0) {
 378                         TRACE("State %s not registered for state client %d",
 379                               lxc_state2str(state), client->clientfd);
 380                         continue;
 381                 }
 382
 383                 TRACE("Sending state %s to state client %d",
 384                       lxc_state2str(state), client->clientfd);
 385
 386         again:
 387                 ret = send(client->clientfd, &msg, sizeof(msg), 0);
 388                 if (ret <= 0) {
 389                         if (errno == EINTR) {
 390                                 TRACE("Caught EINTR; retrying");
 391                                 goto again;
 392                         }
 393
 394                         ERROR("%s - Failed to send message to client",
 395                               strerror(errno));
 396                 }
 397
 398                 /* kick client from list */
 399                 close(client->clientfd);
 400                 lxc_list_del(cur);
 401                 free(cur->elem);
 402                 free(cur);
 403         }
 404         process_unlock();
 405
 406         return 0;
 407 }
 408
 409 static int lxc_serve_state_socket_pair(const char *name,
 410                                        struct lxc_handler *handler,
 411                                        lxc_state_t state)
 412 {
 413         ssize_t ret;
 414
 415         if (!handler->backgrounded ||
 416             handler->state_socket_pair[1] < 0 ||
 417             state == STARTING)
 418                 return 0;
 419
 420         /* Close read end of the socket pair. */
 421         close(handler->state_socket_pair[0]);
 422         handler->state_socket_pair[0] = -1;
 423
 424 again:
 425         ret = lxc_abstract_unix_send_credential(handler->state_socket_pair[1],
 426                                                 &(int){state}, sizeof(int));
 427         if (ret != sizeof(int)) {
 428                 if (errno == EINTR)
 429                         goto again;
 430                 SYSERROR("Failed to send state to %d",
 431                          handler->state_socket_pair[1]);
 432                 return -1;
 433         }
 434
 435         TRACE("Sent container state \"%s\" to %d", lxc_state2str(state),
 436               handler->state_socket_pair[1]);
 437
 438         /* Close write end of the socket pair. */
 439         close(handler->state_socket_pair[1]);
 440         handler->state_socket_pair[1] = -1;
 441
 442         return 0;
 443 }
 444
 445 int lxc_set_state(const char *name, struct lxc_handler *handler,
 446                   lxc_state_t state)
 447 {
 448         int ret;
 449
 450         ret = lxc_serve_state_socket_pair(name, handler, state);
 451         if (ret < 0) {
 452                 ERROR("Failed to synchronize via anonymous pair of unix sockets");
 453                 return -1;
 454         }
 455
 456         ret = lxc_serve_state_clients(name, handler, state);
 457         if (ret < 0)
 458                 return -1;
 459
 460         /* This function will try to connect to the legacy lxc-monitord state
 461          * server and only exists for backwards compatibility.
 462          */
 463         lxc_monitor_send_state(name, state, handler->lxcpath);
 464
 465         return 0;
 466 }
 467
 468 int lxc_poll(const char *name, struct lxc_handler *handler)
 469 {
 470         int sigfd = handler->sigfd;
 471         int pid = handler->pid;
 472         struct lxc_epoll_descr descr;
 473
 474         if (lxc_mainloop_open(&descr)) {
 475                 ERROR("Failed to create LXC mainloop.");
 476                 goto out_sigfd;
 477         }
 478
 479         if (lxc_mainloop_add_handler(&descr, sigfd, signal_handler, &pid)) {
 480                 ERROR("Failed to add signal handler with file descriptor %d to LXC mainloop.", sigfd);
 481                 goto out_mainloop_open;
 482         }
 483
 484         if (lxc_console_mainloop_add(&descr, handler->conf)) {
 485                 ERROR("Failed to add console handler to LXC mainloop.");
 486                 goto out_mainloop_open;
 487         }
 488
 489         if (lxc_cmd_mainloop_add(name, &descr, handler)) {
 490                 ERROR("Failed to add command handler to LXC mainloop.");
 491                 goto out_mainloop_open;
 492         }
 493
 494         TRACE("lxc mainloop is ready");
 495
 496         return lxc_mainloop(&descr, -1);
 497
 498 out_mainloop_open:
 499         lxc_mainloop_close(&descr);
 500
 501 out_sigfd:
 502         close(sigfd);
 503
 504         return -1;
 505 }
 506
 507 void lxc_zero_handler(struct lxc_handler *handler)
 508 {
 509         int i;
 510
 511         memset(handler, 0, sizeof(struct lxc_handler));
 512
 513         handler->clone_flags = -1;
 514
 515         handler->pinfd = -1;
 516
 517         handler->sigfd = -1;
 518
 519         for (i = 0; i < LXC_NS_MAX; i++)
 520                 handler->nsfd[i] = -1;
 521
 522         handler->data_sock[0] = -1;
 523         handler->data_sock[1] = -1;
 524
 525         handler->state_socket_pair[0] = -1;
 526         handler->state_socket_pair[1] = -1;
 527
 528         handler->sync_sock[0] = -1;
 529         handler->sync_sock[1] = -1;
 530 }
 531
 532 void lxc_free_handler(struct lxc_handler *handler)
 533 {
 534         if (handler->conf && handler->conf->maincmd_fd)
 535                 close(handler->conf->maincmd_fd);
 536
 537         if (handler->state_socket_pair[0] >= 0)
 538                 close(handler->state_socket_pair[0]);
 539
 540         if (handler->state_socket_pair[1] >= 0)
 541                 close(handler->state_socket_pair[1]);
 542
 543         handler->conf = NULL;
 544         free(handler);
 545 }
 546
 547 struct lxc_handler *lxc_init_handler(const char *name, struct lxc_conf *conf,
 548                                      const char *lxcpath, bool daemonize)
 549 {
 550         int i, ret;
 551         struct lxc_handler *handler;
 552
 553         handler = malloc(sizeof(*handler));
 554         if (!handler) {
 555                 ERROR("failed to allocate memory");
 556                 return NULL;
 557         }
 558
 559         memset(handler, 0, sizeof(*handler));
 560
 561         /* Note that am_unpriv() checks the effective uid. We probably don't
 562          * care if we are real root only if we are running as root so this
 563          * should be fine.
 564          */
 565         handler->am_root = !am_unpriv();
 566         handler->data_sock[0] = handler->data_sock[1] = -1;
 567         handler->conf = conf;
 568         handler->lxcpath = lxcpath;
 569         handler->pinfd = -1;
 570         handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
 571         if (handler->conf->reboot == 0)
 572                 lxc_list_init(&handler->conf->state_clients);
 573
 574         for (i = 0; i < LXC_NS_MAX; i++)
 575                 handler->nsfd[i] = -1;
 576
 577         handler->name = name;
 578
 579         if (daemonize && !handler->conf->reboot) {
 580                 /* Create socketpair() to synchronize on daemonized startup.
 581                  * When the container reboots we don't need to synchronize again
 582                  * currently so don't open another socketpair().
 583                  */
 584                 ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0,
 585                                  handler->state_socket_pair);
 586                 if (ret < 0) {
 587                         ERROR("Failed to create anonymous pair of unix sockets");
 588                         goto on_error;
 589                 }
 590                 TRACE("Created anonymous pair {%d,%d} of unix sockets",
 591                       handler->state_socket_pair[0],
 592                       handler->state_socket_pair[1]);
 593         }
 594
 595         if (handler->conf->reboot == 0) {
 596                 handler->conf->maincmd_fd = lxc_cmd_init(name, lxcpath, "command");
 597                 if (handler->conf->maincmd_fd < 0) {
 598                         ERROR("Failed to set up command socket");
 599                         goto on_error;
 600                 }
 601         }
 602         TRACE("Unix domain socket %d for command server is ready",
 603               handler->conf->maincmd_fd);
 604
 605         return handler;
 606
 607 on_error:
 608         lxc_free_handler(handler);
 609
 610         return NULL;
 611 }
 612
 613 int lxc_init(const char *name, struct lxc_handler *handler)
 614 {
 615         const char *loglevel;
 616         struct lxc_conf *conf = handler->conf;
 617
 618         lsm_init();
 619         TRACE("initialized LSM");
 620
 621         if (lxc_read_seccomp_config(conf) != 0) {
 622                 ERROR("Failed loading seccomp policy.");
 623                 goto out_close_maincmd_fd;
 624         }
 625         TRACE("read seccomp policy");
 626
 627         /* Begin by setting the state to STARTING. */
 628         if (lxc_set_state(name, handler, STARTING)) {
 629                 ERROR("Failed to set state for container \"%s\" to \"%s\".", name, lxc_state2str(STARTING));
 630                 goto out_close_maincmd_fd;
 631         }
 632         TRACE("set container state to \"STARTING\"");
 633
 634         /* Start of environment variable setup for hooks. */
 635         if (name && setenv("LXC_NAME", name, 1))
 636                 SYSERROR("Failed to set environment variable: LXC_NAME=%s.", name);
 637
 638         if (conf->rcfile && setenv("LXC_CONFIG_FILE", conf->rcfile, 1))
 639                 SYSERROR("Failed to set environment variable: LXC_CONFIG_FILE=%s.", conf->rcfile);
 640
 641         if (conf->rootfs.mount && setenv("LXC_ROOTFS_MOUNT", conf->rootfs.mount, 1))
 642                 SYSERROR("Failed to set environment variable: LXC_ROOTFS_MOUNT=%s.", conf->rootfs.mount);
 643
 644         if (conf->rootfs.path && setenv("LXC_ROOTFS_PATH", conf->rootfs.path, 1))
 645                 SYSERROR("Failed to set environment variable: LXC_ROOTFS_PATH=%s.", conf->rootfs.path);
 646
 647         if (conf->console.path && setenv("LXC_CONSOLE", conf->console.path, 1))
 648                 SYSERROR("Failed to set environment variable: LXC_CONSOLE=%s.", conf->console.path);
 649
 650         if (conf->console.log_path && setenv("LXC_CONSOLE_LOGPATH", conf->console.log_path, 1))
 651                 SYSERROR("Failed to set environment variable: LXC_CONSOLE_LOGPATH=%s.", conf->console.log_path);
 652
 653         if (setenv("LXC_CGNS_AWARE", "1", 1))
 654                 SYSERROR("Failed to set environment variable LXC_CGNS_AWARE=1.");
 655
 656         loglevel = lxc_log_priority_to_string(lxc_log_get_level());
 657         if (setenv("LXC_LOG_LEVEL", loglevel, 1))
 658                 SYSERROR("Failed to set environment variable LXC_LOG_LEVEL=%s", loglevel);
 659         /* End of environment variable setup for hooks. */
 660
 661         TRACE("set environment variables");
 662
 663         if (run_lxc_hooks(name, "pre-start", conf, handler->lxcpath, NULL)) {
 664                 ERROR("Failed to run lxc.hook.pre-start for container \"%s\".", name);
 665                 goto out_aborting;
 666         }
 667         TRACE("ran pre-start hooks");
 668
 669         /* The signal fd has to be created before forking otherwise if the child
 670          * process exits before we setup the signal fd, the event will be lost
 671          * and the command will be stuck.
 672          */
 673         handler->sigfd = setup_signal_fd(&handler->oldmask);
 674         if (handler->sigfd < 0) {
 675                 ERROR("Failed to setup SIGCHLD fd handler.");
 676                 goto out_delete_tty;
 677         }
 678         TRACE("set up signal fd");
 679
 680         /* Do this after setting up signals since it might unblock SIGWINCH. */
 681         if (lxc_console_create(conf)) {
 682                 ERROR("Failed to create console for container \"%s\".", name);
 683                 goto out_restore_sigmask;
 684         }
 685         TRACE("created console");
 686
 687         if (lxc_ttys_shift_ids(conf) < 0) {
 688                 ERROR("Failed to shift tty into container.");
 689                 goto out_restore_sigmask;
 690         }
 691         TRACE("shifted tty ids");
 692
 693         INFO("container \"%s\" is initialized", name);
 694         return 0;
 695
 696 out_restore_sigmask:
 697         sigprocmask(SIG_SETMASK, &handler->oldmask, NULL);
 698 out_delete_tty:
 699         lxc_delete_tty(&conf->tty_info);
 700 out_aborting:
 701         lxc_set_state(name, handler, ABORTING);
 702 out_close_maincmd_fd:
 703         close(conf->maincmd_fd);
 704         conf->maincmd_fd = -1;
 705         return -1;
 706 }
 707
 708 void lxc_fini(const char *name, struct lxc_handler *handler)
 709 {
 710         int i, rc;
 711         struct lxc_list *cur, *next;
 712         pid_t self = getpid();
 713         char *namespaces[LXC_NS_MAX + 1];
 714         size_t namespace_count = 0;
 715
 716         /* The STOPPING state is there for future cleanup code which can take
 717          * awhile.
 718          */
 719         lxc_set_state(name, handler, STOPPING);
 720
 721         for (i = 0; i < LXC_NS_MAX; i++) {
 722                 if (handler->nsfd[i] != -1) {
 723                         rc = asprintf(&namespaces[namespace_count], "%s:/proc/%d/fd/%d",
 724                                       ns_info[i].proc_name, self, handler->nsfd[i]);
 725                         if (rc == -1) {
 726                                 SYSERROR("Failed to allocate memory.");
 727                                 break;
 728                         }
 729                         ++namespace_count;
 730                 }
 731         }
 732         namespaces[namespace_count] = NULL;
 733
 734         if (handler->conf->reboot && setenv("LXC_TARGET", "reboot", 1))
 735                 SYSERROR("Failed to set environment variable: LXC_TARGET=reboot.");
 736
 737         if (!handler->conf->reboot && setenv("LXC_TARGET", "stop", 1))
 738                 SYSERROR("Failed to set environment variable: LXC_TARGET=stop.");
 739
 740         if (run_lxc_hooks(name, "stop", handler->conf, handler->lxcpath, namespaces))
 741                 ERROR("Failed to run lxc.hook.stop for container \"%s\".", name);
 742
 743         while (namespace_count--)
 744                 free(namespaces[namespace_count]);
 745
 746         for (i = 0; i < LXC_NS_MAX; i++) {
 747                 if (handler->nsfd[i] < 0)
 748                         continue;
 749
 750                 close(handler->nsfd[i]);
 751                 handler->nsfd[i] = -1;
 752         }
 753
 754         cgroup_destroy(handler);
 755
 756         /* This function will try to connect to the legacy lxc-monitord state
 757          * server and only exists for backwards compatibility.
 758          */
 759         lxc_monitor_send_state(name, STOPPED, handler->lxcpath);
 760
 761         if (handler->conf->reboot == 0) {
 762                 /* For all new state clients simply close the command socket.
 763                  * This will inform all state clients that the container is
 764                  * STOPPED and also prevents a race between a open()/close() on
 765                  * the command socket causing a new process to get ECONNREFUSED
 766                  * because we haven't yet closed the command socket.
 767                  */
 768                 close(handler->conf->maincmd_fd);
 769                 handler->conf->maincmd_fd = -1;
 770         } else {
 771                 lxc_set_state(name, handler, STOPPED);
 772         }
 773
 774         if (run_lxc_hooks(name, "post-stop", handler->conf, handler->lxcpath, NULL)) {
 775                 ERROR("Failed to run lxc.hook.post-stop for container \"%s\".", name);
 776                 if (handler->conf->reboot) {
 777                         WARN("Container will be stopped instead of rebooted.");
 778                         handler->conf->reboot = 0;
 779                         if (setenv("LXC_TARGET", "stop", 1))
 780                                 WARN("Failed to set environment variable: LXC_TARGET=stop.");
 781                 }
 782         }
 783
 784         /* Reset mask set by setup_signal_fd. */
 785         if (sigprocmask(SIG_SETMASK, &handler->oldmask, NULL))
 786                 WARN("Failed to restore signal mask.");
 787
 788         lxc_console_delete(&handler->conf->console);
 789         lxc_delete_tty(&handler->conf->tty_info);
 790
 791         /* The command socket is now closed, no more state clients can register
 792          * themselves from now on. So free the list of state clients.
 793          */
 794         lxc_list_for_each_safe(cur, &handler->conf->state_clients, next) {
 795                 struct lxc_state_client *client = cur->elem;
 796
 797                 /* Keep state clients that want to be notified about reboots. */
 798                 if ((handler->conf->reboot > 0) && (client->states[RUNNING] == 2))
 799                         continue;
 800
 801                 /* close state client socket */
 802                 close(client->clientfd);
 803                 lxc_list_del(cur);
 804                 free(cur->elem);
 805                 free(cur);
 806         }
 807
 808         if (handler->data_sock[0] != -1) {
 809                 close(handler->data_sock[0]);
 810                 close(handler->data_sock[1]);
 811         }
 812
 813         if (handler->conf->ephemeral == 1 && handler->conf->reboot != 1)
 814                 lxc_destroy_container_on_signal(handler, name);
 815
 816         free(handler);
 817 }
 818
 819 void lxc_abort(const char *name, struct lxc_handler *handler)
 820 {
 821         int ret, status;
 822
 823         lxc_set_state(name, handler, ABORTING);
 824         if (handler->pid > 0)
 825                 kill(handler->pid, SIGKILL);
 826         while ((ret = waitpid(-1, &status, 0)) > 0) {
 827                 ;
 828         }
 829 }
 830
 831 static int do_start(void *data)
 832 {
 833         int fd, ret;
 834         struct lxc_list *iterator;
 835         char path[PATH_MAX];
 836         struct lxc_handler *handler = data;
 837         bool have_cap_setgid;
 838         uid_t new_uid;
 839         gid_t new_gid;
 840         int devnull_fd = -1;
 841
 842         if (sigprocmask(SIG_SETMASK, &handler->oldmask, NULL)) {
 843                 SYSERROR("Failed to set signal mask.");
 844                 return -1;
 845         }
 846
 847         /* This prctl must be before the synchro, so if the parent dies before
 848          * we set the parent death signal, we will detect its death with the
 849          * synchro right after, otherwise we have a window where the parent can
 850          * exit before we set the pdeath signal leading to a unsupervized
 851          * container.
 852          */
 853         if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0)) {
 854                 SYSERROR("Failed to set PR_SET_PDEATHSIG to SIGKILL.");
 855                 return -1;
 856         }
 857
 858         lxc_sync_fini_parent(handler);
 859
 860         /* Don't leak the pinfd to the container. */
 861         if (handler->pinfd >= 0)
 862                 close(handler->pinfd);
 863
 864         if (lxc_sync_wait_parent(handler, LXC_SYNC_STARTUP))
 865                 return -1;
 866
 867         /* Unshare CLONE_NEWNET after CLONE_NEWUSER. See
 868          * https://github.com/lxc/lxd/issues/1978.
 869          */
 870         if ((handler->clone_flags & (CLONE_NEWNET | CLONE_NEWUSER)) ==
 871             (CLONE_NEWNET | CLONE_NEWUSER)) {
 872                 ret = unshare(CLONE_NEWNET);
 873                 if (ret < 0) {
 874                         SYSERROR("Failed to unshare CLONE_NEWNET.");
 875                         goto out_warn_father;
 876                 }
 877                 INFO("Unshared CLONE_NEWNET.");
 878         }
 879
 880         /* Tell the parent task it can begin to configure the container and wait
 881          * for it to finish.
 882          */
 883         if (lxc_sync_barrier_parent(handler, LXC_SYNC_CONFIGURE))
 884                 return -1;
 885
 886         if (lxc_network_recv_veth_names_from_parent(handler) < 0) {
 887                 ERROR("Failed to receive veth names from parent");
 888                 goto out_warn_father;
 889         }
 890
 891         /* If we are in a new user namespace, become root there to have
 892          * privilege over our namespace.
 893          */
 894         if (!lxc_list_empty(&handler->conf->id_map)) {
 895                 if (lxc_switch_uid_gid(0, 0) < 0)
 896                         goto out_warn_father;
 897
 898                 /* Drop groups only after we switched to a valid gid in the new
 899                  * user namespace.
 900                  */
 901                 if (lxc_setgroups(0, NULL) < 0)
 902                         goto out_warn_father;
 903         }
 904
 905         if (access(handler->lxcpath, X_OK)) {
 906                 print_top_failing_dir(handler->lxcpath);
 907                 goto out_warn_father;
 908         }
 909
 910         ret = snprintf(path, sizeof(path), "%s/dev/null", handler->conf->rootfs.mount);
 911         if (ret < 0 || ret >= sizeof(path))
 912                 goto out_warn_father;
 913
 914         /* In order to checkpoint restore, we need to have everything in the
 915          * same mount namespace. However, some containers may not have a
 916          * reasonable /dev (in particular, they may not have /dev/null), so we
 917          * can't set init's std fds to /dev/null by opening it from inside the
 918          * container.
 919          *
 920          * If that's the case, fall back to using the host's /dev/null. This
 921          * means that migration won't work, but at least we won't spew output
 922          * where it isn't wanted.
 923          */
 924         if (handler->backgrounded && !handler->conf->autodev && access(path, F_OK) < 0) {
 925                 devnull_fd = open_devnull();
 926
 927                 if (devnull_fd < 0)
 928                         goto out_warn_father;
 929                 WARN("Using /dev/null from the host for container init's "
 930                      "standard file descriptors. Migration will not work.");
 931         }
 932
 933         /* Ask father to setup cgroups and wait for him to finish. */
 934         if (lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP))
 935                 goto out_error;
 936
 937         /* Unshare cgroup namespace after we have setup our cgroups. If we do it
 938          * earlier we end up with a wrong view of /proc/self/cgroup. For
 939          * example, assume we unshare(CLONE_NEWCGROUP) first, and then create
 940          * the cgroup for the container, say /sys/fs/cgroup/cpuset/lxc/c, then
 941          * /proc/self/cgroup would show us:
 942          *
 943          *      8:cpuset:/lxc/c
 944          *
 945          * whereas it should actually show
 946          *
 947          *      8:cpuset:/
 948          */
 949         if (handler->clone_flags & CLONE_NEWCGROUP) {
 950                 if (unshare(CLONE_NEWCGROUP) < 0) {
 951                         INFO("Failed to unshare CLONE_NEWCGROUP.");
 952                         goto out_warn_father;
 953                 }
 954                 INFO("Unshared CLONE_NEWCGROUP.");
 955         }
 956
 957         /* Add the requested environment variables to the current environment to
 958          * allow them to be used by the various hooks, such as the start hook
 959          * above.
 960          */
 961         lxc_list_for_each(iterator, &handler->conf->environment) {
 962                 if (putenv((char *)iterator->elem)) {
 963                         SYSERROR("Failed to set environment variable: %s.", (char *)iterator->elem);
 964                         goto out_warn_father;
 965                 }
 966         }
 967
 968         /* Setup the container, ip, names, utsname, ... */
 969         ret = lxc_setup(handler);
 970         close(handler->data_sock[1]);
 971         if (ret < 0) {
 972                 ERROR("Failed to setup container \"%s\".", handler->name);
 973                 close(handler->data_sock[0]);
 974                 goto out_warn_father;
 975         }
 976
 977         if (handler->clone_flags & CLONE_NEWCGROUP) {
 978                 fd = lxc_preserve_ns(syscall(SYS_getpid), "cgroup");
 979                 if (fd < 0) {
 980                         ERROR("%s - Failed to preserve cgroup namespace", strerror(errno));
 981                         close(handler->data_sock[0]);
 982                         goto out_warn_father;
 983                 }
 984
 985                 ret = lxc_abstract_unix_send_fds(handler->data_sock[0], &fd, 1, NULL, 0);
 986                 close(fd);
 987                 if (ret < 0) {
 988                         ERROR("%s - Failed to preserve cgroup namespace", strerror(errno));
 989                         close(handler->data_sock[0]);
 990                         goto out_warn_father;
 991                 }
 992         }
 993         close(handler->data_sock[0]);
 994
 995         /* Set the label to change to when we exec(2) the container's init. */
 996         if (lsm_process_label_set(NULL, handler->conf, 1, 1) < 0)
 997                 goto out_warn_father;
 998
 999         /* Set PR_SET_NO_NEW_PRIVS after we changed the lsm label. If we do it
1000          * before we aren't allowed anymore.
1001          */
1002         if (handler->conf->no_new_privs) {
1003                 if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) < 0) {
1004                         SYSERROR("Could not set PR_SET_NO_NEW_PRIVS to block execve() gainable privileges.");
1005                         goto out_warn_father;
1006                 }
1007                 DEBUG("Set PR_SET_NO_NEW_PRIVS to block execve() gainable privileges.");
1008         }
1009
1010         /* Some init's such as busybox will set sane tty settings on stdin,
1011          * stdout, stderr which it thinks is the console. We already set them
1012          * the way we wanted on the real terminal, and we want init to do its
1013          * setup on its console ie. the pty allocated in lxc_console_create() so
1014          * make sure that that pty is stdin,stdout,stderr.
1015          */
1016          if (handler->conf->console.slave >= 0) {
1017                  if (handler->backgrounded || handler->conf->is_execute == 0)
1018                          ret = set_stdfds(handler->conf->console.slave);
1019                  else
1020                          ret = lxc_console_set_stdfds(handler->conf->console.slave);
1021                  if (ret < 0) {
1022                         ERROR("Failed to redirect std{in,out,err} to pty file "
1023                               "descriptor %d",
1024                               handler->conf->console.slave);
1025                         goto out_warn_father;
1026                  }
1027          }
1028
1029         /* If we mounted a temporary proc, then unmount it now. */
1030         tmp_proc_unmount(handler->conf);
1031
1032         if (lxc_seccomp_load(handler->conf) != 0)
1033                 goto out_warn_father;
1034
1035         if (run_lxc_hooks(handler->name, "start", handler->conf, handler->lxcpath, NULL)) {
1036                 ERROR("Failed to run lxc.hook.start for container \"%s\".", handler->name);
1037                 goto out_warn_father;
1038         }
1039
1040         close(handler->sigfd);
1041
1042         if (devnull_fd < 0) {
1043                 devnull_fd = open_devnull();
1044
1045                 if (devnull_fd < 0)
1046                         goto out_warn_father;
1047         }
1048
1049         if (handler->conf->console.slave < 0 && handler->backgrounded)
1050                 if (set_stdfds(devnull_fd) < 0) {
1051                         ERROR("Failed to redirect std{in,out,err} to "
1052                               "\"/dev/null\"");
1053                         goto out_warn_father;
1054                 }
1055
1056         if (devnull_fd >= 0) {
1057                 close(devnull_fd);
1058                 devnull_fd = -1;
1059         }
1060
1061         setsid();
1062
1063         if (handler->conf->init_cwd && chdir(handler->conf->init_cwd)) {
1064                 SYSERROR("Could not change directory to \"%s\"", handler->conf->init_cwd);
1065                 goto out_warn_father;
1066         }
1067
1068         if (lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP_LIMITS))
1069                 goto out_warn_father;
1070
1071         /* Reset the environment variables the user requested in a clear
1072          * environment.
1073          */
1074         if (clearenv()) {
1075                 SYSERROR("Failed to clear environment.");
1076                 /* Don't error out though. */
1077         }
1078
1079         lxc_list_for_each(iterator, &handler->conf->environment) {
1080                 if (putenv((char *)iterator->elem)) {
1081                         SYSERROR("Failed to set environment variable: %s.", (char *)iterator->elem);
1082                         goto out_warn_father;
1083                 }
1084         }
1085
1086         if (putenv("container=lxc")) {
1087                 SYSERROR("Failed to set environment variable: container=lxc.");
1088                 goto out_warn_father;
1089         }
1090
1091         if (handler->conf->pty_names) {
1092                 if (putenv(handler->conf->pty_names)) {
1093                         SYSERROR("Failed to set environment variable for container ptys.");
1094                         goto out_warn_father;
1095                 }
1096         }
1097
1098         /* The container has been setup. We can now switch to an unprivileged
1099          * uid/gid.
1100          */
1101         new_uid = handler->conf->init_uid;
1102         new_gid = handler->conf->init_gid;
1103
1104         /* If we are in a new user namespace we already dropped all
1105          * groups when we switched to root in the new user namespace
1106          * further above. Only drop groups if we can, so ensure that we
1107          * have necessary privilege.
1108          */
1109         #if HAVE_LIBCAP
1110         have_cap_setgid = lxc_proc_cap_is_set(CAP_SETGID, CAP_EFFECTIVE);
1111         #else
1112         have_cap_setgid = false;
1113         #endif
1114         if (lxc_list_empty(&handler->conf->id_map) && have_cap_setgid) {
1115                 if (lxc_setgroups(0, NULL) < 0)
1116                         goto out_warn_father;
1117         }
1118
1119         if (lxc_switch_uid_gid(new_uid, new_gid) < 0)
1120                 goto out_warn_father;
1121
1122         /* After this call, we are in error because this ops should not return
1123          * as it execs.
1124          */
1125         handler->ops->start(handler, handler->data);
1126
1127 out_warn_father:
1128         /* We want the parent to know something went wrong, so we return a
1129          * special error code.
1130          */
1131         lxc_sync_wake_parent(handler, LXC_SYNC_ERROR);
1132
1133 out_error:
1134         if (devnull_fd >= 0)
1135                 close(devnull_fd);
1136
1137         return -1;
1138 }
1139
1140 static int lxc_recv_ttys_from_child(struct lxc_handler *handler)
1141 {
1142         int i;
1143         struct lxc_pty_info *pty_info;
1144         int ret = -1;
1145         int sock = handler->data_sock[1];
1146         struct lxc_conf *conf = handler->conf;
1147         struct lxc_tty_info *tty_info = &conf->tty_info;
1148
1149         if (!conf->tty)
1150                 return 0;
1151
1152         tty_info->pty_info = malloc(sizeof(*tty_info->pty_info) * conf->tty);
1153         if (!tty_info->pty_info)
1154                 return -1;
1155
1156         for (i = 0; i < conf->tty; i++) {
1157                 int ttyfds[2];
1158
1159                 ret = lxc_abstract_unix_recv_fds(sock, ttyfds, 2, NULL, 0);
1160                 if (ret < 0)
1161                         break;
1162
1163                 pty_info = &tty_info->pty_info[i];
1164                 pty_info->busy = 0;
1165                 pty_info->master = ttyfds[0];
1166                 pty_info->slave = ttyfds[1];
1167                 TRACE("Received pty with master fd %d and slave fd %d from "
1168                       "parent", pty_info->master, pty_info->slave);
1169         }
1170         if (ret < 0)
1171                 ERROR("Failed to receive %d ttys from child: %s", conf->tty,
1172                       strerror(errno));
1173         else
1174                 TRACE("Received %d ttys from child", conf->tty);
1175
1176         tty_info->nbtty = conf->tty;
1177
1178         return ret;
1179 }
1180
1181 int resolve_clone_flags(struct lxc_handler *handler)
1182 {
1183         handler->clone_flags = CLONE_NEWNS;
1184
1185         if (!handler->conf->inherit_ns[LXC_NS_USER]) {
1186                 if (!lxc_list_empty(&handler->conf->id_map))
1187                         handler->clone_flags |= CLONE_NEWUSER;
1188         } else {
1189                 INFO("Inheriting user namespace");
1190         }
1191
1192         if (!handler->conf->inherit_ns[LXC_NS_NET]) {
1193                 if (!lxc_requests_empty_network(handler))
1194                         handler->clone_flags |= CLONE_NEWNET;
1195         } else {
1196                 INFO("Inheriting net namespace");
1197         }
1198
1199         if (!handler->conf->inherit_ns[LXC_NS_IPC])
1200                 handler->clone_flags |= CLONE_NEWIPC;
1201         else
1202                 INFO("Inheriting ipc namespace");
1203
1204         if (!handler->conf->inherit_ns[LXC_NS_UTS])
1205                 handler->clone_flags |= CLONE_NEWUTS;
1206         else
1207                 INFO("Inheriting uts namespace");
1208
1209         if (!handler->conf->inherit_ns[LXC_NS_PID])
1210                 handler->clone_flags |= CLONE_NEWPID;
1211         else
1212                 INFO("Inheriting pid namespace");
1213
1214         if (cgns_supported()) {
1215                 if (!handler->conf->inherit_ns[LXC_NS_CGROUP])
1216                         handler->clone_flags |= CLONE_NEWCGROUP;
1217                 else
1218                         INFO("Inheriting cgroup namespace");
1219         } else if (handler->conf->inherit_ns[LXC_NS_CGROUP]) {
1220                         return -EINVAL;
1221         }
1222
1223         return 0;
1224 }
1225
1226 /* Note that this function is used with clone(CLONE_VM). Some glibc versions
1227  * used to reset the pid/tid to -1 when CLONE_VM was used without CLONE_THREAD.
1228  * But since the memory between parent and child is shared on CLONE_VM this
1229  * would invalidate the getpid() cache that glibc used to maintain and so
1230  * getpid() in the child would return the parent's pid. This is all fixed in
1231  * newer glibc versions where the getpid() cache is removed and the pid/tid is
1232  * not reset anymore.
1233  * However, if for whatever reason you - dear commiter - somehow need to get the
1234  * pid of the dummy intermediate process for do_share_ns() you need to call
1235  * syscall(__NR_getpid) directly. The next lxc_clone() call does not employ
1236  * CLONE_VM and will be fine.
1237  */
1238 static inline int do_share_ns(void *arg)
1239 {
1240         int i, flags, ret;
1241         struct lxc_handler *handler = arg;
1242
1243         for (i = 0; i < LXC_NS_MAX; i++) {
1244                 if (handler->nsfd[i] < 0)
1245                         continue;
1246
1247                 ret = setns(handler->nsfd[i], 0);
1248                 if (ret < 0)
1249                         return -1;
1250
1251                 DEBUG("Inherited %s namespace", ns_info[i].proc_name);
1252         }
1253
1254         flags = handler->on_clone_flags;
1255         flags |= CLONE_PARENT;
1256         handler->pid = lxc_clone(do_start, handler, flags);
1257         if (handler->pid < 0)
1258                 return -1;
1259
1260         return 0;
1261 }
1262
1263 /* lxc_spawn() performs crucial setup tasks and clone()s the new process which
1264  * exec()s the requested container binary.
1265  * Note that lxc_spawn() runs in the parent namespaces. Any operations performed
1266  * right here should be double checked if they'd pose a security risk. (For
1267  * example, any {u}mount() operations performed here will be reflected on the
1268  * host!)
1269  */
1270 static int lxc_spawn(struct lxc_handler *handler)
1271 {
1272         int i, ret;
1273         char pidstr[20];
1274         bool wants_to_map_ids;
1275         struct lxc_list *id_map;
1276         const char *name = handler->name;
1277         const char *lxcpath = handler->lxcpath;
1278         bool cgroups_connected = false, share_ns = false;
1279         struct lxc_conf *conf = handler->conf;
1280
1281         id_map = &conf->id_map;
1282         wants_to_map_ids = !lxc_list_empty(id_map);
1283
1284         for (i = 0; i < LXC_NS_MAX; i++) {
1285                 if (!conf->inherit_ns[i])
1286                         continue;
1287
1288                 handler->nsfd[i] = lxc_inherit_namespace(conf->inherit_ns[i], lxcpath, ns_info[i].proc_name);
1289                 if (handler->nsfd[i] < 0)
1290                         return -1;
1291
1292                 share_ns = true;
1293         }
1294
1295         if (lxc_sync_init(handler))
1296                 return -1;
1297
1298         ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0,
1299                          handler->data_sock);
1300         if (ret < 0) {
1301                 lxc_sync_fini(handler);
1302                 return -1;
1303         }
1304
1305         ret = resolve_clone_flags(handler);
1306         if (ret < 0) {
1307                 lxc_sync_fini(handler);
1308                 return -1;
1309         }
1310
1311         if (handler->clone_flags & CLONE_NEWNET) {
1312                 if (!lxc_list_empty(&conf->network)) {
1313
1314                         /* Find gateway addresses from the link device, which is
1315                          * no longer accessible inside the container. Do this
1316                          * before creating network interfaces, since goto
1317                          * out_delete_net does not work before lxc_clone.
1318                          */
1319                         if (lxc_find_gateway_addresses(handler)) {
1320                                 ERROR("Failed to find gateway addresses.");
1321                                 lxc_sync_fini(handler);
1322                                 return -1;
1323                         }
1324
1325                         /* That should be done before the clone because we will
1326                          * fill the netdev index and use them in the child.
1327                          */
1328                         if (lxc_create_network_priv(handler)) {
1329                                 ERROR("Failed to create the network.");
1330                                 lxc_sync_fini(handler);
1331                                 return -1;
1332                         }
1333                 }
1334         }
1335
1336         if (!cgroup_init(handler)) {
1337                 ERROR("Failed initializing cgroup support.");
1338                 goto out_delete_net;
1339         }
1340
1341         cgroups_connected = true;
1342
1343         if (!cgroup_create(handler)) {
1344                 ERROR("Failed creating cgroups.");
1345                 goto out_delete_net;
1346         }
1347
1348         /* If the rootfs is not a blockdev, prevent the container from marking
1349          * it readonly.
1350          * If the container is unprivileged then skip rootfs pinning.
1351          */
1352         if (!wants_to_map_ids) {
1353                 handler->pinfd = pin_rootfs(conf->rootfs.path);
1354                 if (handler->pinfd == -1)
1355                         INFO("Failed to pin the rootfs for container \"%s\".", handler->name);
1356         }
1357
1358         /* Create a process in a new set of namespaces. */
1359         handler->on_clone_flags = handler->clone_flags;
1360         if (handler->clone_flags & CLONE_NEWUSER) {
1361                 /* If CLONE_NEWUSER and CLONE_NEWNET was requested, we need to
1362                  * clone a new user namespace first and only later unshare our
1363                  * network namespace to ensure that network devices ownership is
1364                  * set up correctly.
1365                  */
1366                 handler->on_clone_flags &= ~CLONE_NEWNET;
1367         }
1368         /* The cgroup namespace gets unshare()ed not clone()ed. */
1369         handler->on_clone_flags &= ~CLONE_NEWCGROUP;
1370
1371         if (share_ns)
1372                 ret = lxc_clone(do_share_ns, handler, CLONE_VFORK | CLONE_VM | CLONE_FILES);
1373         else
1374                 handler->pid = lxc_clone(do_start, handler, handler->on_clone_flags);
1375         if (handler->pid < 0 || ret < 0) {
1376                 SYSERROR("Failed to clone a new set of namespaces.");
1377                 goto out_delete_net;
1378         }
1379         TRACE("Cloned child process %d", handler->pid);
1380
1381         for (i = 0; i < LXC_NS_MAX; i++)
1382                 if (handler->on_clone_flags & ns_info[i].clone_flag)
1383                         INFO("Cloned %s", ns_info[i].flag_name);
1384
1385         if (!preserve_ns(handler->nsfd, handler->clone_flags & ~CLONE_NEWNET, handler->pid)) {
1386                 ERROR("Failed to preserve cloned namespaces for lxc.hook.stop");
1387                 goto out_delete_net;
1388         }
1389
1390         lxc_sync_fini_child(handler);
1391
1392         /* Map the container uids. The container became an invalid userid the
1393          * moment it was cloned with CLONE_NEWUSER. This call doesn't change
1394          * anything immediately, but allows the container to setuid(0) (0 being
1395          * mapped to something else on the host.) later to become a valid uid
1396          * again.
1397          */
1398         if (wants_to_map_ids) {
1399                 if (!handler->conf->inherit_ns[LXC_NS_USER]) {
1400                         ret = lxc_map_ids(id_map, handler->pid);
1401                         if (ret < 0) {
1402                                 ERROR("Failed to set up id mapping.");
1403                                 goto out_delete_net;
1404                         }
1405                 }
1406         }
1407
1408         if (lxc_sync_wake_child(handler, LXC_SYNC_STARTUP))
1409                 goto out_delete_net;
1410
1411         if (lxc_sync_wait_child(handler, LXC_SYNC_CONFIGURE))
1412                 goto out_delete_net;
1413
1414         if (!cgroup_create_legacy(handler)) {
1415                 ERROR("Failed to setup legacy cgroups for container \"%s\".", name);
1416                 goto out_delete_net;
1417         }
1418         if (!cgroup_setup_limits(handler, false)) {
1419                 ERROR("Failed to setup cgroup limits for container \"%s\".", name);
1420                 goto out_delete_net;
1421         }
1422
1423         if (!cgroup_enter(handler))
1424                 goto out_delete_net;
1425
1426         if (!cgroup_chown(handler))
1427                 goto out_delete_net;
1428
1429         /* Now we're ready to preserve the network namespace */
1430         ret = lxc_preserve_ns(handler->pid, "net");
1431         if (ret < 0) {
1432                 ERROR("%s - Failed to preserve net namespace", strerror(errno));
1433                 goto out_delete_net;
1434         }
1435         handler->nsfd[LXC_NS_NET] = ret;
1436         DEBUG("Preserved net namespace via fd %d", ret);
1437
1438         /* Create the network configuration. */
1439         if (handler->clone_flags & CLONE_NEWNET) {
1440                 if (lxc_network_move_created_netdev_priv(handler->lxcpath,
1441                                                          handler->name,
1442                                                          &conf->network,
1443                                                          handler->pid)) {
1444                         ERROR("Failed to create the configured network.");
1445                         goto out_delete_net;
1446                 }
1447
1448                 if (lxc_create_network_unpriv(handler->lxcpath, handler->name,
1449                                               &conf->network,
1450                                               handler->pid)) {
1451                         ERROR("Failed to create the configured network.");
1452                         goto out_delete_net;
1453                 }
1454         }
1455
1456         if (lxc_network_send_veth_names_to_child(handler) < 0) {
1457                 ERROR("Failed to send veth names to child");
1458                 goto out_delete_net;
1459         }
1460
1461         if (!lxc_list_empty(&conf->procs)) {
1462                 ret = setup_proc_filesystem(&conf->procs, handler->pid);
1463                 if (ret < 0)
1464                         goto out_delete_net;
1465         }
1466
1467         /* Tell the child to continue its initialization. We'll get
1468          * LXC_SYNC_CGROUP when it is ready for us to setup cgroups.
1469          */
1470         if (lxc_sync_barrier_child(handler, LXC_SYNC_POST_CONFIGURE))
1471                 goto out_delete_net;
1472
1473         if (!lxc_list_empty(&conf->limits) && setup_resource_limits(&conf->limits, handler->pid)) {
1474                 ERROR("failed to setup resource limits for '%s'", name);
1475                 goto out_delete_net;
1476         }
1477
1478         if (lxc_sync_barrier_child(handler, LXC_SYNC_CGROUP_UNSHARE))
1479                 goto out_delete_net;
1480
1481         if (!cgroup_setup_limits(handler, true)) {
1482                 ERROR("Failed to setup the devices cgroup for container \"%s\".", name);
1483                 goto out_delete_net;
1484         }
1485         TRACE("Set up cgroup device limits");
1486
1487         cgroup_disconnect();
1488         cgroups_connected = false;
1489
1490         snprintf(pidstr, 20, "%d", handler->pid);
1491         if (setenv("LXC_PID", pidstr, 1))
1492                 SYSERROR("Failed to set environment variable: LXC_PID=%s.", pidstr);
1493
1494         /* Run any host-side start hooks */
1495         if (run_lxc_hooks(name, "start-host", conf, handler->lxcpath, NULL)) {
1496                 ERROR("Failed to run lxc.hook.start-host for container \"%s\".", name);
1497                 return -1;
1498         }
1499
1500         /* Tell the child to complete its initialization and wait for it to exec
1501          * or return an error. (The child will never return
1502          * LXC_SYNC_READY_START+1. It will either close the sync pipe, causing
1503          * lxc_sync_barrier_child to return success, or return a different
1504          * value, causing us to error out).
1505          */
1506         if (lxc_sync_barrier_child(handler, LXC_SYNC_READY_START))
1507                 return -1;
1508
1509         if (lxc_network_recv_name_and_ifindex_from_child(handler) < 0) {
1510                 ERROR("Failed to receive names and ifindices for network "
1511                       "devices from child");
1512                 goto out_delete_net;
1513         }
1514
1515         /* Now all networks are created, network devices are moved into place,
1516          * and the correct names and ifindeces in the respective namespaces have
1517          * been recorded. The corresponding structs have now all been filled. So
1518          * log them for debugging purposes.
1519          */
1520         lxc_log_configured_netdevs(conf);
1521
1522         /* Read tty fds allocated by child. */
1523         if (lxc_recv_ttys_from_child(handler) < 0) {
1524                 ERROR("Failed to receive tty info from child process.");
1525                 goto out_delete_net;
1526         }
1527
1528         if (handler->clone_flags & CLONE_NEWCGROUP) {
1529                 ret = lxc_abstract_unix_recv_fds(handler->data_sock[1],
1530                                                  &handler->nsfd[LXC_NS_CGROUP],
1531                                                  1, NULL, 0);
1532                 if (ret < 0) {
1533                         ERROR("%s - Failed to preserve cgroup namespace", strerror(errno));
1534                         goto out_delete_net;
1535                 }
1536                 DEBUG("Preserved cgroup namespace via fd %d", handler->nsfd[LXC_NS_CGROUP]);
1537         }
1538
1539         if (handler->ops->post_start(handler, handler->data))
1540                 goto out_abort;
1541
1542         if (lxc_set_state(name, handler, RUNNING)) {
1543                 ERROR("Failed to set state for container \"%s\" to \"%s\".", name,
1544                       lxc_state2str(RUNNING));
1545                 goto out_abort;
1546         }
1547
1548         lxc_sync_fini(handler);
1549
1550         return 0;
1551
1552 out_delete_net:
1553         if (cgroups_connected)
1554                 cgroup_disconnect();
1555
1556         if (handler->clone_flags & CLONE_NEWNET)
1557                 lxc_delete_network(handler);
1558
1559 out_abort:
1560         lxc_abort(name, handler);
1561         lxc_sync_fini(handler);
1562         if (handler->pinfd >= 0) {
1563                 close(handler->pinfd);
1564                 handler->pinfd = -1;
1565         }
1566
1567         return -1;
1568 }
1569
1570 int __lxc_start(const char *name, struct lxc_handler *handler,
1571                 struct lxc_operations* ops, void *data, const char *lxcpath,
1572                 bool backgrounded)
1573 {
1574         int status;
1575         int err = -1;
1576         struct lxc_conf *conf = handler->conf;
1577
1578         if (lxc_init(name, handler) < 0) {
1579                 ERROR("Failed to initialize container \"%s\".", name);
1580                 return -1;
1581         }
1582         handler->ops = ops;
1583         handler->data = data;
1584         handler->backgrounded = backgrounded;
1585
1586         if (!attach_block_device(handler->conf)) {
1587                 ERROR("Failed to attach block device.");
1588                 goto out_fini_nonet;
1589         }
1590
1591         if (geteuid() == 0 && !lxc_list_empty(&conf->id_map)) {
1592                 /* If the backing store is a device, mount it here and now. */
1593                 if (rootfs_is_blockdev(conf)) {
1594                         if (unshare(CLONE_NEWNS) < 0) {
1595                                 ERROR("Failed to unshare CLONE_NEWNS.");
1596                                 goto out_fini_nonet;
1597                         }
1598                         INFO("Unshared CLONE_NEWNS.");
1599
1600                         remount_all_slave();
1601                         if (do_rootfs_setup(conf, name, lxcpath) < 0) {
1602                                 ERROR("Error setting up rootfs mount as root before spawn.");
1603                                 goto out_fini_nonet;
1604                         }
1605                         INFO("Set up container rootfs as host root.");
1606                 }
1607         }
1608
1609         err = lxc_spawn(handler);
1610         if (err) {
1611                 ERROR("Failed to spawn container \"%s\".", name);
1612                 goto out_detach_blockdev;
1613         }
1614         /* close parent side of data socket */
1615         close(handler->data_sock[0]);
1616         handler->data_sock[0] = -1;
1617         close(handler->data_sock[1]);
1618         handler->data_sock[1] = -1;
1619
1620         handler->conf->reboot = 0;
1621
1622         err = lxc_poll(name, handler);
1623         if (err) {
1624                 ERROR("LXC mainloop exited with error: %d.", err);
1625                 goto out_abort;
1626         }
1627
1628         while (waitpid(handler->pid, &status, 0) < 0 && errno == EINTR)
1629                 continue;
1630
1631         /* If the child process exited but was not signaled, it didn't call
1632          * reboot. This should mean it was an lxc-execute which simply exited.
1633          * In any case, treat it as a 'halt'.
1634          */
1635         if (WIFSIGNALED(status)) {
1636                 switch(WTERMSIG(status)) {
1637                 case SIGINT: /* halt */
1638                         DEBUG("Container \"%s\" is halting.", name);
1639                         break;
1640                 case SIGHUP: /* reboot */
1641                         DEBUG("Container \"%s\" is rebooting.", name);
1642                         handler->conf->reboot = 1;
1643                         break;
1644                 case SIGSYS: /* seccomp */
1645                         DEBUG("Container \"%s\" violated its seccomp policy.", name);
1646                         break;
1647                 default:
1648                         DEBUG("Unknown exit status for container \"%s\" init %d.", name, WTERMSIG(status));
1649                         break;
1650                 }
1651         }
1652
1653         err = lxc_restore_phys_nics_to_netns(handler);
1654         if (err < 0)
1655                 ERROR("Failed to move physical network devices back to parent "
1656                       "network namespace");
1657
1658         if (handler->pinfd >= 0) {
1659                 close(handler->pinfd);
1660                 handler->pinfd = -1;
1661         }
1662
1663         lxc_monitor_send_exit_code(name, status, handler->lxcpath);
1664         err =  lxc_error_set_and_log(handler->pid, status);
1665
1666 out_fini:
1667         lxc_delete_network(handler);
1668
1669 out_detach_blockdev:
1670         detach_block_device(handler->conf);
1671
1672 out_fini_nonet:
1673         lxc_fini(name, handler);
1674         return err;
1675
1676 out_abort:
1677         lxc_abort(name, handler);
1678         goto out_fini;
1679 }
1680
1681 struct start_args {
1682         char *const *argv;
1683 };
1684
1685 static int start(struct lxc_handler *handler, void* data)
1686 {
1687         struct start_args *arg = data;
1688
1689         NOTICE("Exec'ing \"%s\".", arg->argv[0]);
1690
1691         execvp(arg->argv[0], arg->argv);
1692         SYSERROR("Failed to exec \"%s\".", arg->argv[0]);
1693         return 0;
1694 }
1695
1696 static int post_start(struct lxc_handler *handler, void* data)
1697 {
1698         struct start_args *arg = data;
1699
1700         NOTICE("Started \"%s\" with pid \"%d\".", arg->argv[0], handler->pid);
1701         return 0;
1702 }
1703
1704 static struct lxc_operations start_ops = {
1705         .start = start,
1706         .post_start = post_start
1707 };
1708
1709 int lxc_start(const char *name, char *const argv[], struct lxc_handler *handler,
1710               const char *lxcpath, bool backgrounded)
1711 {
1712         struct start_args start_arg = {
1713                 .argv = argv,
1714         };
1715
1716         return __lxc_start(name, handler, &start_ops, &start_arg, lxcpath, backgrounded);
1717 }
1718
1719 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
1720                                             const char *name)
1721 {
1722         char destroy[MAXPATHLEN];
1723         bool bret = true;
1724         int ret = 0;
1725         struct lxc_container *c;
1726         if (handler->conf->rootfs.path && handler->conf->rootfs.mount) {
1727                 bret = do_destroy_container(handler);
1728                 if (!bret) {
1729                         ERROR("Error destroying rootfs for container \"%s\".", name);
1730                         return;
1731                 }
1732         }
1733         INFO("Destroyed rootfs for container \"%s\".", name);
1734
1735         ret = snprintf(destroy, MAXPATHLEN, "%s/%s", handler->lxcpath, name);
1736         if (ret < 0 || ret >= MAXPATHLEN) {
1737                 ERROR("Error destroying directory for container \"%s\".", name);
1738                 return;
1739         }
1740
1741         c = lxc_container_new(name, handler->lxcpath);
1742         if (c) {
1743                 if (container_disk_lock(c)) {
1744                         INFO("Could not update lxc_snapshots file.");
1745                         lxc_container_put(c);
1746                 } else {
1747                         mod_all_rdeps(c, false);
1748                         container_disk_unlock(c);
1749                         lxc_container_put(c);
1750                 }
1751         }
1752
1753         if (!handler->am_root)
1754                 ret = userns_exec_full(handler->conf, lxc_rmdir_onedev_wrapper,
1755                                        destroy, "lxc_rmdir_onedev_wrapper");
1756         else
1757                 ret = lxc_rmdir_onedev(destroy, NULL);
1758
1759         if (ret < 0) {
1760                 ERROR("Error destroying directory for container \"%s\".", name);
1761                 return;
1762         }
1763         INFO("Destroyed directory for container \"%s\".", name);
1764 }
1765
1766 static int lxc_rmdir_onedev_wrapper(void *data)
1767 {
1768         char *arg = (char *) data;
1769         return lxc_rmdir_onedev(arg, NULL);
1770 }
1771
1772 static bool do_destroy_container(struct lxc_handler *handler) {
1773         int ret;
1774
1775         if (!handler->am_root) {
1776                 ret = userns_exec_full(handler->conf, storage_destroy_wrapper,
1777                                        handler->conf, "storage_destroy_wrapper");
1778                 if (ret < 0)
1779                         return false;
1780
1781                 return true;
1782         }
1783
1784         return storage_destroy(handler->conf);
1785 }