src/lxc/start.c

   1 /*
   2  * lxc: linux Container library
   3  *
   4  * (C) Copyright IBM Corp. 2007, 2008
   5  *
   6  * Authors:
   7  * Daniel Lezcano <daniel.lezcano at free.fr>
   8  *
   9  * This library is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License as published by the Free Software Foundation; either
  12  * version 2.1 of the License, or (at your option) any later version.
  13  *
  14  * This library is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with this library; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  22  */
  23
  24 #define _GNU_SOURCE
  25 #include "config.h"
  26
  27 #include <alloca.h>
  28 #include <dirent.h>
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <grp.h>
  32 #include <poll.h>
  33 #include <signal.h>
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <unistd.h>
  38 #include <sys/file.h>
  39 #include <sys/mount.h>
  40 #include <sys/param.h>
  41 #include <sys/prctl.h>
  42 #include <sys/socket.h>
  43 #include <sys/stat.h>
  44 #include <sys/syscall.h>
  45 #include <sys/types.h>
  46 #include <sys/un.h>
  47 #include <sys/wait.h>
  48
  49 #if HAVE_LIBCAP
  50 #include <sys/capability.h>
  51 #endif
  52
  53 #ifndef HAVE_DECL_PR_CAPBSET_DROP
  54 #define PR_CAPBSET_DROP 24
  55 #endif
  56
  57 #ifndef HAVE_DECL_PR_SET_NO_NEW_PRIVS
  58 #define PR_SET_NO_NEW_PRIVS 38
  59 #endif
  60
  61 #ifndef HAVE_DECL_PR_GET_NO_NEW_PRIVS
  62 #define PR_GET_NO_NEW_PRIVS 39
  63 #endif
  64
  65 #include "af_unix.h"
  66 #include "bdev.h"
  67 #include "caps.h"
  68 #include "cgroup.h"
  69 #include "commands.h"
  70 #include "commands_utils.h"
  71 #include "conf.h"
  72 #include "console.h"
  73 #include "error.h"
  74 #include "log.h"
  75 #include "lxccontainer.h"
  76 #include "lxclock.h"
  77 #include "lxcseccomp.h"
  78 #include "lxcutmp.h"
  79 #include "mainloop.h"
  80 #include "monitor.h"
  81 #include "namespace.h"
  82 #include "start.h"
  83 #include "storage_utils.h"
  84 #include "sync.h"
  85 #include "utils.h"
  86 #include "lsm/lsm.h"
  87
  88 lxc_log_define(lxc_start, lxc);
  89
  90 extern void mod_all_rdeps(struct lxc_container *c, bool inc);
  91 static bool do_destroy_container(struct lxc_conf *conf);
  92 static int lxc_rmdir_onedev_wrapper(void *data);
  93 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
  94                                             const char *name);
  95
  96 static void print_top_failing_dir(const char *path)
  97 {
  98         size_t len = strlen(path);
  99         char *copy = alloca(len + 1), *p, *e, saved;
 100         strcpy(copy, path);
 101
 102         p = copy;
 103         e = copy + len;
 104         while (p < e) {
 105                 while (p < e && *p == '/')
 106                         p++;
 107                 while (p < e && *p != '/')
 108                         p++;
 109                 saved = *p;
 110                 *p = '\0';
 111                 if (access(copy, X_OK)) {
 112                         SYSERROR("Could not access %s. Please grant it x "
 113                                  "access, or add an ACL for the container "
 114                                  "root.", copy);
 115                         return;
 116                 }
 117                 *p = saved;
 118         }
 119 }
 120
 121 static void close_ns(int ns_fd[LXC_NS_MAX])
 122 {
 123         int i;
 124
 125         for (i = 0; i < LXC_NS_MAX; i++) {
 126                 if (ns_fd[i] > -1) {
 127                         close(ns_fd[i]);
 128                         ns_fd[i] = -1;
 129                 }
 130         }
 131 }
 132
 133 /* preserve_ns: open /proc/@pid/ns/@ns for each namespace specified
 134  * in clone_flags.
 135  * Return true on success, false on failure.
 136  */
 137 static bool preserve_ns(int ns_fd[LXC_NS_MAX], int clone_flags, pid_t pid)
 138 {
 139         int i, ret;
 140
 141         for (i = 0; i < LXC_NS_MAX; i++)
 142                 ns_fd[i] = -1;
 143
 144         ret = lxc_preserve_ns(pid, "");
 145         if (ret < 0) {
 146                 SYSERROR("Kernel does not support attaching to namespaces.");
 147                 return false;
 148         } else {
 149                 close(ret);
 150         }
 151
 152         for (i = 0; i < LXC_NS_MAX; i++) {
 153                 if ((clone_flags & ns_info[i].clone_flag) == 0)
 154                         continue;
 155                 ns_fd[i] = lxc_preserve_ns(pid, ns_info[i].proc_name);
 156                 if (ns_fd[i] < 0)
 157                         goto error;
 158         }
 159
 160         return true;
 161
 162 error:
 163         if (errno == ENOENT)
 164                 SYSERROR("Kernel does not support attaching to %s namespaces.", ns_info[i].proc_name);
 165         else
 166                 SYSERROR("Failed to open file descriptor for %s namespace: %s.", ns_info[i].proc_name, strerror(errno));
 167         close_ns(ns_fd);
 168         return false;
 169 }
 170
 171 static int attach_ns(const int ns_fd[LXC_NS_MAX]) {
 172         int i;
 173
 174         for (i = 0; i < LXC_NS_MAX; i++) {
 175                 if (ns_fd[i] < 0)
 176                         continue;
 177
 178                 if (setns(ns_fd[i], 0) != 0)
 179                         goto error;
 180         }
 181         return 0;
 182
 183 error:
 184         SYSERROR("Failed to attach %s namespace.", ns_info[i].proc_name);
 185         return -1;
 186 }
 187
 188 static int match_fd(int fd)
 189 {
 190         return (fd == 0 || fd == 1 || fd == 2);
 191 }
 192
 193 int lxc_check_inherited(struct lxc_conf *conf, bool closeall,
 194                         int *fds_to_ignore, size_t len_fds)
 195 {
 196         struct dirent *direntp;
 197         int fd, fddir;
 198         size_t i;
 199         DIR *dir;
 200
 201         if (conf && conf->close_all_fds)
 202                 closeall = true;
 203
 204 restart:
 205         dir = opendir("/proc/self/fd");
 206         if (!dir) {
 207                 WARN("Failed to open directory: %s.", strerror(errno));
 208                 return -1;
 209         }
 210
 211         fddir = dirfd(dir);
 212
 213         while ((direntp = readdir(dir))) {
 214                 if (!direntp)
 215                         break;
 216
 217                 if (!strcmp(direntp->d_name, "."))
 218                         continue;
 219
 220                 if (!strcmp(direntp->d_name, ".."))
 221                         continue;
 222
 223                 if (lxc_safe_int(direntp->d_name, &fd) < 0) {
 224                         INFO("Could not parse file descriptor for: %s", direntp->d_name);
 225                         continue;
 226                 }
 227
 228                 for (i = 0; i < len_fds; i++)
 229                         if (fds_to_ignore[i] == fd)
 230                                 break;
 231
 232                 if (fd == fddir || fd == lxc_log_fd ||
 233                     (i < len_fds && fd == fds_to_ignore[i]))
 234                         continue;
 235
 236                 if (current_config && fd == current_config->logfd)
 237                         continue;
 238
 239                 if (match_fd(fd))
 240                         continue;
 241
 242                 if (closeall) {
 243                         close(fd);
 244                         closedir(dir);
 245                         INFO("Closed inherited fd: %d.", fd);
 246                         goto restart;
 247                 }
 248                 WARN("Inherited fd: %d.", fd);
 249         }
 250
 251         /* Only enable syslog at this point to avoid the above logging function
 252          * to open a new fd and make the check_inherited function enter an
 253          * infinite loop.
 254          */
 255         lxc_log_enable_syslog();
 256
 257         closedir(dir); /* cannot fail */
 258         return 0;
 259 }
 260
 261 static int setup_signal_fd(sigset_t *oldmask)
 262 {
 263         sigset_t mask;
 264         int fd;
 265
 266         /* Block everything except serious error signals. */
 267         if (sigfillset(&mask) ||
 268             sigdelset(&mask, SIGILL) ||
 269             sigdelset(&mask, SIGSEGV) ||
 270             sigdelset(&mask, SIGBUS) ||
 271             sigdelset(&mask, SIGWINCH) ||
 272             sigprocmask(SIG_BLOCK, &mask, oldmask)) {
 273                 SYSERROR("Failed to set signal mask.");
 274                 return -1;
 275         }
 276
 277         fd = signalfd(-1, &mask, 0);
 278         if (fd < 0) {
 279                 SYSERROR("Failed to create signal file descriptor.");
 280                 return -1;
 281         }
 282
 283         if (fcntl(fd, F_SETFD, FD_CLOEXEC)) {
 284                 SYSERROR("Failed to set FD_CLOEXEC on the signal file descriptor: %d.", fd);
 285                 close(fd);
 286                 return -1;
 287         }
 288
 289         DEBUG("Set SIGCHLD handler with file descriptor: %d.", fd);
 290
 291         return fd;
 292 }
 293
 294 static int signal_handler(int fd, uint32_t events, void *data,
 295                           struct lxc_epoll_descr *descr)
 296 {
 297         struct signalfd_siginfo siginfo;
 298         siginfo_t info;
 299         int ret;
 300         pid_t *pid = data;
 301         bool init_died = false;
 302
 303         ret = read(fd, &siginfo, sizeof(siginfo));
 304         if (ret < 0) {
 305                 ERROR("Failed to read signal info from signal file descriptor: %d.", fd);
 306                 return -1;
 307         }
 308
 309         if (ret != sizeof(siginfo)) {
 310                 ERROR("Unexpected size for siginfo struct.");
 311                 return -1;
 312         }
 313
 314         /* Check whether init is running. */
 315         info.si_pid = 0;
 316         ret = waitid(P_PID, *pid, &info, WEXITED | WNOWAIT | WNOHANG);
 317         if (ret == 0 && info.si_pid == *pid)
 318                 init_died = true;
 319
 320         if (siginfo.ssi_signo != SIGCHLD) {
 321                 kill(*pid, siginfo.ssi_signo);
 322                 INFO("Forwarded signal %d to pid %d.", siginfo.ssi_signo, *pid);
 323                 return init_died ? 1 : 0;
 324         }
 325
 326         if (siginfo.ssi_code == CLD_STOPPED) {
 327                 INFO("Container init process was stopped.");
 328                 return init_died ? 1 : 0;
 329         } else if (siginfo.ssi_code == CLD_CONTINUED) {
 330                 INFO("Container init process was continued.");
 331                 return init_died ? 1 : 0;
 332         }
 333
 334         /* More robustness, protect ourself from a SIGCHLD sent
 335          * by a process different from the container init.
 336          */
 337         if (siginfo.ssi_pid != *pid) {
 338                 NOTICE("Received SIGCHLD from pid %d instead of container init %d.", siginfo.ssi_pid, *pid);
 339                 return init_died ? 1 : 0;
 340         }
 341
 342         DEBUG("Container init process %d exited.", *pid);
 343         return 1;
 344 }
 345
 346 static int lxc_serve_state_clients(const char *name,
 347                                    struct lxc_handler *handler,
 348                                    lxc_state_t state)
 349 {
 350         ssize_t ret;
 351         struct lxc_list *cur, *next;
 352         struct state_client *client;
 353         struct lxc_msg msg = {.type = lxc_msg_state, .value = state};
 354
 355         process_lock();
 356
 357         /* Only set state under process lock held so that we don't cause
 358         *  lxc_cmd_add_state_client() to miss a state.
 359          */
 360         handler->state = state;
 361         TRACE("set container state to %s", lxc_state2str(state));
 362
 363         if (lxc_list_empty(&handler->state_clients)) {
 364                 TRACE("no state clients registered");
 365                 process_unlock();
 366                 lxc_monitor_send_state(name, state, handler->lxcpath);
 367                 return 0;
 368         }
 369
 370         strncpy(msg.name, name, sizeof(msg.name));
 371         msg.name[sizeof(msg.name) - 1] = 0;
 372
 373         lxc_list_for_each_safe(cur, &handler->state_clients, next) {
 374                 client = cur->elem;
 375
 376                 if (!client->states[state]) {
 377                         TRACE("state %s not registered for state client %d",
 378                               lxc_state2str(state), client->clientfd);
 379                         continue;
 380                 }
 381
 382                 TRACE("sending state %s to state client %d",
 383                       lxc_state2str(state), client->clientfd);
 384
 385         again:
 386                 ret = send(client->clientfd, &msg, sizeof(msg), 0);
 387                 if (ret <= 0) {
 388                         if (errno == EINTR) {
 389                                 TRACE("Caught EINTR; retrying");
 390                                 goto again;
 391                         }
 392
 393                         ERROR("failed to send message to client");
 394                 }
 395
 396                 /* kick client from list */
 397                 close(client->clientfd);
 398                 lxc_list_del(cur);
 399                 free(cur->elem);
 400                 free(cur);
 401         }
 402         process_unlock();
 403
 404         return 0;
 405 }
 406
 407 static int lxc_serve_state_socket_pair(const char *name,
 408                                        struct lxc_handler *handler,
 409                                        lxc_state_t state)
 410 {
 411         ssize_t ret;
 412
 413         if (!handler->backgrounded ||
 414             handler->state_socket_pair[1] < 0 ||
 415             state == STARTING)
 416                 return 0;
 417
 418         /* Close read end of the socket pair. */
 419         close(handler->state_socket_pair[0]);
 420         handler->state_socket_pair[0] = -1;
 421
 422 again:
 423         ret = lxc_abstract_unix_send_credential(handler->state_socket_pair[1],
 424                                                 &(int){state}, sizeof(int));
 425         if (ret != sizeof(int)) {
 426                 if (errno == EINTR)
 427                         goto again;
 428                 SYSERROR("Failed to send state to %d",
 429                          handler->state_socket_pair[1]);
 430                 return -1;
 431         }
 432
 433         TRACE("Sent container state \"%s\" to %d", lxc_state2str(state),
 434               handler->state_socket_pair[1]);
 435
 436         /* Close write end of the socket pair. */
 437         close(handler->state_socket_pair[1]);
 438         handler->state_socket_pair[1] = -1;
 439
 440         return 0;
 441 }
 442
 443 int lxc_set_state(const char *name, struct lxc_handler *handler,
 444                   lxc_state_t state)
 445 {
 446         int ret;
 447
 448         ret = lxc_serve_state_socket_pair(name, handler, state);
 449         if (ret < 0) {
 450                 ERROR("Failed to synchronize via anonymous pair of unix sockets");
 451                 return -1;
 452         }
 453
 454         ret = lxc_serve_state_clients(name, handler, state);
 455         if (ret < 0)
 456                 return -1;
 457
 458         /* This function will try to connect to the legacy lxc-monitord state
 459          * server and only exists for backwards compatibility.
 460          */
 461         lxc_monitor_send_state(name, state, handler->lxcpath);
 462
 463         return 0;
 464 }
 465
 466 int lxc_poll(const char *name, struct lxc_handler *handler)
 467 {
 468         int sigfd = handler->sigfd;
 469         int pid = handler->pid;
 470         struct lxc_epoll_descr descr;
 471
 472         if (lxc_mainloop_open(&descr)) {
 473                 ERROR("Failed to create LXC mainloop.");
 474                 goto out_sigfd;
 475         }
 476
 477         if (lxc_mainloop_add_handler(&descr, sigfd, signal_handler, &pid)) {
 478                 ERROR("Failed to add signal handler with file descriptor %d to LXC mainloop.", sigfd);
 479                 goto out_mainloop_open;
 480         }
 481
 482         if (lxc_console_mainloop_add(&descr, handler->conf)) {
 483                 ERROR("Failed to add console handler to LXC mainloop.");
 484                 goto out_mainloop_open;
 485         }
 486
 487         if (lxc_cmd_mainloop_add(name, &descr, handler)) {
 488                 ERROR("Failed to add command handler to LXC mainloop.");
 489                 goto out_mainloop_open;
 490         }
 491
 492         if (handler->conf->need_utmp_watch) {
 493                 #if HAVE_LIBCAP
 494                 if (lxc_utmp_mainloop_add(&descr, handler)) {
 495                         ERROR("Failed to add utmp handler to LXC mainloop.");
 496                         goto out_mainloop_open;
 497                 }
 498                 #else
 499                         DEBUG("Not starting utmp handler as CAP_SYS_BOOT cannot be dropped without capabilities support.");
 500                 #endif
 501         }
 502         TRACE("lxc mainloop is ready");
 503
 504         return lxc_mainloop(&descr, -1);
 505
 506 out_mainloop_open:
 507         lxc_mainloop_close(&descr);
 508
 509 out_sigfd:
 510         close(sigfd);
 511
 512         return -1;
 513 }
 514
 515 void lxc_free_handler(struct lxc_handler *handler)
 516 {
 517         if (handler->conf && handler->conf->maincmd_fd)
 518                 close(handler->conf->maincmd_fd);
 519
 520         if (handler->state_socket_pair[0] >= 0)
 521                 close(handler->state_socket_pair[0]);
 522
 523         if (handler->state_socket_pair[1] >= 0)
 524                 close(handler->state_socket_pair[1]);
 525
 526         if (handler->name)
 527                 free(handler->name);
 528
 529         handler->conf = NULL;
 530         free(handler);
 531 }
 532
 533 struct lxc_handler *lxc_init_handler(const char *name, struct lxc_conf *conf,
 534                                      const char *lxcpath, bool daemonize)
 535 {
 536         int i, ret;
 537         struct lxc_handler *handler;
 538
 539         handler = malloc(sizeof(*handler));
 540         if (!handler) {
 541                 ERROR("failed to allocate memory");
 542                 return NULL;
 543         }
 544
 545         memset(handler, 0, sizeof(*handler));
 546
 547         handler->ttysock[0] = handler->ttysock[1] = -1;
 548         handler->conf = conf;
 549         handler->lxcpath = lxcpath;
 550         handler->pinfd = -1;
 551         handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
 552         lxc_list_init(&handler->state_clients);
 553
 554         for (i = 0; i < LXC_NS_MAX; i++)
 555                 handler->nsfd[i] = -1;
 556
 557         handler->name = strdup(name);
 558         if (!handler->name) {
 559                 ERROR("failed to allocate memory");
 560                 goto on_error;
 561         }
 562
 563         if (daemonize && !handler->conf->reboot) {
 564                 /* Create socketpair() to synchronize on daemonized startup.
 565                  * When the container reboots we don't need to synchronize again
 566                  * currently so don't open another socketpair().
 567                  */
 568                 ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0,
 569                                  handler->state_socket_pair);
 570                 if (ret < 0) {
 571                         ERROR("Failed to create anonymous pair of unix sockets");
 572                         goto on_error;
 573                 }
 574                 TRACE("Created anonymous pair {%d,%d} of unix sockets",
 575                       handler->state_socket_pair[0],
 576                       handler->state_socket_pair[1]);
 577         }
 578
 579         if (lxc_cmd_init(name, handler, lxcpath)) {
 580                 ERROR("failed to set up command socket");
 581                 goto on_error;
 582         }
 583
 584         TRACE("unix domain socket %d for command server is ready",
 585               handler->conf->maincmd_fd);
 586
 587         return handler;
 588
 589 on_error:
 590         lxc_free_handler(handler);
 591
 592         return NULL;
 593 }
 594
 595 int lxc_init(const char *name, struct lxc_handler *handler)
 596 {
 597         struct lxc_conf *conf = handler->conf;
 598
 599         lsm_init();
 600         TRACE("initialized LSM");
 601
 602         if (lxc_read_seccomp_config(conf) != 0) {
 603                 ERROR("Failed loading seccomp policy.");
 604                 goto out_close_maincmd_fd;
 605         }
 606         TRACE("read seccomp policy");
 607
 608         /* Begin by setting the state to STARTING. */
 609         if (lxc_set_state(name, handler, STARTING)) {
 610                 ERROR("Failed to set state for container \"%s\" to \"%s\".", name, lxc_state2str(STARTING));
 611                 goto out_close_maincmd_fd;
 612         }
 613         TRACE("set container state to \"STARTING\"");
 614
 615         /* Start of environment variable setup for hooks. */
 616         if (name && setenv("LXC_NAME", name, 1))
 617                 SYSERROR("Failed to set environment variable: LXC_NAME=%s.", name);
 618
 619         if (conf->rcfile && setenv("LXC_CONFIG_FILE", conf->rcfile, 1))
 620                 SYSERROR("Failed to set environment variable: LXC_CONFIG_FILE=%s.", conf->rcfile);
 621
 622         if (conf->rootfs.mount && setenv("LXC_ROOTFS_MOUNT", conf->rootfs.mount, 1))
 623                 SYSERROR("Failed to set environment variable: LXC_ROOTFS_MOUNT=%s.", conf->rootfs.mount);
 624
 625         if (conf->rootfs.path && setenv("LXC_ROOTFS_PATH", conf->rootfs.path, 1))
 626                 SYSERROR("Failed to set environment variable: LXC_ROOTFS_PATH=%s.", conf->rootfs.path);
 627
 628         if (conf->console.path && setenv("LXC_CONSOLE", conf->console.path, 1))
 629                 SYSERROR("Failed to set environment variable: LXC_CONSOLE=%s.", conf->console.path);
 630
 631         if (conf->console.log_path && setenv("LXC_CONSOLE_LOGPATH", conf->console.log_path, 1))
 632                 SYSERROR("Failed to set environment variable: LXC_CONSOLE_LOGPATH=%s.", conf->console.log_path);
 633
 634         if (setenv("LXC_CGNS_AWARE", "1", 1))
 635                 SYSERROR("Failed to set environment variable LXC_CGNS_AWARE=1.");
 636         /* End of environment variable setup for hooks. */
 637
 638         TRACE("set environment variables");
 639
 640         if (run_lxc_hooks(name, "pre-start", conf, handler->lxcpath, NULL)) {
 641                 ERROR("Failed to run lxc.hook.pre-start for container \"%s\".", name);
 642                 goto out_aborting;
 643         }
 644         TRACE("ran pre-start hooks");
 645
 646         /* The signal fd has to be created before forking otherwise if the child
 647          * process exits before we setup the signal fd, the event will be lost
 648          * and the command will be stuck.
 649          */
 650         handler->sigfd = setup_signal_fd(&handler->oldmask);
 651         if (handler->sigfd < 0) {
 652                 ERROR("Failed to setup SIGCHLD fd handler.");
 653                 goto out_delete_tty;
 654         }
 655         TRACE("set up signal fd");
 656
 657         /* Do this after setting up signals since it might unblock SIGWINCH. */
 658         if (lxc_console_create(conf)) {
 659                 ERROR("Failed to create console for container \"%s\".", name);
 660                 goto out_restore_sigmask;
 661         }
 662         TRACE("created console");
 663
 664         if (lxc_ttys_shift_ids(conf) < 0) {
 665                 ERROR("Failed to shift tty into container.");
 666                 goto out_restore_sigmask;
 667         }
 668         TRACE("shifted tty ids");
 669
 670         INFO("container \"%s\" is initialized", name);
 671         return 0;
 672
 673 out_restore_sigmask:
 674         sigprocmask(SIG_SETMASK, &handler->oldmask, NULL);
 675 out_delete_tty:
 676         lxc_delete_tty(&conf->tty_info);
 677 out_aborting:
 678         lxc_set_state(name, handler, ABORTING);
 679 out_close_maincmd_fd:
 680         close(conf->maincmd_fd);
 681         conf->maincmd_fd = -1;
 682         return -1;
 683 }
 684
 685 void lxc_fini(const char *name, struct lxc_handler *handler)
 686 {
 687         int i, rc;
 688         struct lxc_list *cur, *next;
 689         pid_t self = getpid();
 690         char *namespaces[LXC_NS_MAX + 1];
 691         size_t namespace_count = 0;
 692
 693         /* The STOPPING state is there for future cleanup code which can take
 694          * awhile.
 695          */
 696         lxc_set_state(name, handler, STOPPING);
 697
 698         for (i = 0; i < LXC_NS_MAX; i++) {
 699                 if (handler->nsfd[i] != -1) {
 700                         rc = asprintf(&namespaces[namespace_count], "%s:/proc/%d/fd/%d",
 701                                       ns_info[i].proc_name, self, handler->nsfd[i]);
 702                         if (rc == -1) {
 703                                 SYSERROR("Failed to allocate memory.");
 704                                 break;
 705                         }
 706                         ++namespace_count;
 707                 }
 708         }
 709         namespaces[namespace_count] = NULL;
 710
 711         if (handler->conf->reboot && setenv("LXC_TARGET", "reboot", 1))
 712                 SYSERROR("Failed to set environment variable: LXC_TARGET=reboot.");
 713
 714         if (!handler->conf->reboot && setenv("LXC_TARGET", "stop", 1))
 715                 SYSERROR("Failed to set environment variable: LXC_TARGET=stop.");
 716
 717         if (run_lxc_hooks(name, "stop", handler->conf, handler->lxcpath, namespaces))
 718                 ERROR("Failed to run lxc.hook.stop for container \"%s\".", name);
 719
 720         while (namespace_count--)
 721                 free(namespaces[namespace_count]);
 722         for (i = 0; i < LXC_NS_MAX; i++) {
 723                 if (handler->nsfd[i] != -1) {
 724                         close(handler->nsfd[i]);
 725                         handler->nsfd[i] = -1;
 726                 }
 727         }
 728
 729         if (handler->netnsfd >= 0) {
 730                 close(handler->netnsfd);
 731                 handler->netnsfd = -1;
 732         }
 733
 734         lxc_set_state(name, handler, STOPPED);
 735
 736         if (run_lxc_hooks(name, "post-stop", handler->conf, handler->lxcpath, NULL)) {
 737                 ERROR("Failed to run lxc.hook.post-stop for container \"%s\".", name);
 738                 if (handler->conf->reboot) {
 739                         WARN("Container will be stopped instead of rebooted.");
 740                         handler->conf->reboot = 0;
 741                         if (setenv("LXC_TARGET", "stop", 1))
 742                                 WARN("Failed to set environment variable: LXC_TARGET=stop.");
 743                 }
 744         }
 745
 746         /* Reset mask set by setup_signal_fd. */
 747         if (sigprocmask(SIG_SETMASK, &handler->oldmask, NULL))
 748                 WARN("Failed to restore signal mask.");
 749
 750         lxc_console_delete(&handler->conf->console);
 751         lxc_delete_tty(&handler->conf->tty_info);
 752
 753         /* close the command socket */
 754         close(handler->conf->maincmd_fd);
 755         handler->conf->maincmd_fd = -1;
 756
 757         /* The command socket is now closed, no more state clients can register
 758          * themselves from now on. So free the list of state clients.
 759          */
 760         lxc_list_for_each_safe(cur, &handler->state_clients, next) {
 761                 struct state_client *client = cur->elem;
 762                 /* close state client socket */
 763                 close(client->clientfd);
 764                 lxc_list_del(cur);
 765                 free(cur->elem);
 766                 free(cur);
 767         }
 768
 769         free(handler->name);
 770         if (handler->ttysock[0] != -1) {
 771                 close(handler->ttysock[0]);
 772                 close(handler->ttysock[1]);
 773         }
 774
 775         if (handler->conf->ephemeral == 1 && handler->conf->reboot != 1)
 776                 lxc_destroy_container_on_signal(handler, name);
 777
 778         cgroup_destroy(handler);
 779         free(handler);
 780 }
 781
 782 void lxc_abort(const char *name, struct lxc_handler *handler)
 783 {
 784         int ret, status;
 785
 786         lxc_set_state(name, handler, ABORTING);
 787         if (handler->pid > 0)
 788                 kill(handler->pid, SIGKILL);
 789         while ((ret = waitpid(-1, &status, 0)) > 0) {
 790                 ;
 791         }
 792 }
 793
 794 #include <sys/reboot.h>
 795 #include <linux/reboot.h>
 796
 797 /* reboot(LINUX_REBOOT_CMD_CAD_ON) will return -EINVAL in a child pid namespace
 798  * if container reboot support exists.  Otherwise, it will either succeed or
 799  * return -EPERM.
 800  */
 801 static int container_reboot_supported(void *arg)
 802 {
 803         int *cmd = arg;
 804         int ret;
 805
 806         ret = reboot(*cmd);
 807         if (ret == -1 && errno == EINVAL)
 808                 return 1;
 809         return 0;
 810 }
 811
 812 static int must_drop_cap_sys_boot(struct lxc_conf *conf)
 813 {
 814         FILE *f;
 815         int ret, cmd, v, flags;
 816         long stack_size = 4096;
 817         void *stack = alloca(stack_size);
 818         int status;
 819         pid_t pid;
 820
 821         f = fopen("/proc/sys/kernel/ctrl-alt-del", "r");
 822         if (!f) {
 823                 DEBUG("failed to open /proc/sys/kernel/ctrl-alt-del");
 824                 return 1;
 825         }
 826
 827         ret = fscanf(f, "%d", &v);
 828         fclose(f);
 829         if (ret != 1) {
 830                 DEBUG("Failed to read /proc/sys/kernel/ctrl-alt-del.");
 831                 return 1;
 832         }
 833         cmd = v ? LINUX_REBOOT_CMD_CAD_ON : LINUX_REBOOT_CMD_CAD_OFF;
 834
 835         flags = CLONE_NEWPID | SIGCHLD;
 836         if (!lxc_list_empty(&conf->id_map))
 837                 flags |= CLONE_NEWUSER;
 838
 839 #ifdef __ia64__
 840         pid = __clone2(container_reboot_supported, stack, stack_size, flags,  &cmd);
 841 #else
 842         stack += stack_size;
 843         pid = clone(container_reboot_supported, stack, flags, &cmd);
 844 #endif
 845         if (pid < 0) {
 846                 if (flags & CLONE_NEWUSER)
 847                         ERROR("Failed to clone (%#x): %s (includes CLONE_NEWUSER).", flags, strerror(errno));
 848                 else
 849                         ERROR("Failed to clone (%#x): %s.", flags, strerror(errno));
 850                 return -1;
 851         }
 852         if (wait(&status) < 0) {
 853                 SYSERROR("Unexpected wait error: %s.", strerror(errno));
 854                 return -1;
 855         }
 856
 857         if (WEXITSTATUS(status) != 1)
 858                 return 1;
 859
 860         return 0;
 861 }
 862
 863 /* netpipe is used in the unprivileged case to transfer the ifindexes from
 864  * parent to child
 865  */
 866 static int netpipe = -1;
 867
 868 static inline int count_veths(struct lxc_list *network)
 869 {
 870         struct lxc_list *iterator;
 871         struct lxc_netdev *netdev;
 872         int count = 0;
 873
 874         lxc_list_for_each(iterator, network) {
 875                 netdev = iterator->elem;
 876                 if (netdev->type != LXC_NET_VETH)
 877                         continue;
 878                 count++;
 879         }
 880         return count;
 881 }
 882
 883 static int read_unpriv_netifindex(struct lxc_list *network)
 884 {
 885         struct lxc_list *iterator;
 886         struct lxc_netdev *netdev;
 887
 888         if (netpipe == -1)
 889                 return 0;
 890         lxc_list_for_each(iterator, network) {
 891                 netdev = iterator->elem;
 892                 if (netdev->type != LXC_NET_VETH)
 893                         continue;
 894                 if (!(netdev->name = malloc(IFNAMSIZ))) {
 895                         ERROR("Out of memory.");
 896                         close(netpipe);
 897                         return -1;
 898                 }
 899                 if (read(netpipe, netdev->name, IFNAMSIZ) != IFNAMSIZ) {
 900                         close(netpipe);
 901                         return -1;
 902                 }
 903         }
 904         close(netpipe);
 905         return 0;
 906 }
 907
 908 static int do_start(void *data)
 909 {
 910         struct lxc_list *iterator;
 911         struct lxc_handler *handler = data;
 912         int devnull_fd = -1, ret;
 913         char path[PATH_MAX];
 914
 915         if (sigprocmask(SIG_SETMASK, &handler->oldmask, NULL)) {
 916                 SYSERROR("Failed to set signal mask.");
 917                 return -1;
 918         }
 919
 920         /* This prctl must be before the synchro, so if the parent dies before
 921          * we set the parent death signal, we will detect its death with the
 922          * synchro right after, otherwise we have a window where the parent can
 923          * exit before we set the pdeath signal leading to a unsupervized
 924          * container.
 925          */
 926         if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0)) {
 927                 SYSERROR("Failed to set PR_SET_PDEATHSIG to SIGKILL.");
 928                 return -1;
 929         }
 930
 931         lxc_sync_fini_parent(handler);
 932
 933         /* Don't leak the pinfd to the container. */
 934         if (handler->pinfd >= 0) {
 935                 close(handler->pinfd);
 936         }
 937
 938         if (lxc_sync_wait_parent(handler, LXC_SYNC_STARTUP))
 939                 return -1;
 940
 941         /* Unshare CLONE_NEWNET after CLONE_NEWUSER. See
 942          * https://github.com/lxc/lxd/issues/1978.
 943          */
 944         if ((handler->clone_flags & (CLONE_NEWNET | CLONE_NEWUSER)) ==
 945             (CLONE_NEWNET | CLONE_NEWUSER)) {
 946                 ret = unshare(CLONE_NEWNET);
 947                 if (ret < 0) {
 948                         SYSERROR("Failed to unshare CLONE_NEWNET.");
 949                         goto out_warn_father;
 950                 }
 951                 INFO("Unshared CLONE_NEWNET.");
 952         }
 953
 954         /* Tell the parent task it can begin to configure the container and wait
 955          * for it to finish.
 956          */
 957         if (lxc_sync_barrier_parent(handler, LXC_SYNC_CONFIGURE))
 958                 return -1;
 959
 960         if (read_unpriv_netifindex(&handler->conf->network) < 0)
 961                 goto out_warn_father;
 962
 963         /* If we are in a new user namespace, become root there to have
 964          * privilege over our namespace.
 965          */
 966         if (!lxc_list_empty(&handler->conf->id_map)) {
 967                 if (lxc_switch_uid_gid(0, 0) < 0)
 968                         goto out_warn_father;
 969
 970                 /* Drop groups only after we switched to a valid gid in the new
 971                  * user namespace.
 972                  */
 973                 if (lxc_setgroups(0, NULL) < 0)
 974                         goto out_warn_father;
 975         }
 976
 977         if (access(handler->lxcpath, X_OK)) {
 978                 print_top_failing_dir(handler->lxcpath);
 979                 goto out_warn_father;
 980         }
 981
 982         #if HAVE_LIBCAP
 983         if (handler->conf->need_utmp_watch) {
 984                 if (prctl(PR_CAPBSET_DROP, CAP_SYS_BOOT, 0, 0, 0)) {
 985                         SYSERROR("Failed to remove the CAP_SYS_BOOT capability.");
 986                         goto out_warn_father;
 987                 }
 988                 DEBUG("Dropped the CAP_SYS_BOOT capability.");
 989         }
 990         #endif
 991
 992         ret = snprintf(path, sizeof(path), "%s/dev/null", handler->conf->rootfs.mount);
 993         if (ret < 0 || ret >= sizeof(path))
 994                 goto out_warn_father;
 995
 996         /* In order to checkpoint restore, we need to have everything in the
 997          * same mount namespace. However, some containers may not have a
 998          * reasonable /dev (in particular, they may not have /dev/null), so we
 999          * can't set init's std fds to /dev/null by opening it from inside the
1000          * container.
1001          *
1002          * If that's the case, fall back to using the host's /dev/null. This
1003          * means that migration won't work, but at least we won't spew output
1004          * where it isn't wanted.
1005          */
1006         if (handler->backgrounded && !handler->conf->autodev && access(path, F_OK) < 0) {
1007                 devnull_fd = open_devnull();
1008
1009                 if (devnull_fd < 0)
1010                         goto out_warn_father;
1011                 WARN("Using /dev/null from the host for container init's "
1012                      "standard file descriptors. Migration will not work.");
1013         }
1014
1015         /* Setup the container, ip, names, utsname, ... */
1016         if (lxc_setup(handler)) {
1017                 ERROR("Failed to setup container \"%s\".", handler->name);
1018                 goto out_warn_father;
1019         }
1020
1021         /* Ask father to setup cgroups and wait for him to finish. */
1022         if (lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP))
1023                 goto out_error;
1024
1025         /* Unshare cgroup namespace after we have setup our cgroups. If we do it
1026          * earlier we end up with a wrong view of /proc/self/cgroup. For
1027          * example, assume we unshare(CLONE_NEWCGROUP) first, and then create
1028          * the cgroup for the container, say /sys/fs/cgroup/cpuset/lxc/c, then
1029          * /proc/self/cgroup would show us:
1030          *
1031          *      8:cpuset:/lxc/c
1032          *
1033          * whereas it should actually show
1034          *
1035          *      8:cpuset:/
1036          */
1037         if (cgns_supported()) {
1038                 if (unshare(CLONE_NEWCGROUP) < 0) {
1039                         INFO("Failed to unshare CLONE_NEWCGROUP.");
1040                         goto out_warn_father;
1041                 }
1042                 INFO("Unshared CLONE_NEWCGROUP.");
1043         }
1044
1045         /* Set the label to change to when we exec(2) the container's init. */
1046         if (lsm_process_label_set(NULL, handler->conf, 1, 1) < 0)
1047                 goto out_warn_father;
1048
1049         /* Set PR_SET_NO_NEW_PRIVS after we changed the lsm label. If we do it
1050          * before we aren't allowed anymore.
1051          */
1052         if (handler->conf->no_new_privs) {
1053                 if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) < 0) {
1054                         SYSERROR("Could not set PR_SET_NO_NEW_PRIVS to block execve() gainable privileges.");
1055                         goto out_warn_father;
1056                 }
1057                 DEBUG("Set PR_SET_NO_NEW_PRIVS to block execve() gainable privileges.");
1058         }
1059
1060         /* Some init's such as busybox will set sane tty settings on stdin,
1061          * stdout, stderr which it thinks is the console. We already set them
1062          * the way we wanted on the real terminal, and we want init to do its
1063          * setup on its console ie. the pty allocated in lxc_console_create() so
1064          * make sure that that pty is stdin,stdout,stderr.
1065          */
1066         if (lxc_console_set_stdfds(handler->conf->console.slave) < 0)
1067                 goto out_warn_father;
1068
1069         /* If we mounted a temporary proc, then unmount it now. */
1070         tmp_proc_unmount(handler->conf);
1071
1072         if (lxc_seccomp_load(handler->conf) != 0)
1073                 goto out_warn_father;
1074
1075         if (run_lxc_hooks(handler->name, "start", handler->conf, handler->lxcpath, NULL)) {
1076                 ERROR("Failed to run lxc.hook.start for container \"%s\".", handler->name);
1077                 goto out_warn_father;
1078         }
1079
1080         /* The container has been setup. We can now switch to an unprivileged
1081          * uid/gid.
1082          */
1083         if (handler->conf->is_execute) {
1084                 bool have_cap_setgid;
1085                 uid_t new_uid = handler->conf->init_uid;
1086                 gid_t new_gid = handler->conf->init_gid;
1087
1088                 /* If we are in a new user namespace we already dropped all
1089                  * groups when we switched to root in the new user namespace
1090                  * further above. Only drop groups if we can, so ensure that we
1091                  * have necessary privilege.
1092                  */
1093                 #if HAVE_LIBCAP
1094                 have_cap_setgid = lxc_proc_cap_is_set(CAP_SETGID, CAP_EFFECTIVE);
1095                 #else
1096                 have_cap_setgid = false;
1097                 #endif
1098                 if (lxc_list_empty(&handler->conf->id_map) && have_cap_setgid) {
1099                         if (lxc_setgroups(0, NULL) < 0)
1100                                 goto out_warn_father;
1101                 }
1102
1103                 if (lxc_switch_uid_gid(new_uid, new_gid) < 0)
1104                         goto out_warn_father;
1105         }
1106
1107         /* The clearenv() and putenv() calls have been moved here to allow us to
1108          * use environment variables passed to the various hooks, such as the
1109          * start hook above. Not all of the variables like CONFIG_PATH or ROOTFS
1110          * are valid in this context but others are.
1111          */
1112         if (clearenv()) {
1113                 SYSERROR("Failed to clear environment.");
1114                 /* Don't error out though. */
1115         }
1116
1117         lxc_list_for_each(iterator, &handler->conf->environment) {
1118                 if (putenv((char *)iterator->elem)) {
1119                         SYSERROR("Failed to set environment variable: %s.", (char *)iterator->elem);
1120                         goto out_warn_father;
1121                 }
1122         }
1123
1124         if (putenv("container=lxc")) {
1125                 SYSERROR("Failed to set environment variable: container=lxc.");
1126                 goto out_warn_father;
1127         }
1128
1129         if (handler->conf->pty_names) {
1130                 if (putenv(handler->conf->pty_names)) {
1131                         SYSERROR("Failed to set environment variable for container ptys.");
1132                         goto out_warn_father;
1133                 }
1134         }
1135
1136         close(handler->sigfd);
1137
1138         if (devnull_fd < 0) {
1139                 devnull_fd = open_devnull();
1140
1141                 if (devnull_fd < 0)
1142                         goto out_warn_father;
1143         }
1144
1145         if (handler->backgrounded && set_stdfds(devnull_fd))
1146                 goto out_warn_father;
1147
1148         if (devnull_fd >= 0) {
1149                 close(devnull_fd);
1150                 devnull_fd = -1;
1151         }
1152
1153         setsid();
1154
1155         /* After this call, we are in error because this ops should not return
1156          * as it execs.
1157          */
1158         handler->ops->start(handler, handler->data);
1159
1160 out_warn_father:
1161         /* We want the parent to know something went wrong, so we return a
1162          * special error code.
1163          */
1164         lxc_sync_wake_parent(handler, LXC_SYNC_ERROR);
1165
1166 out_error:
1167         if (devnull_fd >= 0)
1168                 close(devnull_fd);
1169
1170         return -1;
1171 }
1172
1173 static int save_phys_nics(struct lxc_conf *conf)
1174 {
1175         struct lxc_list *iterator;
1176         int am_root = (getuid() == 0);
1177
1178         if (!am_root)
1179                 return 0;
1180
1181         lxc_list_for_each(iterator, &conf->network) {
1182                 struct lxc_netdev *netdev = iterator->elem;
1183
1184                 if (netdev->type != LXC_NET_PHYS)
1185                         continue;
1186                 conf->saved_nics = realloc(conf->saved_nics,
1187                                 (conf->num_savednics+1)*sizeof(struct saved_nic));
1188                 if (!conf->saved_nics)
1189                         return -1;
1190                 conf->saved_nics[conf->num_savednics].ifindex = netdev->ifindex;
1191                 conf->saved_nics[conf->num_savednics].orig_name = strdup(netdev->link);
1192                 if (!conf->saved_nics[conf->num_savednics].orig_name)
1193                         return -1;
1194                 INFO("Stored saved_nic #%d idx %d name %s.", conf->num_savednics,
1195                         conf->saved_nics[conf->num_savednics].ifindex,
1196                         conf->saved_nics[conf->num_savednics].orig_name);
1197                 conf->num_savednics++;
1198         }
1199
1200         return 0;
1201 }
1202
1203 static int lxc_recv_ttys_from_child(struct lxc_handler *handler)
1204 {
1205         int i;
1206         int *ttyfds;
1207         struct lxc_pty_info *pty_info;
1208         int ret = -1;
1209         int sock = handler->ttysock[1];
1210         struct lxc_conf *conf = handler->conf;
1211         struct lxc_tty_info *tty_info = &conf->tty_info;
1212         size_t num_ttyfds = (2 * conf->tty);
1213
1214         if (!conf->tty)
1215                 return 0;
1216
1217         tty_info->pty_info = malloc(sizeof(*tty_info->pty_info) * conf->tty);
1218         if (!tty_info->pty_info)
1219                 return -1;
1220
1221         ttyfds = malloc(num_ttyfds * sizeof(int));
1222         if (!ttyfds)
1223                 return -1;
1224
1225         ret = lxc_abstract_unix_recv_fds(sock, ttyfds, num_ttyfds, NULL, 0);
1226         for (i = 0; (ret >= 0 && *ttyfds != -1) && (i < num_ttyfds); i++) {
1227                 pty_info = &tty_info->pty_info[i / 2];
1228                 pty_info->busy = 0;
1229                 pty_info->slave = ttyfds[i++];
1230                 pty_info->master = ttyfds[i];
1231                 TRACE("received pty with master fd %d and slave fd %d from "
1232                       "parent", pty_info->master, pty_info->slave);
1233         }
1234
1235         tty_info->nbtty = conf->tty;
1236
1237         free(ttyfds);
1238
1239         if (ret < 0)
1240                 ERROR("failed to receive %d ttys from child: %s", conf->tty,
1241                       strerror(errno));
1242         else
1243                 TRACE("received %d ttys from child", conf->tty);
1244
1245         return ret;
1246 }
1247
1248 void resolve_clone_flags(struct lxc_handler *handler)
1249 {
1250         handler->clone_flags = CLONE_NEWPID | CLONE_NEWNS;
1251
1252         if (!lxc_list_empty(&handler->conf->id_map))
1253                 handler->clone_flags |= CLONE_NEWUSER;
1254
1255         if (handler->conf->inherit_ns_fd[LXC_NS_NET] == -1) {
1256                 if (!lxc_requests_empty_network(handler))
1257                         handler->clone_flags |= CLONE_NEWNET;
1258         } else {
1259                 INFO("Inheriting a NET namespace.");
1260         }
1261
1262         if (handler->conf->inherit_ns_fd[LXC_NS_IPC] == -1)
1263                 handler->clone_flags |= CLONE_NEWIPC;
1264         else
1265                 INFO("Inheriting an IPC namespace.");
1266
1267         if (handler->conf->inherit_ns_fd[LXC_NS_UTS] == -1)
1268                 handler->clone_flags |= CLONE_NEWUTS;
1269         else
1270                 INFO("Inheriting a UTS namespace.");
1271 }
1272
1273 /* lxc_spawn() performs crucial setup tasks and clone()s the new process which
1274  * exec()s the requested container binary.
1275  * Note that lxc_spawn() runs in the parent namespaces. Any operations performed
1276  * right here should be double checked if they'd pose a security risk. (For
1277  * example, any {u}mount() operations performed here will be reflected on the
1278  * host!)
1279  */
1280 static int lxc_spawn(struct lxc_handler *handler)
1281 {
1282         int failed_before_rename = 0;
1283         const char *name = handler->name;
1284         bool cgroups_connected = false;
1285         int saved_ns_fd[LXC_NS_MAX];
1286         int preserve_mask = 0, i, flags;
1287         int netpipepair[2], nveths;
1288         bool wants_to_map_ids;
1289         struct lxc_list *id_map;
1290
1291         netpipe = -1;
1292         id_map = &handler->conf->id_map;
1293         wants_to_map_ids = !lxc_list_empty(id_map);
1294
1295         for (i = 0; i < LXC_NS_MAX; i++)
1296                 if (handler->conf->inherit_ns_fd[i] != -1)
1297                         preserve_mask |= ns_info[i].clone_flag;
1298
1299         if (lxc_sync_init(handler))
1300                 return -1;
1301
1302         if (socketpair(AF_UNIX, SOCK_DGRAM, 0, handler->ttysock) < 0) {
1303                 lxc_sync_fini(handler);
1304                 return -1;
1305         }
1306
1307         resolve_clone_flags(handler);
1308
1309         if (handler->clone_flags & CLONE_NEWNET) {
1310                 if (!lxc_list_empty(&handler->conf->network)) {
1311
1312                         /* Find gateway addresses from the link device, which is
1313                          * no longer accessible inside the container. Do this
1314                          * before creating network interfaces, since goto
1315                          * out_delete_net does not work before lxc_clone.
1316                          */
1317                         if (lxc_find_gateway_addresses(handler)) {
1318                                 ERROR("Failed to find gateway addresses.");
1319                                 lxc_sync_fini(handler);
1320                                 return -1;
1321                         }
1322
1323                         /* That should be done before the clone because we will
1324                          * fill the netdev index and use them in the child.
1325                          */
1326                         if (lxc_setup_networks_in_parent_namespaces(handler)) {
1327                                 ERROR("Failed to create the network.");
1328                                 lxc_sync_fini(handler);
1329                                 return -1;
1330                         }
1331                 }
1332
1333                 if (save_phys_nics(handler->conf)) {
1334                         ERROR("Failed to save physical nic info.");
1335                         goto out_abort;
1336                 }
1337         }
1338
1339         if (!cgroup_init(handler)) {
1340                 ERROR("Failed initializing cgroup support.");
1341                 goto out_delete_net;
1342         }
1343
1344         cgroups_connected = true;
1345
1346         if (!cgroup_create(handler)) {
1347                 ERROR("Failed creating cgroups.");
1348                 goto out_delete_net;
1349         }
1350
1351         /* If the rootfs is not a blockdev, prevent the container from marking
1352          * it readonly.
1353          * If the container is unprivileged then skip rootfs pinning.
1354          */
1355         if (!wants_to_map_ids) {
1356                 handler->pinfd = pin_rootfs(handler->conf->rootfs.path);
1357                 if (handler->pinfd == -1)
1358                         INFO("Failed to pin the rootfs for container \"%s\".", handler->name);
1359         }
1360
1361         if (!preserve_ns(saved_ns_fd, preserve_mask, getpid()))
1362                 goto out_delete_net;
1363
1364         if (attach_ns(handler->conf->inherit_ns_fd) < 0)
1365                 goto out_delete_net;
1366
1367         if (am_unpriv() && (nveths = count_veths(&handler->conf->network))) {
1368                 if (pipe(netpipepair) < 0) {
1369                         SYSERROR("Failed to create pipe.");
1370                         goto out_delete_net;
1371                 }
1372                 /* Store netpipe in the global var for do_start's use. */
1373                 netpipe = netpipepair[0];
1374         }
1375
1376         /* Create a process in a new set of namespaces. */
1377         flags = handler->clone_flags;
1378         if (handler->clone_flags & CLONE_NEWUSER) {
1379                 /* If CLONE_NEWUSER and CLONE_NEWNET was requested, we need to
1380                  * clone a new user namespace first and only later unshare our
1381                  * network namespace to ensure that network devices ownership is
1382                  * set up correctly.
1383                  */
1384                 flags &= ~CLONE_NEWNET;
1385         }
1386         handler->pid = lxc_clone(do_start, handler, flags);
1387         if (handler->pid < 0) {
1388                 SYSERROR("Failed to clone a new set of namespaces.");
1389                 goto out_delete_net;
1390         }
1391         for (i = 0; i < LXC_NS_MAX; i++)
1392                 if (flags & ns_info[i].clone_flag)
1393                         INFO("Cloned %s.", ns_info[i].flag_name);
1394
1395         if (!preserve_ns(handler->nsfd, handler->clone_flags | preserve_mask, handler->pid))
1396                 INFO("Failed to preserve namespace for lxc.hook.stop.");
1397
1398         if (attach_ns(saved_ns_fd))
1399                 WARN("Failed to restore saved namespaces.");
1400
1401         lxc_sync_fini_child(handler);
1402
1403         /* Map the container uids. The container became an invalid userid the
1404          * moment it was cloned with CLONE_NEWUSER. This call doesn't change
1405          * anything immediately, but allows the container to setuid(0) (0 being
1406          * mapped to something else on the host.) later to become a valid uid
1407          * again.
1408          */
1409         if (wants_to_map_ids && lxc_map_ids(id_map, handler->pid)) {
1410                 ERROR("Failed to set up id mapping.");
1411                 goto out_delete_net;
1412         }
1413
1414         if (lxc_sync_wake_child(handler, LXC_SYNC_STARTUP)) {
1415                 failed_before_rename = 1;
1416                 goto out_delete_net;
1417         }
1418
1419         if (lxc_sync_wait_child(handler, LXC_SYNC_CONFIGURE)) {
1420                 failed_before_rename = 1;
1421                 goto out_delete_net;
1422         }
1423
1424         if (!cgroup_create_legacy(handler)) {
1425                 ERROR("Failed to setup legacy cgroups for container \"%s\".", name);
1426                 goto out_delete_net;
1427         }
1428         if (!cgroup_setup_limits(handler, false)) {
1429                 ERROR("Failed to setup cgroup limits for container \"%s\".", name);
1430                 goto out_delete_net;
1431         }
1432
1433         if (!cgroup_enter(handler))
1434                 goto out_delete_net;
1435
1436         if (!cgroup_chown(handler))
1437                 goto out_delete_net;
1438
1439         if (failed_before_rename)
1440                 goto out_delete_net;
1441
1442         /* Create the network configuration. */
1443         if (handler->clone_flags & CLONE_NEWNET) {
1444                 if (lxc_assign_network(handler->lxcpath, handler->name,
1445                                        &handler->conf->network, handler->pid)) {
1446                         ERROR("Failed to create the configured network.");
1447                         goto out_delete_net;
1448                 }
1449         }
1450
1451         if (netpipe != -1) {
1452                 struct lxc_list *iterator;
1453                 struct lxc_netdev *netdev;
1454
1455                 close(netpipe);
1456                 lxc_list_for_each(iterator, &handler->conf->network) {
1457                         netdev = iterator->elem;
1458                         if (netdev->type != LXC_NET_VETH)
1459                                 continue;
1460                         if (write(netpipepair[1], netdev->name, IFNAMSIZ) != IFNAMSIZ) {
1461                                 ERROR("Error writing veth name to container.");
1462                                 goto out_delete_net;
1463                         }
1464                 }
1465                 close(netpipepair[1]);
1466         }
1467
1468         /* Tell the child to continue its initialization. We'll get
1469          * LXC_SYNC_CGROUP when it is ready for us to setup cgroups.
1470          */
1471         if (lxc_sync_barrier_child(handler, LXC_SYNC_POST_CONFIGURE))
1472                 goto out_delete_net;
1473
1474         if (!lxc_list_empty(&handler->conf->limits) && setup_resource_limits(&handler->conf->limits, handler->pid)) {
1475                 ERROR("failed to setup resource limits for '%s'", name);
1476                 goto out_delete_net;
1477         }
1478
1479         if (!cgroup_setup_limits(handler, true)) {
1480                 ERROR("Failed to setup the devices cgroup for container \"%s\".", name);
1481                 goto out_delete_net;
1482         }
1483
1484         cgroup_disconnect();
1485         cgroups_connected = false;
1486
1487         /* Read tty fds allocated by child. */
1488         if (lxc_recv_ttys_from_child(handler) < 0) {
1489                 ERROR("Failed to receive tty info from child process.");
1490                 goto out_delete_net;
1491         }
1492
1493         /* Tell the child to complete its initialization and wait for it to exec
1494          * or return an error. (The child will never return
1495          * LXC_SYNC_POST_CGROUP+1. It will either close the sync pipe, causing
1496          * lxc_sync_barrier_child to return success, or return a different
1497          * value, causing us to error out).
1498          */
1499         if (lxc_sync_barrier_child(handler, LXC_SYNC_POST_CGROUP))
1500                 return -1;
1501
1502         if (handler->ops->post_start(handler, handler->data))
1503                 goto out_abort;
1504
1505         if (lxc_set_state(name, handler, RUNNING)) {
1506                 ERROR("Failed to set state for container \"%s\" to \"%s\".", name,
1507                       lxc_state2str(RUNNING));
1508                 goto out_abort;
1509         }
1510
1511         lxc_sync_fini(handler);
1512         handler->netnsfd = lxc_preserve_ns(handler->pid, "net");
1513
1514         return 0;
1515
1516 out_delete_net:
1517         if (cgroups_connected)
1518                 cgroup_disconnect();
1519         if (handler->clone_flags & CLONE_NEWNET)
1520                 lxc_delete_network(handler);
1521 out_abort:
1522         lxc_abort(name, handler);
1523         lxc_sync_fini(handler);
1524         if (handler->pinfd >= 0) {
1525                 close(handler->pinfd);
1526                 handler->pinfd = -1;
1527         }
1528
1529         return -1;
1530 }
1531
1532 int __lxc_start(const char *name, struct lxc_handler *handler,
1533                 struct lxc_operations* ops, void *data, const char *lxcpath,
1534                 bool backgrounded)
1535 {
1536         int status;
1537         int err = -1;
1538         bool removed_all_netdevs = true;
1539         struct lxc_conf *conf = handler->conf;
1540
1541         if (lxc_init(name, handler) < 0) {
1542                 ERROR("Failed to initialize container \"%s\".", name);
1543                 return -1;
1544         }
1545         handler->ops = ops;
1546         handler->data = data;
1547         handler->backgrounded = backgrounded;
1548         handler->netnsfd = -1;
1549
1550         if (must_drop_cap_sys_boot(handler->conf)) {
1551                 #if HAVE_LIBCAP
1552                 DEBUG("Dropping CAP_SYS_BOOT capability.");
1553                 #else
1554                 DEBUG("Not dropping CAP_SYS_BOOT capability as capabilities aren't supported.");
1555                 #endif
1556         } else {
1557                 DEBUG("Not dropping CAP_SYS_BOOT or watching utmp.");
1558                 handler->conf->need_utmp_watch = 0;
1559         }
1560
1561         if (!attach_block_device(handler->conf)) {
1562                 ERROR("Failed to attach block device.");
1563                 goto out_fini_nonet;
1564         }
1565
1566         if (geteuid() == 0 && !lxc_list_empty(&conf->id_map)) {
1567                 /* If the backing store is a device, mount it here and now. */
1568                 if (rootfs_is_blockdev(conf)) {
1569                         if (unshare(CLONE_NEWNS) < 0) {
1570                                 ERROR("Failed to unshare CLONE_NEWNS.");
1571                                 goto out_fini_nonet;
1572                         }
1573                         INFO("Unshared CLONE_NEWNS.");
1574
1575                         remount_all_slave();
1576                         if (do_rootfs_setup(conf, name, lxcpath) < 0) {
1577                                 ERROR("Error setting up rootfs mount as root before spawn.");
1578                                 goto out_fini_nonet;
1579                         }
1580                         INFO("Set up container rootfs as host root.");
1581                 }
1582         }
1583
1584         err = lxc_spawn(handler);
1585         if (err) {
1586                 ERROR("Failed to spawn container \"%s\".", name);
1587                 goto out_detach_blockdev;
1588         }
1589
1590         handler->conf->reboot = 0;
1591
1592         err = lxc_poll(name, handler);
1593         if (err) {
1594                 ERROR("LXC mainloop exited with error: %d.", err);
1595                 if (handler->netnsfd >= 0) {
1596                         close(handler->netnsfd);
1597                         handler->netnsfd = -1;
1598                 }
1599                 goto out_abort;
1600         }
1601
1602         while (waitpid(handler->pid, &status, 0) < 0 && errno == EINTR)
1603                 continue;
1604
1605         /* If the child process exited but was not signaled, it didn't call
1606          * reboot. This should mean it was an lxc-execute which simply exited.
1607          * In any case, treat it as a 'halt'.
1608          */
1609         if (WIFSIGNALED(status)) {
1610                 switch(WTERMSIG(status)) {
1611                 case SIGINT: /* halt */
1612                         DEBUG("Container \"%s\" is halting.", name);
1613                         break;
1614                 case SIGHUP: /* reboot */
1615                         DEBUG("Container \"%s\" is rebooting.", name);
1616                         handler->conf->reboot = 1;
1617                         break;
1618                 case SIGSYS: /* seccomp */
1619                         DEBUG("Container \"%s\" violated its seccomp policy.", name);
1620                         break;
1621                 default:
1622                         DEBUG("Unknown exit status for container \"%s\" init %d.", name, WTERMSIG(status));
1623                         break;
1624                 }
1625         }
1626
1627         DEBUG("Pushing physical nics back to host namespace");
1628         lxc_restore_phys_nics_to_netns(handler->netnsfd, handler->conf);
1629
1630         DEBUG("Tearing down virtual network devices used by container \"%s\".", name);
1631         removed_all_netdevs = lxc_delete_network(handler);
1632
1633         if (handler->pinfd >= 0) {
1634                 close(handler->pinfd);
1635                 handler->pinfd = -1;
1636         }
1637
1638         lxc_monitor_send_exit_code(name, status, handler->lxcpath);
1639         err =  lxc_error_set_and_log(handler->pid, status);
1640 out_fini:
1641         if (!removed_all_netdevs) {
1642                 DEBUG("Failed tearing down network devices used by container. Trying again!");
1643                 removed_all_netdevs = lxc_delete_network(handler);
1644                 if (!removed_all_netdevs)
1645                         DEBUG("Failed tearing down network devices used by container. Not trying again!");
1646         }
1647
1648 out_detach_blockdev:
1649         detach_block_device(handler->conf);
1650
1651 out_fini_nonet:
1652         lxc_fini(name, handler);
1653         return err;
1654
1655 out_abort:
1656         lxc_abort(name, handler);
1657         goto out_fini;
1658 }
1659
1660 struct start_args {
1661         char *const *argv;
1662 };
1663
1664 static int start(struct lxc_handler *handler, void* data)
1665 {
1666         struct start_args *arg = data;
1667
1668         NOTICE("Exec'ing \"%s\".", arg->argv[0]);
1669
1670         execvp(arg->argv[0], arg->argv);
1671         SYSERROR("Failed to exec \"%s\".", arg->argv[0]);
1672         return 0;
1673 }
1674
1675 static int post_start(struct lxc_handler *handler, void* data)
1676 {
1677         struct start_args *arg = data;
1678
1679         NOTICE("Started \"%s\" with pid \"%d\".", arg->argv[0], handler->pid);
1680         return 0;
1681 }
1682
1683 static struct lxc_operations start_ops = {
1684         .start = start,
1685         .post_start = post_start
1686 };
1687
1688 int lxc_start(const char *name, char *const argv[], struct lxc_handler *handler,
1689               const char *lxcpath, bool backgrounded)
1690 {
1691         struct start_args start_arg = {
1692                 .argv = argv,
1693         };
1694
1695         handler->conf->need_utmp_watch = 1;
1696         return __lxc_start(name, handler, &start_ops, &start_arg, lxcpath, backgrounded);
1697 }
1698
1699 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
1700                                             const char *name)
1701 {
1702         char destroy[MAXPATHLEN];
1703         bool bret = true;
1704         int ret = 0;
1705         struct lxc_container *c;
1706         if (handler->conf->rootfs.path && handler->conf->rootfs.mount) {
1707                 bret = do_destroy_container(handler->conf);
1708                 if (!bret) {
1709                         ERROR("Error destroying rootfs for container \"%s\".", name);
1710                         return;
1711                 }
1712         }
1713         INFO("Destroyed rootfs for container \"%s\".", name);
1714
1715         ret = snprintf(destroy, MAXPATHLEN, "%s/%s", handler->lxcpath, name);
1716         if (ret < 0 || ret >= MAXPATHLEN) {
1717                 ERROR("Error destroying directory for container \"%s\".", name);
1718                 return;
1719         }
1720
1721         c = lxc_container_new(name, handler->lxcpath);
1722         if (c) {
1723                 if (container_disk_lock(c)) {
1724                         INFO("Could not update lxc_snapshots file.");
1725                         lxc_container_put(c);
1726                 } else {
1727                         mod_all_rdeps(c, false);
1728                         container_disk_unlock(c);
1729                         lxc_container_put(c);
1730                 }
1731         }
1732
1733         if (am_unpriv())
1734                 ret = userns_exec_1(handler->conf, lxc_rmdir_onedev_wrapper,
1735                                     destroy, "lxc_rmdir_onedev_wrapper");
1736         else
1737                 ret = lxc_rmdir_onedev(destroy, NULL);
1738
1739         if (ret < 0) {
1740                 ERROR("Error destroying directory for container \"%s\".", name);
1741                 return;
1742         }
1743         INFO("Destroyed directory for container \"%s\".", name);
1744 }
1745
1746 static int lxc_rmdir_onedev_wrapper(void *data)
1747 {
1748         char *arg = (char *) data;
1749         return lxc_rmdir_onedev(arg, NULL);
1750 }
1751
1752 static bool do_destroy_container(struct lxc_conf *conf) {
1753         if (am_unpriv()) {
1754                 if (userns_exec_1(conf, bdev_destroy_wrapper, conf,
1755                                   "bdev_destroy_wrapper") < 0)
1756                         return false;
1757                 return true;
1758         }
1759         return bdev_destroy(conf);
1760 }