src/lxc/start.c

   1 /*
   2  * lxc: linux Container library
   3  *
   4  * (C) Copyright IBM Corp. 2007, 2008
   5  *
   6  * Authors:
   7  * Daniel Lezcano <daniel.lezcano at free.fr>
   8  * Serge Hallyn <serge@hallyn.com>
   9  * Christian Brauner <christian.brauner@ubuntu.com>
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2.1 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  24  */
  25
  26 #define _GNU_SOURCE
  27 #include "config.h"
  28
  29 #include <alloca.h>
  30 #include <dirent.h>
  31 #include <errno.h>
  32 #include <fcntl.h>
  33 #include <grp.h>
  34 #include <poll.h>
  35 #include <signal.h>
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <sys/file.h>
  41 #include <sys/mount.h>
  42 #include <sys/param.h>
  43 #include <sys/prctl.h>
  44 #include <sys/socket.h>
  45 #include <sys/stat.h>
  46 #include <sys/syscall.h>
  47 #include <sys/types.h>
  48 #include <sys/un.h>
  49 #include <sys/wait.h>
  50
  51 #if HAVE_LIBCAP
  52 #include <sys/capability.h>
  53 #endif
  54
  55 #if !HAVE_DECL_PR_CAPBSET_DROP
  56 #define PR_CAPBSET_DROP 24
  57 #endif
  58
  59 #if !HAVE_DECL_PR_SET_NO_NEW_PRIVS
  60 #define PR_SET_NO_NEW_PRIVS 38
  61 #endif
  62
  63 #if !HAVE_DECL_PR_GET_NO_NEW_PRIVS
  64 #define PR_GET_NO_NEW_PRIVS 39
  65 #endif
  66
  67 #include "af_unix.h"
  68 #include "caps.h"
  69 #include "cgroup.h"
  70 #include "commands.h"
  71 #include "commands_utils.h"
  72 #include "conf.h"
  73 #include "confile_utils.h"
  74 #include "console.h"
  75 #include "error.h"
  76 #include "list.h"
  77 #include "log.h"
  78 #include "lxccontainer.h"
  79 #include "lxclock.h"
  80 #include "lxcseccomp.h"
  81 #include "mainloop.h"
  82 #include "monitor.h"
  83 #include "namespace.h"
  84 #include "network.h"
  85 #include "start.h"
  86 #include "sync.h"
  87 #include "utils.h"
  88 #include "lsm/lsm.h"
  89 #include "storage/storage.h"
  90 #include "storage/storage_utils.h"
  91
  92 lxc_log_define(lxc_start, lxc);
  93
  94 extern void mod_all_rdeps(struct lxc_container *c, bool inc);
  95 static bool do_destroy_container(struct lxc_handler *handler);
  96 static int lxc_rmdir_onedev_wrapper(void *data);
  97 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
  98                                             const char *name);
  99
 100 static void print_top_failing_dir(const char *path)
 101 {
 102         int ret;
 103         size_t len;
 104         char *copy, *e, *p, saved;
 105
 106         len = strlen(path);
 107         copy = alloca(len + 1);
 108         strcpy(copy, path);
 109         p = copy;
 110         e = copy + len;
 111         while (p < e) {
 112                 while (p < e && *p == '/')
 113                         p++;
 114
 115                 while (p < e && *p != '/')
 116                         p++;
 117
 118                 saved = *p;
 119                 *p = '\0';
 120
 121                 ret = access(copy, X_OK);
 122                 if (ret != 0) {
 123                         SYSERROR("Could not access %s. Please grant it x "
 124                                  "access, or add an ACL for the container "
 125                                  "root", copy);
 126                         return;
 127                 }
 128                 *p = saved;
 129         }
 130 }
 131
 132 static void lxc_put_nsfds(struct lxc_handler *handler)
 133 {
 134         int i;
 135
 136         for (i = 0; i < LXC_NS_MAX; i++) {
 137                 if (handler->nsfd[i] < 0)
 138                         continue;
 139
 140                 close(handler->nsfd[i]);
 141                 handler->nsfd[i] = -EBADF;
 142         }
 143 }
 144
 145 /* lxc_preserve_namespaces: open /proc/@pid/ns/@ns for each namespace specified
 146  * in ns_clone_flags.
 147  * Return true on success, false on failure.
 148  */
 149 static bool lxc_preserve_namespaces(struct lxc_handler *handler, int ns_clone_flags, pid_t pid)
 150 {
 151         int i;
 152
 153         for (i = 0; i < LXC_NS_MAX; i++)
 154                 handler->nsfd[i] = -EBADF;
 155
 156         for (i = 0; i < LXC_NS_MAX; i++) {
 157                 if ((ns_clone_flags & ns_info[i].clone_flag) == 0)
 158                         continue;
 159
 160                 handler->nsfd[i] = lxc_preserve_ns(pid, ns_info[i].proc_name);
 161                 if (handler->nsfd[i] < 0)
 162                         goto error;
 163
 164                 DEBUG("Preserved %s namespace via fd %d", ns_info[i].proc_name, handler->nsfd[i]);
 165         }
 166
 167         return true;
 168
 169 error:
 170         if (errno == ENOENT)
 171                 SYSERROR("Kernel does not support attaching to %s namespaces",
 172                          ns_info[i].proc_name);
 173         else
 174                 SYSERROR("Failed to open file descriptor for %s namespace",
 175                          ns_info[i].proc_name);
 176         lxc_put_nsfds(handler);
 177         return false;
 178 }
 179
 180 static int match_fd(int fd)
 181 {
 182         return (fd == 0 || fd == 1 || fd == 2);
 183 }
 184
 185 int lxc_check_inherited(struct lxc_conf *conf, bool closeall,
 186                         int *fds_to_ignore, size_t len_fds)
 187 {
 188         int fd, fddir;
 189         size_t i;
 190         DIR *dir;
 191         struct dirent *direntp;
 192
 193         if (conf && conf->close_all_fds)
 194                 closeall = true;
 195
 196 restart:
 197         dir = opendir("/proc/self/fd");
 198         if (!dir) {
 199                 WARN("%s - Failed to open directory", strerror(errno));
 200                 return -1;
 201         }
 202
 203         fddir = dirfd(dir);
 204
 205         while ((direntp = readdir(dir))) {
 206                 int ret;
 207                 struct lxc_list *cur;
 208                 bool matched = false;
 209
 210                 if (strcmp(direntp->d_name, ".") == 0)
 211                         continue;
 212
 213                 if (strcmp(direntp->d_name, "..") == 0)
 214                         continue;
 215
 216                 ret = lxc_safe_int(direntp->d_name, &fd);
 217                 if (ret < 0) {
 218                         INFO("Could not parse file descriptor for \"%s\"", direntp->d_name);
 219                         continue;
 220                 }
 221
 222                 for (i = 0; i < len_fds; i++)
 223                         if (fds_to_ignore[i] == fd)
 224                                 break;
 225
 226                 if (fd == fddir || fd == lxc_log_fd ||
 227                     (i < len_fds && fd == fds_to_ignore[i]))
 228                         continue;
 229
 230                 /* Keep state clients that wait on reboots. */
 231                 if (conf) {
 232                         lxc_list_for_each(cur, &conf->state_clients) {
 233                                 struct lxc_state_client *client = cur->elem;
 234
 235                                 if (client->clientfd != fd)
 236                                         continue;
 237
 238                                 matched = true;
 239                                 break;
 240                         }
 241                 }
 242
 243                 if (matched)
 244                         continue;
 245
 246                 if (current_config && fd == current_config->logfd)
 247                         continue;
 248
 249                 if (match_fd(fd))
 250                         continue;
 251
 252                 if (closeall) {
 253                         close(fd);
 254                         closedir(dir);
 255                         INFO("Closed inherited fd %d", fd);
 256                         goto restart;
 257                 }
 258                 WARN("Inherited fd %d", fd);
 259         }
 260
 261         /* Only enable syslog at this point to avoid the above logging function
 262          * to open a new fd and make the check_inherited function enter an
 263          * infinite loop.
 264          */
 265         lxc_log_enable_syslog();
 266
 267         closedir(dir); /* cannot fail */
 268         return 0;
 269 }
 270
 271 static int setup_signal_fd(sigset_t *oldmask)
 272 {
 273         int ret, sig;
 274         sigset_t mask;
 275         int signals[] = {SIGBUS, SIGILL, SIGSEGV, SIGWINCH};
 276
 277         /* Block everything except serious error signals. */
 278         ret = sigfillset(&mask);
 279         if (ret < 0)
 280                 return -EBADF;
 281
 282         for (sig = 0; sig < (sizeof(signals) / sizeof(signals[0])); sig++) {
 283                 ret = sigdelset(&mask, signals[sig]);
 284                 if (ret < 0)
 285                         return -EBADF;
 286         }
 287
 288         ret = sigprocmask(SIG_BLOCK, &mask, oldmask);
 289         if (ret < 0) {
 290                 SYSERROR("Failed to set signal mask");
 291                 return -EBADF;
 292         }
 293
 294         ret = signalfd(-1, &mask, SFD_CLOEXEC);
 295         if (ret < 0) {
 296                 SYSERROR("Failed to create signal file descriptor");
 297                 return -EBADF;
 298         }
 299
 300         TRACE("Created signal file descriptor %d", ret);
 301
 302         return ret;
 303 }
 304
 305 static int signal_handler(int fd, uint32_t events, void *data,
 306                           struct lxc_epoll_descr *descr)
 307 {
 308         int ret;
 309         siginfo_t info;
 310         struct signalfd_siginfo siginfo;
 311         struct lxc_handler *hdlr = data;
 312
 313         ret = read(fd, &siginfo, sizeof(siginfo));
 314         if (ret < 0) {
 315                 ERROR("Failed to read signal info from signal file descriptor %d", fd);
 316                 return -1;
 317         }
 318
 319         if (ret != sizeof(siginfo)) {
 320                 ERROR("Unexpected size for struct signalfd_siginfo");
 321                 return -EINVAL;
 322         }
 323
 324         /* Check whether init is running. */
 325         info.si_pid = 0;
 326         ret = waitid(P_PID, hdlr->pid, &info, WEXITED | WNOWAIT | WNOHANG);
 327         if (ret == 0 && info.si_pid == hdlr->pid)
 328                 hdlr->init_died = true;
 329
 330         /* Try to figure out a reasonable exit status to report. */
 331         if (hdlr->init_died) {
 332                 switch (info.si_code) {
 333                 case CLD_EXITED:
 334                         hdlr->exit_status = info.si_status << 8;
 335                         break;
 336                 case CLD_KILLED:
 337                 case CLD_DUMPED:
 338                 case CLD_STOPPED:
 339                         hdlr->exit_status = info.si_status << 8 | 0x7f;
 340                         break;
 341                 case CLD_CONTINUED:
 342                         /* Huh? The waitid() told us it's dead *and* continued? */
 343                         WARN("Init %d dead and continued?", hdlr->pid);
 344                         hdlr->exit_status = 1;
 345                         break;
 346                 default:
 347                         ERROR("Unknown si_code: %d", hdlr->init_died);
 348                         hdlr->exit_status = 1;
 349                 }
 350         }
 351
 352         if (siginfo.ssi_signo == SIGHUP) {
 353                 kill(hdlr->pid, SIGTERM);
 354                 INFO("Killing %d since terminal hung up", hdlr->pid);
 355                 return hdlr->init_died ? LXC_MAINLOOP_CLOSE : 0;
 356         }
 357
 358         /* More robustness, protect ourself from a SIGCHLD sent
 359          * by a process different from the container init.
 360          */
 361         if (siginfo.ssi_pid != hdlr->pid) {
 362                 NOTICE("Received %d from pid %d instead of container init %d",
 363                        siginfo.ssi_signo, siginfo.ssi_pid, hdlr->pid);
 364                 return hdlr->init_died ? LXC_MAINLOOP_CLOSE : 0;
 365         }
 366
 367         if (siginfo.ssi_signo != SIGCHLD) {
 368                 kill(hdlr->pid, siginfo.ssi_signo);
 369                 INFO("Forwarded signal %d to pid %d", siginfo.ssi_signo, hdlr->pid);
 370                 return hdlr->init_died ? LXC_MAINLOOP_CLOSE : 0;
 371         }
 372
 373         if (siginfo.ssi_code == CLD_STOPPED) {
 374                 INFO("Container init process was stopped");
 375                 return hdlr->init_died ? LXC_MAINLOOP_CLOSE : 0;
 376         } else if (siginfo.ssi_code == CLD_CONTINUED) {
 377                 INFO("Container init process was continued");
 378                 return hdlr->init_died ? LXC_MAINLOOP_CLOSE : 0;
 379         }
 380
 381         DEBUG("Container init process %d exited", hdlr->pid);
 382         return LXC_MAINLOOP_CLOSE;
 383 }
 384
 385 int lxc_serve_state_clients(const char *name, struct lxc_handler *handler,
 386                             lxc_state_t state)
 387 {
 388         ssize_t ret;
 389         struct lxc_list *cur, *next;
 390         struct lxc_state_client *client;
 391         struct lxc_msg msg = {.type = lxc_msg_state, .value = state};
 392
 393         if (state == THAWED)
 394                 handler->state = RUNNING;
 395         else
 396                 handler->state = state;
 397
 398         TRACE("Set container state to %s", lxc_state2str(state));
 399
 400         if (lxc_list_empty(&handler->conf->state_clients)) {
 401                 TRACE("No state clients registered");
 402                 return 0;
 403         }
 404
 405         strncpy(msg.name, name, sizeof(msg.name));
 406         msg.name[sizeof(msg.name) - 1] = 0;
 407
 408         lxc_list_for_each_safe(cur, &handler->conf->state_clients, next) {
 409                 client = cur->elem;
 410
 411                 if (client->states[state] == 0) {
 412                         TRACE("State %s not registered for state client %d",
 413                               lxc_state2str(state), client->clientfd);
 414                         continue;
 415                 }
 416
 417                 TRACE("Sending state %s to state client %d",
 418                       lxc_state2str(state), client->clientfd);
 419
 420         again:
 421                 ret = send(client->clientfd, &msg, sizeof(msg), 0);
 422                 if (ret <= 0) {
 423                         if (errno == EINTR) {
 424                                 TRACE("Caught EINTR; retrying");
 425                                 goto again;
 426                         }
 427
 428                         ERROR("%s - Failed to send message to client",
 429                               strerror(errno));
 430                 }
 431
 432                 /* kick client from list */
 433                 lxc_list_del(cur);
 434                 close(client->clientfd);
 435                 free(cur->elem);
 436                 free(cur);
 437         }
 438
 439         return 0;
 440 }
 441
 442 static int lxc_serve_state_socket_pair(const char *name,
 443                                        struct lxc_handler *handler,
 444                                        lxc_state_t state)
 445 {
 446         ssize_t ret;
 447
 448         if (!handler->backgrounded ||
 449             handler->state_socket_pair[1] < 0 ||
 450             state == STARTING)
 451                 return 0;
 452
 453         /* Close read end of the socket pair. */
 454         close(handler->state_socket_pair[0]);
 455         handler->state_socket_pair[0] = -1;
 456
 457 again:
 458         ret = lxc_abstract_unix_send_credential(handler->state_socket_pair[1],
 459                                                 &(int){state}, sizeof(int));
 460         if (ret != sizeof(int)) {
 461                 if (errno == EINTR)
 462                         goto again;
 463                 SYSERROR("Failed to send state to %d",
 464                          handler->state_socket_pair[1]);
 465                 return -1;
 466         }
 467
 468         TRACE("Sent container state \"%s\" to %d", lxc_state2str(state),
 469               handler->state_socket_pair[1]);
 470
 471         /* Close write end of the socket pair. */
 472         close(handler->state_socket_pair[1]);
 473         handler->state_socket_pair[1] = -1;
 474
 475         return 0;
 476 }
 477
 478 int lxc_set_state(const char *name, struct lxc_handler *handler,
 479                   lxc_state_t state)
 480 {
 481         int ret;
 482
 483         ret = lxc_serve_state_socket_pair(name, handler, state);
 484         if (ret < 0) {
 485                 ERROR("Failed to synchronize via anonymous pair of unix sockets");
 486                 return -1;
 487         }
 488
 489         ret = lxc_serve_state_clients(name, handler, state);
 490         if (ret < 0)
 491                 return -1;
 492
 493         /* This function will try to connect to the legacy lxc-monitord state
 494          * server and only exists for backwards compatibility.
 495          */
 496         lxc_monitor_send_state(name, state, handler->lxcpath);
 497
 498         return 0;
 499 }
 500
 501 int lxc_poll(const char *name, struct lxc_handler *handler)
 502 {
 503         int ret;
 504         bool has_console = true;
 505         struct lxc_epoll_descr descr, descr_console;
 506
 507         if (handler->conf->console.path &&
 508             strcmp(handler->conf->console.path, "none") == 0)
 509                 has_console = false;
 510
 511         ret = lxc_mainloop_open(&descr);
 512         if (ret < 0) {
 513                 ERROR("Failed to create mainloop");
 514                 goto out_sigfd;
 515         }
 516
 517         if (has_console) {
 518                 ret = lxc_mainloop_open(&descr_console);
 519                 if (ret < 0) {
 520                         ERROR("Failed to create console mainloop");
 521                         goto out_mainloop;
 522                 }
 523         }
 524
 525         ret = lxc_mainloop_add_handler(&descr, handler->sigfd, signal_handler, handler);
 526         if (ret < 0) {
 527                 ERROR("Failed to add signal handler for %d to mainloop", handler->sigfd);
 528                 goto out_mainloop_console;
 529         }
 530
 531         if (has_console) {
 532                 struct lxc_terminal *console = &handler->conf->console;
 533
 534                 ret = lxc_terminal_mainloop_add(&descr, console);
 535                 if (ret < 0) {
 536                         ERROR("Failed to add console handlers to mainloop");
 537                         goto out_mainloop_console;
 538                 }
 539
 540                 ret = lxc_terminal_mainloop_add(&descr_console, console);
 541                 if (ret < 0) {
 542                         ERROR("Failed to add console handlers to console mainloop");
 543                         goto out_mainloop_console;
 544                 }
 545
 546                 handler->conf->console.descr = &descr;
 547         }
 548
 549         ret = lxc_cmd_mainloop_add(name, &descr, handler);
 550         if (ret < 0) {
 551                 ERROR("Failed to add command handler to mainloop");
 552                 goto out_mainloop_console;
 553         }
 554
 555         TRACE("Mainloop is ready");
 556
 557         ret = lxc_mainloop(&descr, -1);
 558         close(descr.epfd);
 559         descr.epfd = -EBADF;
 560         if (ret < 0 || !handler->init_died)
 561                 goto out_mainloop_console;
 562
 563         if (has_console)
 564                 ret = lxc_mainloop(&descr_console, 0);
 565
 566 out_mainloop_console:
 567         if (has_console) {
 568                 lxc_mainloop_close(&descr_console);
 569                 TRACE("Closed console mainloop");
 570         }
 571
 572 out_mainloop:
 573         lxc_mainloop_close(&descr);
 574         TRACE("Closed mainloop");
 575
 576 out_sigfd:
 577         close(handler->sigfd);
 578         TRACE("Closed signal file descriptor %d", handler->sigfd);
 579         handler->sigfd = -EBADF;
 580
 581         return ret;
 582 }
 583
 584 void lxc_zero_handler(struct lxc_handler *handler)
 585 {
 586         int i;
 587
 588         memset(handler, 0, sizeof(struct lxc_handler));
 589
 590         handler->ns_clone_flags = -1;
 591
 592         handler->pinfd = -1;
 593
 594         handler->sigfd = -1;
 595
 596         for (i = 0; i < LXC_NS_MAX; i++)
 597                 handler->nsfd[i] = -1;
 598
 599         handler->data_sock[0] = -1;
 600         handler->data_sock[1] = -1;
 601
 602         handler->state_socket_pair[0] = -1;
 603         handler->state_socket_pair[1] = -1;
 604
 605         handler->sync_sock[0] = -1;
 606         handler->sync_sock[1] = -1;
 607 }
 608
 609 void lxc_free_handler(struct lxc_handler *handler)
 610 {
 611         if (handler->pinfd >= 0)
 612                 close(handler->pinfd);
 613
 614         if (handler->sigfd >= 0)
 615                 close(handler->sigfd);
 616
 617         lxc_put_nsfds(handler);
 618
 619         if (handler->conf && handler->conf->reboot == 0)
 620                 if (handler->conf->maincmd_fd >= 0)
 621                         close(handler->conf->maincmd_fd);
 622
 623         if (handler->state_socket_pair[0] >= 0)
 624                 close(handler->state_socket_pair[0]);
 625
 626         if (handler->state_socket_pair[1] >= 0)
 627                 close(handler->state_socket_pair[1]);
 628
 629         handler->conf = NULL;
 630         free(handler);
 631         handler = NULL;
 632 }
 633
 634 struct lxc_handler *lxc_init_handler(const char *name, struct lxc_conf *conf,
 635                                      const char *lxcpath, bool daemonize)
 636 {
 637         int i, ret;
 638         struct lxc_handler *handler;
 639
 640         handler = malloc(sizeof(*handler));
 641         if (!handler)
 642                 return NULL;
 643         memset(handler, 0, sizeof(*handler));
 644
 645         /* Note that am_guest_unpriv() checks the effective uid. We
 646          * probably don't care if we are real root only if we are running
 647          * as root so this should be fine.
 648          */
 649         handler->am_root = !am_guest_unpriv();
 650         handler->data_sock[0] = handler->data_sock[1] = -1;
 651         handler->conf = conf;
 652         handler->lxcpath = lxcpath;
 653         handler->pinfd = -1;
 654         handler->sigfd = -EBADF;
 655         handler->init_died = false;
 656         handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
 657         if (handler->conf->reboot == 0)
 658                 lxc_list_init(&handler->conf->state_clients);
 659
 660         for (i = 0; i < LXC_NS_MAX; i++)
 661                 handler->nsfd[i] = -1;
 662
 663         handler->name = name;
 664
 665         if (daemonize && !handler->conf->reboot) {
 666                 /* Create socketpair() to synchronize on daemonized startup.
 667                  * When the container reboots we don't need to synchronize
 668                  * again currently so don't open another socketpair().
 669                  */
 670                 ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0,
 671                                  handler->state_socket_pair);
 672                 if (ret < 0) {
 673                         ERROR("Failed to create anonymous pair of unix sockets");
 674                         goto on_error;
 675                 }
 676                 TRACE("Created anonymous pair {%d,%d} of unix sockets",
 677                       handler->state_socket_pair[0],
 678                       handler->state_socket_pair[1]);
 679         }
 680
 681         if (handler->conf->reboot == 0) {
 682                 handler->conf->maincmd_fd = lxc_cmd_init(name, lxcpath, "command");
 683                 if (handler->conf->maincmd_fd < 0) {
 684                         ERROR("Failed to set up command socket");
 685                         goto on_error;
 686                 }
 687         }
 688         TRACE("Unix domain socket %d for command server is ready",
 689               handler->conf->maincmd_fd);
 690
 691         return handler;
 692
 693 on_error:
 694         lxc_free_handler(handler);
 695
 696         return NULL;
 697 }
 698
 699 int lxc_init(const char *name, struct lxc_handler *handler)
 700 {
 701         int ret;
 702         const char *loglevel;
 703         struct lxc_conf *conf = handler->conf;
 704
 705         lsm_init();
 706         TRACE("Initialized LSM");
 707
 708         ret = lxc_read_seccomp_config(conf);
 709         if (ret < 0) {
 710                 ERROR("Failed loading seccomp policy");
 711                 goto out_close_maincmd_fd;
 712         }
 713         TRACE("Read seccomp policy");
 714
 715         /* Begin by setting the state to STARTING. */
 716         ret = lxc_set_state(name, handler, STARTING);
 717         if (ret < 0) {
 718                 ERROR("Failed to set state to \"%s\"", lxc_state2str(STARTING));
 719                 goto out_close_maincmd_fd;
 720         }
 721         TRACE("Set container state to \"STARTING\"");
 722
 723         /* Start of environment variable setup for hooks. */
 724         if (name) {
 725                 ret = setenv("LXC_NAME", name, 1);
 726                 if (ret < 0)
 727                         SYSERROR("Failed to set environment variable: LXC_NAME=%s", name);
 728         }
 729
 730         if (conf->rcfile) {
 731                 ret = setenv("LXC_CONFIG_FILE", conf->rcfile, 1);
 732                 if (ret < 0)
 733                         SYSERROR("Failed to set environment variable: "
 734                                  "LXC_CONFIG_FILE=%s", conf->rcfile);
 735         }
 736
 737         if (conf->rootfs.mount) {
 738                 ret = setenv("LXC_ROOTFS_MOUNT", conf->rootfs.mount, 1);
 739                 if (ret < 0)
 740                         SYSERROR("Failed to set environment variable: "
 741                                  "LXC_ROOTFS_MOUNT=%s", conf->rootfs.mount);
 742         }
 743
 744         if (conf->rootfs.path) {
 745                 ret = setenv("LXC_ROOTFS_PATH", conf->rootfs.path, 1);
 746                 if (ret < 0)
 747                         SYSERROR("Failed to set environment variable: "
 748                                  "LXC_ROOTFS_PATH=%s", conf->rootfs.path);
 749         }
 750
 751         if (conf->console.path) {
 752                 ret = setenv("LXC_CONSOLE", conf->console.path, 1);
 753                 if (ret < 0)
 754                         SYSERROR("Failed to set environment variable: "
 755                                  "LXC_CONSOLE=%s", conf->console.path);
 756         }
 757
 758         if (conf->console.log_path) {
 759                 ret = setenv("LXC_CONSOLE_LOGPATH", conf->console.log_path, 1);
 760                 if (ret < 0)
 761                         SYSERROR("Failed to set environment variable: "
 762                                  "LXC_CONSOLE_LOGPATH=%s", conf->console.log_path);
 763         }
 764
 765         if (cgns_supported()) {
 766                 ret = setenv("LXC_CGNS_AWARE", "1", 1);
 767                 if (ret < 0)
 768                         SYSERROR("Failed to set environment variable "
 769                                  "LXC_CGNS_AWARE=1");
 770         }
 771
 772         loglevel = lxc_log_priority_to_string(lxc_log_get_level());
 773         ret = setenv("LXC_LOG_LEVEL", loglevel, 1);
 774         if (ret < 0)
 775                 SYSERROR("Set environment variable LXC_LOG_LEVEL=%s",
 776                          loglevel);
 777
 778         if (conf->hooks_version == 0)
 779                 ret = setenv("LXC_HOOK_VERSION", "0", 1);
 780         else
 781                 ret = setenv("LXC_HOOK_VERSION", "1", 1);
 782         if (ret < 0)
 783                 SYSERROR("Failed to set environment variable LXC_HOOK_VERSION=%u", conf->hooks_version);
 784         /* End of environment variable setup for hooks. */
 785
 786         TRACE("Set environment variables");
 787
 788         ret = run_lxc_hooks(name, "pre-start", conf, NULL);
 789         if (ret < 0) {
 790                 ERROR("Failed to run lxc.hook.pre-start for container \"%s\"", name);
 791                 goto out_aborting;
 792         }
 793         TRACE("Ran pre-start hooks");
 794
 795         /* The signal fd has to be created before forking otherwise if the child
 796          * process exits before we setup the signal fd, the event will be lost
 797          * and the command will be stuck.
 798          */
 799         handler->sigfd = setup_signal_fd(&handler->oldmask);
 800         if (handler->sigfd < 0) {
 801                 ERROR("Failed to setup SIGCHLD fd handler.");
 802                 goto out_delete_tty;
 803         }
 804         TRACE("Set up signal fd");
 805
 806         /* Do this after setting up signals since it might unblock SIGWINCH. */
 807         ret = lxc_terminal_setup(conf);
 808         if (ret < 0) {
 809                 ERROR("Failed to create console");
 810                 goto out_restore_sigmask;
 811         }
 812         TRACE("Created console");
 813
 814         ret = lxc_terminal_map_ids(conf, &conf->console);
 815         if (ret < 0) {
 816                 ERROR("Failed to chown console");
 817                 goto out_restore_sigmask;
 818         }
 819         TRACE("Chowned console");
 820
 821         INFO("Container \"%s\" is initialized", name);
 822         return 0;
 823
 824 out_restore_sigmask:
 825         sigprocmask(SIG_SETMASK, &handler->oldmask, NULL);
 826 out_delete_tty:
 827         lxc_delete_tty(&conf->tty_info);
 828 out_aborting:
 829         lxc_set_state(name, handler, ABORTING);
 830 out_close_maincmd_fd:
 831         close(conf->maincmd_fd);
 832         conf->maincmd_fd = -1;
 833         return -1;
 834 }
 835
 836 void lxc_fini(const char *name, struct lxc_handler *handler)
 837 {
 838         int i, ret;
 839         pid_t self;
 840         struct lxc_list *cur, *next;
 841         char *namespaces[LXC_NS_MAX + 1];
 842         size_t namespace_count = 0;
 843
 844         /* The STOPPING state is there for future cleanup code which can take
 845          * awhile.
 846          */
 847         lxc_set_state(name, handler, STOPPING);
 848
 849         self = lxc_raw_getpid();
 850         for (i = 0; i < LXC_NS_MAX; i++) {
 851                 if (handler->nsfd[i] < 0)
 852                         continue;
 853
 854                 if (handler->conf->hooks_version == 0)
 855                         ret = asprintf(&namespaces[namespace_count],
 856                                       "%s:/proc/%d/fd/%d", ns_info[i].proc_name,
 857                                       self, handler->nsfd[i]);
 858                 else
 859                         ret = asprintf(&namespaces[namespace_count],
 860                                       "/proc/%d/fd/%d", self, handler->nsfd[i]);
 861                 if (ret == -1) {
 862                         SYSERROR("Failed to allocate memory");
 863                         break;
 864                 }
 865
 866                 if (handler->conf->hooks_version == 0) {
 867                         namespace_count++;
 868                         continue;
 869                 }
 870
 871                 ret = setenv(ns_info[i].env_name, namespaces[namespace_count], 1);
 872                 if (ret < 0)
 873                         SYSERROR("Failed to set environment variable %s=%s",
 874                                  ns_info[i].env_name, namespaces[namespace_count]);
 875                 else
 876                         TRACE("Set environment variable %s=%s",
 877                               ns_info[i].env_name, namespaces[namespace_count]);
 878
 879                 namespace_count++;
 880         }
 881         namespaces[namespace_count] = NULL;
 882
 883         if (handler->conf->reboot) {
 884                 ret = setenv("LXC_TARGET", "reboot", 1);
 885                 if (ret < 0)
 886                         SYSERROR("Failed to set environment variable: "
 887                                  "LXC_TARGET=reboot");
 888         }
 889
 890         if (!handler->conf->reboot) {
 891                 ret = setenv("LXC_TARGET", "stop", 1);
 892                 if (ret < 0)
 893                         SYSERROR("Failed to set environment variable: "
 894                                  "LXC_TARGET=stop");
 895         }
 896
 897         if (handler->conf->hooks_version == 0)
 898                 ret = run_lxc_hooks(name, "stop", handler->conf, namespaces);
 899         else
 900                 ret = run_lxc_hooks(name, "stop", handler->conf, NULL);
 901         if (ret < 0)
 902                 ERROR("Failed to run \"lxc.hook.stop\" hook");
 903
 904         while (namespace_count--)
 905                 free(namespaces[namespace_count]);
 906
 907         cgroup_destroy(handler);
 908
 909         if (handler->conf->reboot == 0) {
 910                 /* For all new state clients simply close the command socket.
 911                  * This will inform all state clients that the container is
 912                  * STOPPED and also prevents a race between a open()/close() on
 913                  * the command socket causing a new process to get ECONNREFUSED
 914                  * because we haven't yet closed the command socket.
 915                  */
 916                 close(handler->conf->maincmd_fd);
 917                 handler->conf->maincmd_fd = -1;
 918                 TRACE("Closed command socket");
 919
 920                 /* This function will try to connect to the legacy lxc-monitord
 921                  * state server and only exists for backwards compatibility.
 922                  */
 923                 lxc_monitor_send_state(name, STOPPED, handler->lxcpath);
 924
 925                 /* The command socket is closed so no one can acces the command
 926                  * socket anymore so there's no need to lock it.
 927                  */
 928                 handler->state = STOPPED;
 929                 TRACE("Set container state to \"STOPPED\"");
 930         } else {
 931                 lxc_set_state(name, handler, STOPPED);
 932         }
 933
 934         ret = run_lxc_hooks(name, "post-stop", handler->conf, NULL);
 935         if (ret < 0) {
 936                 ERROR("Failed to run lxc.hook.post-stop for container \"%s\"", name);
 937                 if (handler->conf->reboot) {
 938                         WARN("Container will be stopped instead of rebooted");
 939                         handler->conf->reboot = 0;
 940
 941                         ret = setenv("LXC_TARGET", "stop", 1);
 942                         if (ret < 0)
 943                                 WARN("Failed to set environment variable: "
 944                                      "LXC_TARGET=stop");
 945                 }
 946         }
 947
 948         /* Reset mask set by setup_signal_fd. */
 949         ret = sigprocmask(SIG_SETMASK, &handler->oldmask, NULL);
 950         if (ret < 0)
 951                 WARN("%s - Failed to restore signal mask", strerror(errno));
 952
 953         lxc_terminal_delete(&handler->conf->console);
 954         lxc_delete_tty(&handler->conf->tty_info);
 955
 956         /* The command socket is now closed, no more state clients can register
 957          * themselves from now on. So free the list of state clients.
 958          */
 959         lxc_list_for_each_safe(cur, &handler->conf->state_clients, next) {
 960                 struct lxc_state_client *client = cur->elem;
 961
 962                 /* Keep state clients that want to be notified about reboots. */
 963                 if ((handler->conf->reboot > 0) && (client->states[RUNNING] == 2))
 964                         continue;
 965
 966                 /* close state client socket */
 967                 lxc_list_del(cur);
 968                 close(client->clientfd);
 969                 free(cur->elem);
 970                 free(cur);
 971         }
 972
 973         if (handler->conf->ephemeral == 1 && handler->conf->reboot != 1)
 974                 lxc_destroy_container_on_signal(handler, name);
 975
 976         lxc_free_handler(handler);
 977 }
 978
 979 void lxc_abort(const char *name, struct lxc_handler *handler)
 980 {
 981         int ret, status;
 982
 983         lxc_set_state(name, handler, ABORTING);
 984
 985         if (handler->pid > 0) {
 986                 ret = kill(handler->pid, SIGKILL);
 987                 if (ret < 0)
 988                         SYSERROR("Failed to send SIGKILL to %d", handler->pid);
 989         }
 990
 991         while ((ret = waitpid(-1, &status, 0)) > 0) {
 992                 ;
 993         }
 994 }
 995
 996 static int do_start(void *data)
 997 {
 998         int ret;
 999         char path[PATH_MAX];
1000         bool have_cap_setgid;
1001         uid_t new_uid;
1002         gid_t new_gid;
1003         struct lxc_list *iterator;
1004         int devnull_fd = -1;
1005         struct lxc_handler *handler = data;
1006
1007         lxc_sync_fini_parent(handler);
1008
1009         /* This prctl must be before the synchro, so if the parent dies before
1010          * we set the parent death signal, we will detect its death with the
1011          * synchro right after, otherwise we have a window where the parent can
1012          * exit before we set the pdeath signal leading to a unsupervized
1013          * container.
1014          */
1015         ret = lxc_set_death_signal(SIGKILL);
1016         if (ret < 0) {
1017                 SYSERROR("Failed to set PR_SET_PDEATHSIG to SIGKILL");
1018                 goto out_warn_father;
1019         }
1020
1021         ret = sigprocmask(SIG_SETMASK, &handler->oldmask, NULL);
1022         if (ret < 0) {
1023                 SYSERROR("Failed to set signal mask");
1024                 goto out_warn_father;
1025         }
1026
1027         /* Don't leak the pinfd to the container. */
1028         if (handler->pinfd >= 0)
1029                 close(handler->pinfd);
1030
1031         ret = lxc_sync_wait_parent(handler, LXC_SYNC_STARTUP);
1032         if (ret < 0)
1033                 goto out_warn_father;
1034
1035         /* Unshare CLONE_NEWNET after CLONE_NEWUSER. See
1036          * https://github.com/lxc/lxd/issues/1978.
1037          */
1038         if ((handler->ns_clone_flags & (CLONE_NEWNET | CLONE_NEWUSER)) ==
1039             (CLONE_NEWNET | CLONE_NEWUSER)) {
1040                 ret = unshare(CLONE_NEWNET);
1041                 if (ret < 0) {
1042                         SYSERROR("Failed to unshare CLONE_NEWNET");
1043                         goto out_warn_father;
1044                 }
1045                 INFO("Unshared CLONE_NEWNET");
1046         }
1047
1048         /* Tell the parent task it can begin to configure the container and wait
1049          * for it to finish.
1050          */
1051         ret = lxc_sync_barrier_parent(handler, LXC_SYNC_CONFIGURE);
1052         if (ret < 0)
1053                 return -1;
1054
1055         ret = lxc_network_recv_veth_names_from_parent(handler);
1056         if (ret < 0) {
1057                 ERROR("Failed to receive veth names from parent");
1058                 goto out_warn_father;
1059         }
1060
1061         /* If we are in a new user namespace, become root there to have
1062          * privilege over our namespace.
1063          */
1064         if (!lxc_list_empty(&handler->conf->id_map)) {
1065                 uid_t nsuid = (handler->conf->root_nsuid_map != NULL)
1066                                   ? 0
1067                                   : handler->conf->init_uid;
1068                 gid_t nsgid = (handler->conf->root_nsgid_map != NULL)
1069                                   ? 0
1070                                   : handler->conf->init_gid;
1071
1072                 ret = lxc_switch_uid_gid(nsuid, nsgid);
1073                 if (ret < 0)
1074                         goto out_warn_father;
1075
1076                 /* Drop groups only after we switched to a valid gid in the new
1077                  * user namespace.
1078                  */
1079                 ret = lxc_setgroups(0, NULL);
1080                 if (ret < 0 && (handler->am_root || errno != EPERM))
1081                         goto out_warn_father;
1082
1083                 if (!handler->am_root) {
1084                         ret = prctl(PR_SET_DUMPABLE, 1, 0, 0, 0);
1085                         if (ret < 0)
1086                                 goto out_warn_father;
1087                 }
1088
1089                 /* set{g,u}id() clears deathsignal */
1090                 ret = lxc_set_death_signal(SIGKILL);
1091                 if (ret < 0) {
1092                         SYSERROR("Failed to set PR_SET_PDEATHSIG to SIGKILL");
1093                         goto out_warn_father;
1094                 }
1095         }
1096
1097         ret = access(handler->lxcpath, X_OK);
1098         if (ret != 0) {
1099                 print_top_failing_dir(handler->lxcpath);
1100                 goto out_warn_father;
1101         }
1102
1103         ret = snprintf(path, sizeof(path), "%s/dev/null",
1104                        handler->conf->rootfs.mount);
1105         if (ret < 0 || ret >= sizeof(path))
1106                 goto out_warn_father;
1107
1108         /* In order to checkpoint restore, we need to have everything in the
1109          * same mount namespace. However, some containers may not have a
1110          * reasonable /dev (in particular, they may not have /dev/null), so we
1111          * can't set init's std fds to /dev/null by opening it from inside the
1112          * container.
1113          *
1114          * If that's the case, fall back to using the host's /dev/null. This
1115          * means that migration won't work, but at least we won't spew output
1116          * where it isn't wanted.
1117          */
1118         if (handler->backgrounded && !handler->conf->autodev) {
1119                 ret = access(path, F_OK);
1120                 if (ret != 0) {
1121                         devnull_fd = open_devnull();
1122
1123                         if (devnull_fd < 0)
1124                                 goto out_warn_father;
1125                         WARN("Using /dev/null from the host for container "
1126                              "init's standard file descriptors. Migration will "
1127                              "not work");
1128                 }
1129         }
1130
1131         /* Ask father to setup cgroups and wait for him to finish. */
1132         ret = lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP);
1133         if (ret < 0)
1134                 goto out_error;
1135
1136         /* Unshare cgroup namespace after we have setup our cgroups. If we do it
1137          * earlier we end up with a wrong view of /proc/self/cgroup. For
1138          * example, assume we unshare(CLONE_NEWCGROUP) first, and then create
1139          * the cgroup for the container, say /sys/fs/cgroup/cpuset/lxc/c, then
1140          * /proc/self/cgroup would show us:
1141          *
1142          *      8:cpuset:/lxc/c
1143          *
1144          * whereas it should actually show
1145          *
1146          *      8:cpuset:/
1147          */
1148         if (handler->ns_clone_flags & CLONE_NEWCGROUP) {
1149                 ret = unshare(CLONE_NEWCGROUP);
1150                 if (ret < 0) {
1151                         INFO("Failed to unshare CLONE_NEWCGROUP");
1152                         goto out_warn_father;
1153                 }
1154                 INFO("Unshared CLONE_NEWCGROUP");
1155         }
1156
1157         /* Add the requested environment variables to the current environment to
1158          * allow them to be used by the various hooks, such as the start hook
1159          * above.
1160          */
1161         lxc_list_for_each(iterator, &handler->conf->environment) {
1162                 ret = putenv((char *)iterator->elem);
1163                 if (ret < 0) {
1164                         SYSERROR("Failed to set environment variable: %s",
1165                                  (char *)iterator->elem);
1166                         goto out_warn_father;
1167                 }
1168         }
1169
1170         /* Setup the container, ip, names, utsname, ... */
1171         ret = lxc_setup(handler);
1172         close(handler->data_sock[1]);
1173         close(handler->data_sock[0]);
1174         if (ret < 0) {
1175                 ERROR("Failed to setup container \"%s\"", handler->name);
1176                 goto out_warn_father;
1177         }
1178
1179         /* Set the label to change to when we exec(2) the container's init. */
1180         ret = lsm_process_label_set(NULL, handler->conf, 1, 1);
1181         if (ret < 0)
1182                 goto out_warn_father;
1183
1184         /* Set PR_SET_NO_NEW_PRIVS after we changed the lsm label. If we do it
1185          * before we aren't allowed anymore.
1186          */
1187         if (handler->conf->no_new_privs) {
1188                 ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0);
1189                 if (ret < 0) {
1190                         SYSERROR("Could not set PR_SET_NO_NEW_PRIVS to block "
1191                                  "execve() gainable privileges");
1192                         goto out_warn_father;
1193                 }
1194                 DEBUG("Set PR_SET_NO_NEW_PRIVS to block execve() gainable "
1195                       "privileges");
1196         }
1197
1198         /* Some init's such as busybox will set sane tty settings on stdin,
1199          * stdout, stderr which it thinks is the console. We already set them
1200          * the way we wanted on the real terminal, and we want init to do its
1201          * setup on its console ie. the pty allocated in lxc_terminal_setup() so
1202          * make sure that that pty is stdin,stdout,stderr.
1203          */
1204          if (handler->conf->console.slave >= 0) {
1205                  if (handler->backgrounded || handler->conf->is_execute == 0)
1206                          ret = set_stdfds(handler->conf->console.slave);
1207                  else
1208                          ret = lxc_terminal_set_stdfds(handler->conf->console.slave);
1209                  if (ret < 0) {
1210                         ERROR("Failed to redirect std{in,out,err} to pty file "
1211                               "descriptor %d", handler->conf->console.slave);
1212                         goto out_warn_father;
1213                  }
1214          }
1215
1216         /* If we mounted a temporary proc, then unmount it now. */
1217         tmp_proc_unmount(handler->conf);
1218
1219         ret = lxc_seccomp_load(handler->conf);
1220         if (ret < 0)
1221                 goto out_warn_father;
1222
1223         ret = run_lxc_hooks(handler->name, "start", handler->conf, NULL);
1224         if (ret < 0) {
1225                 ERROR("Failed to run lxc.hook.start for container \"%s\"",
1226                       handler->name);
1227                 goto out_warn_father;
1228         }
1229
1230         close(handler->sigfd);
1231
1232         if (devnull_fd < 0) {
1233                 devnull_fd = open_devnull();
1234                 if (devnull_fd < 0)
1235                         goto out_warn_father;
1236         }
1237
1238         if (handler->conf->console.slave < 0 && handler->backgrounded) {
1239                 ret = set_stdfds(devnull_fd);
1240                 if (ret < 0) {
1241                         ERROR("Failed to redirect std{in,out,err} to \"/dev/null\"");
1242                         goto out_warn_father;
1243                 }
1244         }
1245
1246         if (devnull_fd >= 0) {
1247                 close(devnull_fd);
1248                 devnull_fd = -1;
1249         }
1250
1251         setsid();
1252
1253         if (handler->conf->init_cwd) {
1254                 ret = chdir(handler->conf->init_cwd);
1255                 if (ret < 0) {
1256                         SYSERROR("Could not change directory to \"%s\"",
1257                                  handler->conf->init_cwd);
1258                         goto out_warn_father;
1259                 }
1260         }
1261
1262         ret = lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP_LIMITS);
1263         if (ret < 0)
1264                 goto out_warn_father;
1265
1266         /* Reset the environment variables the user requested in a clear
1267          * environment.
1268          */
1269         ret = clearenv();
1270         /* Don't error out though. */
1271         if (ret < 0)
1272                 SYSERROR("Failed to clear environment.");
1273
1274         lxc_list_for_each(iterator, &handler->conf->environment) {
1275                 ret = putenv((char *)iterator->elem);
1276                 if (ret < 0) {
1277                         SYSERROR("Failed to set environment variable: %s",
1278                                  (char *)iterator->elem);
1279                         goto out_warn_father;
1280                 }
1281         }
1282
1283         ret = putenv("container=lxc");
1284         if (ret < 0) {
1285                 SYSERROR("Failed to set environment variable: container=lxc");
1286                 goto out_warn_father;
1287         }
1288
1289         if (handler->conf->pty_names) {
1290                 ret = putenv(handler->conf->pty_names);
1291                 if (ret < 0) {
1292                         SYSERROR("Failed to set environment variable for container ptys");
1293                         goto out_warn_father;
1294                 }
1295         }
1296
1297         /* The container has been setup. We can now switch to an unprivileged
1298          * uid/gid.
1299          */
1300         new_uid = handler->conf->init_uid;
1301         new_gid = handler->conf->init_gid;
1302
1303         /* If we are in a new user namespace we already dropped all groups when
1304         *  we switched to root in the new user namespace further above. Only
1305         *  drop groups if we can, so ensure that we have necessary privilege.
1306          */
1307         #if HAVE_LIBCAP
1308         have_cap_setgid = lxc_proc_cap_is_set(CAP_SETGID, CAP_EFFECTIVE);
1309         #else
1310         have_cap_setgid = false;
1311         #endif
1312         if (lxc_list_empty(&handler->conf->id_map) && have_cap_setgid) {
1313                 ret = lxc_setgroups(0, NULL);
1314                 if (ret < 0)
1315                         goto out_warn_father;
1316         }
1317
1318         ret = lxc_switch_uid_gid(new_uid, new_gid);
1319         if (ret < 0)
1320                 goto out_warn_father;
1321
1322         /* After this call, we are in error because this ops should not return
1323          * as it execs.
1324          */
1325         handler->ops->start(handler, handler->data);
1326
1327 out_warn_father:
1328         /* We want the parent to know something went wrong, so we return a
1329          * special error code.
1330          */
1331         lxc_sync_wake_parent(handler, LXC_SYNC_ERROR);
1332
1333 out_error:
1334         if (devnull_fd >= 0)
1335                 close(devnull_fd);
1336
1337         return -1;
1338 }
1339
1340 static int lxc_recv_ttys_from_child(struct lxc_handler *handler)
1341 {
1342         int i;
1343         struct lxc_pty_info *pty_info;
1344         int ret = -1;
1345         int sock = handler->data_sock[1];
1346         struct lxc_conf *conf = handler->conf;
1347         struct lxc_tty_info *tty_info = &conf->tty_info;
1348
1349         if (!conf->tty)
1350                 return 0;
1351
1352         tty_info->pty_info = malloc(sizeof(*tty_info->pty_info) * conf->tty);
1353         if (!tty_info->pty_info)
1354                 return -1;
1355
1356         for (i = 0; i < conf->tty; i++) {
1357                 int ttyfds[2];
1358
1359                 ret = lxc_abstract_unix_recv_fds(sock, ttyfds, 2, NULL, 0);
1360                 if (ret < 0)
1361                         break;
1362
1363                 pty_info = &tty_info->pty_info[i];
1364                 pty_info->busy = 0;
1365                 pty_info->master = ttyfds[0];
1366                 pty_info->slave = ttyfds[1];
1367                 TRACE("Received pty with master fd %d and slave fd %d from "
1368                       "parent", pty_info->master, pty_info->slave);
1369         }
1370         if (ret < 0)
1371                 ERROR("Failed to receive %d ttys from child: %s", conf->tty,
1372                       strerror(errno));
1373         else
1374                 TRACE("Received %d ttys from child", conf->tty);
1375
1376         tty_info->nbtty = conf->tty;
1377
1378         return ret;
1379 }
1380
1381 int resolve_clone_flags(struct lxc_handler *handler)
1382 {
1383         int i;
1384         struct lxc_conf *conf = handler->conf;
1385
1386         for (i = 0; i < LXC_NS_MAX; i++) {
1387                 if (conf->ns_keep != 0) {
1388                         if ((conf->ns_keep & ns_info[i].clone_flag) == 0)
1389                                 handler->ns_clone_flags |= ns_info[i].clone_flag;
1390                 } else if (conf->ns_clone != 0) {
1391                         if ((conf->ns_clone & ns_info[i].clone_flag) > 0)
1392                                 handler->ns_clone_flags |= ns_info[i].clone_flag;
1393                 } else {
1394                         if (i == LXC_NS_USER && lxc_list_empty(&handler->conf->id_map))
1395                                 continue;
1396
1397                         if (i == LXC_NS_NET && lxc_requests_empty_network(handler))
1398                                 continue;
1399
1400                         if (i == LXC_NS_CGROUP && !cgns_supported())
1401                                 continue;
1402
1403                         handler->ns_clone_flags |= ns_info[i].clone_flag;
1404                 }
1405
1406                 if (!conf->ns_share[i])
1407                         continue;
1408
1409                 handler->ns_clone_flags &= ~ns_info[i].clone_flag;
1410                 TRACE("Sharing %s namespace", ns_info[i].proc_name);
1411         }
1412
1413         return 0;
1414 }
1415
1416 /* Note that this function is used with clone(CLONE_VM). Some glibc versions
1417  * used to reset the pid/tid to -1 when CLONE_VM was used without CLONE_THREAD.
1418  * But since the memory between parent and child is shared on CLONE_VM this
1419  * would invalidate the getpid() cache that glibc used to maintain and so
1420  * getpid() in the child would return the parent's pid. This is all fixed in
1421  * newer glibc versions where the getpid() cache is removed and the pid/tid is
1422  * not reset anymore.
1423  * However, if for whatever reason you - dear commiter - somehow need to get the
1424  * pid of the dummy intermediate process for do_share_ns() you need to call
1425  * lxc_raw_getpid(). The next lxc_raw_clone() call does not employ CLONE_VM and
1426  * will be fine.
1427  */
1428 static inline int do_share_ns(void *arg)
1429 {
1430         int i, flags, ret;
1431         struct lxc_handler *handler = arg;
1432
1433         for (i = 0; i < LXC_NS_MAX; i++) {
1434                 if (handler->nsfd[i] < 0)
1435                         continue;
1436
1437                 ret = setns(handler->nsfd[i], 0);
1438                 if (ret < 0)
1439                         return -1;
1440
1441                 DEBUG("Inherited %s namespace", ns_info[i].proc_name);
1442         }
1443
1444         flags = handler->ns_on_clone_flags;
1445         flags |= CLONE_PARENT;
1446         handler->pid = lxc_raw_clone_cb(do_start, handler, flags);
1447         if (handler->pid < 0)
1448                 return -1;
1449
1450         return 0;
1451 }
1452
1453 /* lxc_spawn() performs crucial setup tasks and clone()s the new process which
1454  * exec()s the requested container binary.
1455  * Note that lxc_spawn() runs in the parent namespaces. Any operations performed
1456  * right here should be double checked if they'd pose a security risk. (For
1457  * example, any {u}mount() operations performed here will be reflected on the
1458  * host!)
1459  */
1460 static int lxc_spawn(struct lxc_handler *handler)
1461 {
1462         int i, ret;
1463         char pidstr[20];
1464         bool wants_to_map_ids;
1465         struct lxc_list *id_map;
1466         const char *name = handler->name;
1467         const char *lxcpath = handler->lxcpath;
1468         bool cgroups_connected = false, share_ns = false;
1469         struct lxc_conf *conf = handler->conf;
1470
1471         id_map = &conf->id_map;
1472         wants_to_map_ids = !lxc_list_empty(id_map);
1473
1474         for (i = 0; i < LXC_NS_MAX; i++) {
1475                 if (!conf->ns_share[i])
1476                         continue;
1477
1478                 handler->nsfd[i] = lxc_inherit_namespace(conf->ns_share[i], lxcpath, ns_info[i].proc_name);
1479                 if (handler->nsfd[i] < 0)
1480                         return -1;
1481
1482                 share_ns = true;
1483         }
1484
1485         ret = lxc_sync_init(handler);
1486         if (ret < 0)
1487                 return -1;
1488
1489         ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0,
1490                          handler->data_sock);
1491         if (ret < 0) {
1492                 lxc_sync_fini(handler);
1493                 return -1;
1494         }
1495
1496         ret = resolve_clone_flags(handler);
1497         if (ret < 0) {
1498                 lxc_sync_fini(handler);
1499                 return -1;
1500         }
1501
1502         if (handler->ns_clone_flags & CLONE_NEWNET) {
1503                 if (!lxc_list_empty(&conf->network)) {
1504
1505                         /* Find gateway addresses from the link device, which is
1506                          * no longer accessible inside the container. Do this
1507                          * before creating network interfaces, since goto
1508                          * out_delete_net does not work before lxc_clone.
1509                          */
1510                         ret = lxc_find_gateway_addresses(handler);
1511                         if (ret < 0) {
1512                                 ERROR("Failed to find gateway addresses");
1513                                 lxc_sync_fini(handler);
1514                                 return -1;
1515                         }
1516
1517                         /* That should be done before the clone because we will
1518                          * fill the netdev index and use them in the child.
1519                          */
1520                         ret = lxc_create_network_priv(handler);
1521                         if (ret < 0) {
1522                                 ERROR("Failed to create the network");
1523                                 lxc_sync_fini(handler);
1524                                 return -1;
1525                         }
1526                 }
1527         }
1528
1529         if (!cgroup_init(handler)) {
1530                 ERROR("Failed initializing cgroup support");
1531                 goto out_delete_net;
1532         }
1533
1534         cgroups_connected = true;
1535
1536         if (!cgroup_create(handler)) {
1537                 ERROR("Failed creating cgroups");
1538                 goto out_delete_net;
1539         }
1540
1541         /* If the rootfs is not a blockdev, prevent the container from marking
1542          * it readonly.
1543          * If the container is unprivileged then skip rootfs pinning.
1544          */
1545         if (!wants_to_map_ids) {
1546                 handler->pinfd = pin_rootfs(conf->rootfs.path);
1547                 if (handler->pinfd == -1)
1548                         INFO("Failed to pin the rootfs for container \"%s\"", handler->name);
1549         }
1550
1551         /* Create a process in a new set of namespaces. */
1552         handler->ns_on_clone_flags = handler->ns_clone_flags;
1553         if (handler->ns_clone_flags & CLONE_NEWUSER) {
1554                 /* If CLONE_NEWUSER and CLONE_NEWNET was requested, we need to
1555                  * clone a new user namespace first and only later unshare our
1556                  * network namespace to ensure that network devices ownership is
1557                  * set up correctly.
1558                  */
1559                 handler->ns_on_clone_flags &= ~CLONE_NEWNET;
1560         }
1561         /* The cgroup namespace gets unshare()ed not clone()ed. */
1562         handler->ns_on_clone_flags &= ~CLONE_NEWCGROUP;
1563
1564         if (share_ns) {
1565                 pid_t attacher_pid;
1566
1567                 attacher_pid = lxc_clone(do_share_ns, handler,
1568                                          CLONE_VFORK | CLONE_VM | CLONE_FILES);
1569                 if (attacher_pid < 0) {
1570                         SYSERROR(LXC_CLONE_ERROR);
1571                         goto out_delete_net;
1572                 }
1573
1574                 ret = wait_for_pid(attacher_pid);
1575                 if (ret < 0) {
1576                         SYSERROR("Intermediate process failed");
1577                         goto out_delete_net;
1578                 }
1579         } else {
1580                 handler->pid = lxc_raw_clone_cb(do_start, handler,
1581                                                 handler->ns_on_clone_flags);
1582         }
1583         if (handler->pid < 0) {
1584                 SYSERROR(LXC_CLONE_ERROR);
1585                 goto out_delete_net;
1586         }
1587         TRACE("Cloned child process %d", handler->pid);
1588
1589         for (i = 0; i < LXC_NS_MAX; i++)
1590                 if (handler->ns_on_clone_flags & ns_info[i].clone_flag)
1591                         INFO("Cloned %s", ns_info[i].flag_name);
1592
1593         if (!lxc_preserve_namespaces(handler, handler->ns_on_clone_flags, handler->pid)) {
1594                 ERROR("Failed to preserve cloned namespaces for lxc.hook.stop");
1595                 goto out_delete_net;
1596         }
1597
1598         lxc_sync_fini_child(handler);
1599
1600         /* Map the container uids. The container became an invalid userid the
1601          * moment it was cloned with CLONE_NEWUSER. This call doesn't change
1602          * anything immediately, but allows the container to setuid(0) (0 being
1603          * mapped to something else on the host.) later to become a valid uid
1604          * again.
1605          */
1606         if (wants_to_map_ids) {
1607                 if (!handler->conf->ns_share[LXC_NS_USER] &&
1608                     (handler->conf->ns_keep & CLONE_NEWUSER) == 0) {
1609                         ret = lxc_map_ids(id_map, handler->pid);
1610                         if (ret < 0) {
1611                                 ERROR("Failed to set up id mapping.");
1612                                 goto out_delete_net;
1613                         }
1614                 }
1615         }
1616
1617         ret = lxc_sync_wake_child(handler, LXC_SYNC_STARTUP);
1618         if (ret < 0)
1619                 goto out_delete_net;
1620
1621         ret = lxc_sync_wait_child(handler, LXC_SYNC_CONFIGURE);
1622         if (ret < 0)
1623                 goto out_delete_net;
1624
1625         if (!cgroup_setup_limits(handler, false)) {
1626                 ERROR("Failed to setup cgroup limits for container \"%s\"", name);
1627                 goto out_delete_net;
1628         }
1629
1630         if (!cgroup_enter(handler))
1631                 goto out_delete_net;
1632
1633         if (!cgroup_chown(handler))
1634                 goto out_delete_net;
1635
1636         /* Now we're ready to preserve the network namespace */
1637         ret = lxc_preserve_ns(handler->pid, "net");
1638         if (ret < 0) {
1639                 ERROR("%s - Failed to preserve net namespace", strerror(errno));
1640                 goto out_delete_net;
1641         }
1642         handler->nsfd[LXC_NS_NET] = ret;
1643         DEBUG("Preserved net namespace via fd %d", ret);
1644
1645         /* Create the network configuration. */
1646         if (handler->ns_clone_flags & CLONE_NEWNET) {
1647                 ret = lxc_network_move_created_netdev_priv(handler->lxcpath,
1648                                                            handler->name,
1649                                                            &conf->network,
1650                                                            handler->pid);
1651                 if (ret < 0) {
1652                         ERROR("Failed to create the configured network");
1653                         goto out_delete_net;
1654                 }
1655
1656                 ret = lxc_create_network_unpriv(handler->lxcpath, handler->name,
1657                                                 &conf->network, handler->pid);
1658                 if (ret < 0) {
1659                         ERROR("Failed to create the configured network");
1660                         goto out_delete_net;
1661                 }
1662         }
1663
1664         ret = lxc_network_send_veth_names_to_child(handler);
1665         if (ret < 0) {
1666                 ERROR("Failed to send veth names to child");
1667                 goto out_delete_net;
1668         }
1669
1670         if (!lxc_list_empty(&conf->procs)) {
1671                 ret = setup_proc_filesystem(&conf->procs, handler->pid);
1672                 if (ret < 0)
1673                         goto out_delete_net;
1674         }
1675
1676         /* Tell the child to continue its initialization. We'll get
1677          * LXC_SYNC_CGROUP when it is ready for us to setup cgroups.
1678          */
1679         ret = lxc_sync_barrier_child(handler, LXC_SYNC_POST_CONFIGURE);
1680         if (ret < 0)
1681                 goto out_delete_net;
1682
1683         if (!lxc_list_empty(&conf->limits)) {
1684                 ret = setup_resource_limits(&conf->limits, handler->pid);
1685                 if (ret < 0) {
1686                         ERROR("Failed to setup resource limits");
1687                         goto out_delete_net;
1688                 }
1689         }
1690
1691         ret = lxc_sync_barrier_child(handler, LXC_SYNC_CGROUP_UNSHARE);
1692         if (ret < 0)
1693                 goto out_delete_net;
1694
1695         if (!cgroup_setup_limits(handler, true)) {
1696                 ERROR("Failed to setup legacy device cgroup controller limits");
1697                 goto out_delete_net;
1698         }
1699         TRACE("Set up legacy device cgroup controller limits");
1700
1701         cgroup_disconnect();
1702         cgroups_connected = false;
1703
1704         if (handler->ns_clone_flags & CLONE_NEWCGROUP) {
1705                 /* Now we're ready to preserve the cgroup namespace */
1706                 ret = lxc_preserve_ns(handler->pid, "cgroup");
1707                 if (ret < 0) {
1708                         ERROR("%s - Failed to preserve cgroup namespace", strerror(errno));
1709                         goto out_delete_net;
1710                 }
1711                 handler->nsfd[LXC_NS_CGROUP] = ret;
1712                 DEBUG("Preserved cgroup namespace via fd %d", ret);
1713         }
1714
1715         ret = snprintf(pidstr, 20, "%d", handler->pid);
1716         if (ret < 0 || ret >= 20)
1717                 goto out_delete_net;
1718
1719         ret = setenv("LXC_PID", pidstr, 1);
1720         if (ret < 0)
1721                 SYSERROR("Failed to set environment variable: LXC_PID=%s", pidstr);
1722
1723         /* Run any host-side start hooks */
1724         ret = run_lxc_hooks(name, "start-host", conf, NULL);
1725         if (ret < 0) {
1726                 ERROR("Failed to run lxc.hook.start-host");
1727                 goto out_delete_net;
1728         }
1729
1730         /* Tell the child to complete its initialization and wait for it to exec
1731          * or return an error. (The child will never return
1732          * LXC_SYNC_READY_START+1. It will either close the sync pipe, causing
1733          * lxc_sync_barrier_child to return success, or return a different
1734          * value, causing us to error out).
1735          */
1736         ret = lxc_sync_barrier_child(handler, LXC_SYNC_READY_START);
1737         if (ret < 0)
1738                 goto out_delete_net;
1739
1740         ret = lxc_network_recv_name_and_ifindex_from_child(handler);
1741         if (ret < 0) {
1742                 ERROR("Failed to receive names and ifindices for network "
1743                       "devices from child");
1744                 goto out_delete_net;
1745         }
1746
1747         /* Now all networks are created, network devices are moved into place,
1748          * and the correct names and ifindeces in the respective namespaces have
1749          * been recorded. The corresponding structs have now all been filled. So
1750          * log them for debugging purposes.
1751          */
1752         lxc_log_configured_netdevs(conf);
1753
1754         /* Read tty fds allocated by child. */
1755         ret = lxc_recv_ttys_from_child(handler);
1756         if (ret < 0) {
1757                 ERROR("Failed to receive tty info from child process");
1758                 goto out_delete_net;
1759         }
1760
1761         ret = handler->ops->post_start(handler, handler->data);
1762         if (ret < 0)
1763                 goto out_abort;
1764
1765         ret = lxc_set_state(name, handler, RUNNING);
1766         if (ret < 0) {
1767                 ERROR("Failed to set state to \"%s\"", lxc_state2str(RUNNING));
1768                 goto out_abort;
1769         }
1770
1771         lxc_sync_fini(handler);
1772
1773         return 0;
1774
1775 out_delete_net:
1776         if (cgroups_connected)
1777                 cgroup_disconnect();
1778
1779         if (handler->ns_clone_flags & CLONE_NEWNET)
1780                 lxc_delete_network(handler);
1781
1782 out_abort:
1783         lxc_abort(name, handler);
1784         lxc_sync_fini(handler);
1785         if (handler->pinfd >= 0) {
1786                 close(handler->pinfd);
1787                 handler->pinfd = -1;
1788         }
1789
1790         return -1;
1791 }
1792
1793 int __lxc_start(const char *name, struct lxc_handler *handler,
1794                 struct lxc_operations* ops, void *data, const char *lxcpath,
1795                 bool backgrounded)
1796 {
1797         int ret, status;
1798         struct lxc_conf *conf = handler->conf;
1799
1800         ret = lxc_init(name, handler);
1801         if (ret < 0) {
1802                 ERROR("Failed to initialize container \"%s\"", name);
1803                 return -1;
1804         }
1805         handler->ops = ops;
1806         handler->data = data;
1807         handler->backgrounded = backgrounded;
1808
1809         if (!attach_block_device(handler->conf)) {
1810                 ERROR("Failed to attach block device");
1811                 goto out_fini_nonet;
1812         }
1813
1814         if (geteuid() == 0 && !lxc_list_empty(&conf->id_map)) {
1815                 /* If the backing store is a device, mount it here and now. */
1816                 if (rootfs_is_blockdev(conf)) {
1817                         ret = unshare(CLONE_NEWNS);
1818                         if (ret < 0) {
1819                                 ERROR("Failed to unshare CLONE_NEWNS");
1820                                 goto out_fini_nonet;
1821                         }
1822                         INFO("Unshared CLONE_NEWNS");
1823
1824                         remount_all_slave();
1825                         ret = do_rootfs_setup(conf, name, lxcpath);
1826                         if (ret < 0) {
1827                                 ERROR("Error setting up rootfs mount as root before spawn");
1828                                 goto out_fini_nonet;
1829                         }
1830                         INFO("Set up container rootfs as host root");
1831                 }
1832         }
1833
1834         ret = lxc_spawn(handler);
1835         if (ret < 0) {
1836                 ERROR("Failed to spawn container \"%s\"", name);
1837                 goto out_detach_blockdev;
1838         }
1839         /* close parent side of data socket */
1840         close(handler->data_sock[0]);
1841         handler->data_sock[0] = -1;
1842         close(handler->data_sock[1]);
1843         handler->data_sock[1] = -1;
1844
1845         handler->conf->reboot = 0;
1846
1847         ret = lxc_poll(name, handler);
1848         if (ret) {
1849                 ERROR("LXC mainloop exited with error: %d", ret);
1850                 goto out_abort;
1851         }
1852
1853         status = lxc_wait_for_pid_status(handler->pid);
1854         if (status < 0)
1855                 SYSERROR("Failed to retrieve status for %d", handler->pid);
1856
1857         /* If the child process exited but was not signaled, it didn't call
1858          * reboot. This should mean it was an lxc-execute which simply exited.
1859          * In any case, treat it as a 'halt'.
1860          */
1861         if (WIFSIGNALED(status)) {
1862                 switch(WTERMSIG(status)) {
1863                 case SIGINT: /* halt */
1864                         DEBUG("Container \"%s\" is halting", name);
1865                         break;
1866                 case SIGHUP: /* reboot */
1867                         DEBUG("Container \"%s\" is rebooting", name);
1868                         handler->conf->reboot = 1;
1869                         break;
1870                 case SIGSYS: /* seccomp */
1871                         DEBUG("Container \"%s\" violated its seccomp policy", name);
1872                         break;
1873                 default:
1874                         DEBUG("Unknown exit status for container \"%s\" init %d", name, WTERMSIG(status));
1875                         break;
1876                 }
1877         }
1878
1879         ret = lxc_restore_phys_nics_to_netns(handler);
1880         if (ret < 0)
1881                 ERROR("Failed to move physical network devices back to parent "
1882                       "network namespace");
1883
1884         if (handler->pinfd >= 0) {
1885                 close(handler->pinfd);
1886                 handler->pinfd = -1;
1887         }
1888
1889         lxc_monitor_send_exit_code(name, status, handler->lxcpath);
1890         lxc_error_set_and_log(handler->pid, status);
1891
1892 out_fini:
1893         lxc_delete_network(handler);
1894
1895 out_detach_blockdev:
1896         detach_block_device(handler->conf);
1897
1898 out_fini_nonet:
1899         lxc_fini(name, handler);
1900         return ret;
1901
1902 out_abort:
1903         lxc_abort(name, handler);
1904         goto out_fini;
1905 }
1906
1907 struct start_args {
1908         char *const *argv;
1909 };
1910
1911 static int start(struct lxc_handler *handler, void* data)
1912 {
1913         struct start_args *arg = data;
1914
1915         NOTICE("Exec'ing \"%s\"", arg->argv[0]);
1916
1917         execvp(arg->argv[0], arg->argv);
1918         SYSERROR("Failed to exec \"%s\"", arg->argv[0]);
1919         return 0;
1920 }
1921
1922 static int post_start(struct lxc_handler *handler, void* data)
1923 {
1924         struct start_args *arg = data;
1925
1926         NOTICE("Started \"%s\" with pid \"%d\"", arg->argv[0], handler->pid);
1927         return 0;
1928 }
1929
1930 static struct lxc_operations start_ops = {
1931         .start = start,
1932         .post_start = post_start
1933 };
1934
1935 int lxc_start(const char *name, char *const argv[], struct lxc_handler *handler,
1936               const char *lxcpath, bool backgrounded)
1937 {
1938         struct start_args start_arg = {
1939                 .argv = argv,
1940         };
1941
1942         return __lxc_start(name, handler, &start_ops, &start_arg, lxcpath, backgrounded);
1943 }
1944
1945 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
1946                                             const char *name)
1947 {
1948         char destroy[MAXPATHLEN];
1949         struct lxc_container *c;
1950         int ret = 0;
1951         bool bret = true;
1952
1953         if (handler->conf->rootfs.path && handler->conf->rootfs.mount) {
1954                 bret = do_destroy_container(handler);
1955                 if (!bret) {
1956                         ERROR("Error destroying rootfs for container \"%s\"", name);
1957                         return;
1958                 }
1959         }
1960         INFO("Destroyed rootfs for container \"%s\"", name);
1961
1962         ret = snprintf(destroy, MAXPATHLEN, "%s/%s", handler->lxcpath, name);
1963         if (ret < 0 || ret >= MAXPATHLEN) {
1964                 ERROR("Error destroying directory for container \"%s\"", name);
1965                 return;
1966         }
1967
1968         c = lxc_container_new(name, handler->lxcpath);
1969         if (c) {
1970                 if (container_disk_lock(c)) {
1971                         INFO("Could not update lxc_snapshots file");
1972                         lxc_container_put(c);
1973                 } else {
1974                         mod_all_rdeps(c, false);
1975                         container_disk_unlock(c);
1976                         lxc_container_put(c);
1977                 }
1978         }
1979
1980         if (!handler->am_root)
1981                 ret = userns_exec_full(handler->conf, lxc_rmdir_onedev_wrapper,
1982                                        destroy, "lxc_rmdir_onedev_wrapper");
1983         else
1984                 ret = lxc_rmdir_onedev(destroy, NULL);
1985
1986         if (ret < 0) {
1987                 ERROR("Error destroying directory for container \"%s\"", name);
1988                 return;
1989         }
1990         INFO("Destroyed directory for container \"%s\"", name);
1991 }
1992
1993 static int lxc_rmdir_onedev_wrapper(void *data)
1994 {
1995         char *arg = (char *) data;
1996         return lxc_rmdir_onedev(arg, NULL);
1997 }
1998
1999 static bool do_destroy_container(struct lxc_handler *handler)
2000 {
2001         int ret;
2002
2003         if (!handler->am_root) {
2004                 ret = userns_exec_full(handler->conf, storage_destroy_wrapper,
2005                                        handler->conf, "storage_destroy_wrapper");
2006                 if (ret < 0)
2007                         return false;
2008
2009                 return true;
2010         }
2011
2012         return storage_destroy(handler->conf);
2013 }