ceph/src/global/global_init.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2011 New Dream Network
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include <filesystem>
  16 #include "common/async/context_pool.h"
  17 #include "common/ceph_argparse.h"
  18 #include "common/code_environment.h"
  19 #include "common/config.h"
  20 #include "common/debug.h"
  21 #include "common/errno.h"
  22 #include "common/signal.h"
  23 #include "common/version.h"
  24 #include "erasure-code/ErasureCodePlugin.h"
  25 #include "extblkdev/ExtBlkDevPlugin.h"
  26 #include "global/global_context.h"
  27 #include "global/global_init.h"
  28 #include "global/pidfile.h"
  29 #include "global/signal_handler.h"
  30 #include "include/compat.h"
  31 #include "include/str_list.h"
  32 #include "mon/MonClient.h"
  33
  34 #ifndef _WIN32
  35 #include <pwd.h>
  36 #include <grp.h>
  37 #endif
  38 #include <errno.h>
  39
  40 #ifdef HAVE_SYS_PRCTL_H
  41 #include <sys/prctl.h>
  42 #endif
  43
  44 #define dout_context g_ceph_context
  45 #define dout_subsys ceph_subsys_
  46
  47 namespace fs = std::filesystem;
  48
  49 using std::cerr;
  50 using std::string;
  51
  52 static void global_init_set_globals(CephContext *cct)
  53 {
  54   g_ceph_context = cct;
  55   get_process_name(g_process_name, sizeof(g_process_name));
  56 }
  57
  58 static void output_ceph_version()
  59 {
  60   char buf[1024];
  61   snprintf(buf, sizeof(buf), "%s, process %s, pid %d",
  62            pretty_version_to_str().c_str(),
  63            get_process_name_cpp().c_str(), getpid());
  64   generic_dout(0) << buf << dendl;
  65 }
  66
  67 static const char* c_str_or_null(const std::string &str)
  68 {
  69   if (str.empty())
  70     return NULL;
  71   return str.c_str();
  72 }
  73
  74 static int chown_path(const std::string &pathname, const uid_t owner, const gid_t group,
  75                       const std::string &uid_str, const std::string &gid_str)
  76 {
  77   #ifdef _WIN32
  78   return 0;
  79   #else
  80
  81   const char *pathname_cstr = c_str_or_null(pathname);
  82
  83   if (!pathname_cstr) {
  84     return 0;
  85   }
  86
  87   int r = ::chown(pathname_cstr, owner, group);
  88
  89   if (r < 0) {
  90     r = -errno;
  91     cerr << "warning: unable to chown() " << pathname << " as "
  92          << uid_str << ":" << gid_str << ": " << cpp_strerror(r) << std::endl;
  93   }
  94
  95   return r;
  96   #endif
  97 }
  98
  99 void global_pre_init(
 100   const std::map<std::string,std::string> *defaults,
 101   std::vector < const char* >& args,
 102   uint32_t module_type, code_environment_t code_env,
 103   int flags)
 104 {
 105   std::string conf_file_list;
 106   std::string cluster = "";
 107
 108   // ensure environment arguments are included in early processing
 109   env_to_vec(args);
 110
 111   CephInitParameters iparams = ceph_argparse_early_args(
 112     args, module_type,
 113     &cluster, &conf_file_list);
 114
 115   CephContext *cct = common_preinit(iparams, code_env, flags);
 116   cct->_conf->cluster = cluster;
 117   global_init_set_globals(cct);
 118   auto& conf = cct->_conf;
 119
 120   if (flags & (CINIT_FLAG_NO_DEFAULT_CONFIG_FILE|
 121                CINIT_FLAG_NO_MON_CONFIG)) {
 122     conf->no_mon_config = true;
 123   }
 124
 125   // alternate defaults
 126   if (defaults) {
 127     for (auto& i : *defaults) {
 128       conf.set_val_default(i.first, i.second);
 129     }
 130   }
 131
 132   if (conf.get_val<bool>("no_config_file")) {
 133     flags |= CINIT_FLAG_NO_DEFAULT_CONFIG_FILE;
 134   }
 135
 136   int ret = conf.parse_config_files(c_str_or_null(conf_file_list),
 137                                     &cerr, flags);
 138   if (ret == -EDOM) {
 139     cct->_log->flush();
 140     cerr << "global_init: error parsing config file." << std::endl;
 141     _exit(1);
 142   }
 143   else if (ret == -ENOENT) {
 144     if (!(flags & CINIT_FLAG_NO_DEFAULT_CONFIG_FILE)) {
 145       if (conf_file_list.length()) {
 146         cct->_log->flush();
 147         cerr << "global_init: unable to open config file from search list "
 148              << conf_file_list << std::endl;
 149         _exit(1);
 150       } else {
 151         cerr << "did not load config file, using default settings."
 152              << std::endl;
 153       }
 154     }
 155   }
 156   else if (ret) {
 157     cct->_log->flush();
 158     cerr << "global_init: error reading config file. "
 159          << conf.get_parse_error() << std::endl;
 160     _exit(1);
 161   }
 162
 163   // environment variables override (CEPH_ARGS, CEPH_KEYRING)
 164   conf.parse_env(cct->get_module_type());
 165
 166   // command line (as passed by caller)
 167   conf.parse_argv(args);
 168
 169   if (!cct->_log->is_started()) {
 170     cct->_log->start();
 171   }
 172
 173   // do the --show-config[-val], if present in argv
 174   conf.do_argv_commands();
 175
 176   // Now we're ready to complain about config file parse errors
 177   g_conf().complain_about_parse_error(g_ceph_context);
 178 }
 179
 180 boost::intrusive_ptr<CephContext>
 181 global_init(const std::map<std::string,std::string> *defaults,
 182             std::vector < const char* >& args,
 183             uint32_t module_type, code_environment_t code_env,
 184             int flags, bool run_pre_init)
 185 {
 186   // Ensure we're not calling the global init functions multiple times.
 187   static bool first_run = true;
 188   if (run_pre_init) {
 189     // We will run pre_init from here (default).
 190     ceph_assert(!g_ceph_context && first_run);
 191     global_pre_init(defaults, args, module_type, code_env, flags);
 192   } else {
 193     // Caller should have invoked pre_init manually.
 194     ceph_assert(g_ceph_context && first_run);
 195   }
 196   first_run = false;
 197
 198   // Verify flags have not changed if global_pre_init() has been called
 199   // manually. If they have, update them.
 200   if (g_ceph_context->get_init_flags() != flags) {
 201     g_ceph_context->set_init_flags(flags);
 202     if (flags & (CINIT_FLAG_NO_DEFAULT_CONFIG_FILE|
 203                  CINIT_FLAG_NO_MON_CONFIG)) {
 204       g_conf()->no_mon_config = true;
 205     }
 206   }
 207
 208   #ifndef _WIN32
 209   // signal stuff
 210   int siglist[] = { SIGPIPE, 0 };
 211   block_signals(siglist, NULL);
 212   #endif
 213
 214   if (g_conf()->fatal_signal_handlers) {
 215     install_standard_sighandlers();
 216   }
 217   ceph::register_assert_context(g_ceph_context);
 218
 219   if (g_conf()->log_flush_on_exit)
 220     g_ceph_context->_log->set_flush_on_exit();
 221
 222   // drop privileges?
 223   std::ostringstream priv_ss;
 224
 225   #ifndef _WIN32
 226   // consider --setuser root a no-op, even if we're not root
 227   if (getuid() != 0) {
 228     if (g_conf()->setuser.length()) {
 229       cerr << "ignoring --setuser " << g_conf()->setuser << " since I am not root"
 230            << std::endl;
 231     }
 232     if (g_conf()->setgroup.length()) {
 233       cerr << "ignoring --setgroup " << g_conf()->setgroup
 234            << " since I am not root" << std::endl;
 235     }
 236   } else if (g_conf()->setgroup.length() ||
 237              g_conf()->setuser.length()) {
 238     uid_t uid = 0;  // zero means no change; we can only drop privs here.
 239     gid_t gid = 0;
 240     std::string uid_string;
 241     std::string gid_string;
 242     std::string home_directory;
 243     if (g_conf()->setuser.length()) {
 244       char buf[4096];
 245       struct passwd pa;
 246       struct passwd *p = 0;
 247
 248       uid = atoi(g_conf()->setuser.c_str());
 249       if (uid) {
 250         getpwuid_r(uid, &pa, buf, sizeof(buf), &p);
 251       } else {
 252         getpwnam_r(g_conf()->setuser.c_str(), &pa, buf, sizeof(buf), &p);
 253         if (!p) {
 254           cerr << "unable to look up user '" << g_conf()->setuser << "'"
 255                << std::endl;
 256           exit(1);
 257         }
 258
 259         uid = p->pw_uid;
 260         gid = p->pw_gid;
 261         uid_string = g_conf()->setuser;
 262       }
 263
 264       if (p && p->pw_dir != nullptr) {
 265         home_directory = std::string(p->pw_dir);
 266       }
 267     }
 268     if (g_conf()->setgroup.length() > 0) {
 269       gid = atoi(g_conf()->setgroup.c_str());
 270       if (!gid) {
 271         char buf[4096];
 272         struct group gr;
 273         struct group *g = 0;
 274         getgrnam_r(g_conf()->setgroup.c_str(), &gr, buf, sizeof(buf), &g);
 275         if (!g) {
 276           cerr << "unable to look up group '" << g_conf()->setgroup << "'"
 277                << ": " << cpp_strerror(errno) << std::endl;
 278           exit(1);
 279         }
 280         gid = g->gr_gid;
 281         gid_string = g_conf()->setgroup;
 282       }
 283     }
 284     if ((uid || gid) &&
 285         g_conf()->setuser_match_path.length()) {
 286       // induce early expansion of setuser_match_path config option
 287       string match_path = g_conf()->setuser_match_path;
 288       g_conf().early_expand_meta(match_path, &cerr);
 289       struct stat st;
 290       int r = ::stat(match_path.c_str(), &st);
 291       if (r < 0) {
 292         cerr << "unable to stat setuser_match_path "
 293              << g_conf()->setuser_match_path
 294              << ": " << cpp_strerror(errno) << std::endl;
 295         exit(1);
 296       }
 297       if ((uid && uid != st.st_uid) ||
 298           (gid && gid != st.st_gid)) {
 299         cerr << "WARNING: will not setuid/gid: " << match_path
 300              << " owned by " << st.st_uid << ":" << st.st_gid
 301              << " and not requested " << uid << ":" << gid
 302              << std::endl;
 303         uid = 0;
 304         gid = 0;
 305         uid_string.erase();
 306         gid_string.erase();
 307       } else {
 308         priv_ss << "setuser_match_path "
 309                 << match_path << " owned by "
 310                 << st.st_uid << ":" << st.st_gid << ". ";
 311       }
 312     }
 313     g_ceph_context->set_uid_gid(uid, gid);
 314     g_ceph_context->set_uid_gid_strings(uid_string, gid_string);
 315     if ((flags & CINIT_FLAG_DEFER_DROP_PRIVILEGES) == 0) {
 316       if (setgid(gid) != 0) {
 317         cerr << "unable to setgid " << gid << ": " << cpp_strerror(errno)
 318              << std::endl;
 319         exit(1);
 320       }
 321 #if defined(HAVE_SYS_PRCTL_H)
 322       if (g_conf().get_val<bool>("set_keepcaps")) {
 323         if (prctl(PR_SET_KEEPCAPS, 1) == -1) {
 324           cerr << "warning: unable to set keepcaps flag: " << cpp_strerror(errno) << std::endl;
 325         }
 326       }
 327 #endif
 328       if (setuid(uid) != 0) {
 329         cerr << "unable to setuid " << uid << ": " << cpp_strerror(errno)
 330              << std::endl;
 331         exit(1);
 332       }
 333       if (setenv("HOME", home_directory.c_str(), 1) != 0) {
 334         cerr << "warning: unable to set HOME to " << home_directory << ": "
 335              << cpp_strerror(errno) << std::endl;
 336       }
 337       priv_ss << "set uid:gid to " << uid << ":" << gid << " (" << uid_string << ":" << gid_string << ")";
 338     } else {
 339       priv_ss << "deferred set uid:gid to " << uid << ":" << gid << " (" << uid_string << ":" << gid_string << ")";
 340     }
 341   }
 342   #endif /* _WIN32 */
 343
 344 #if defined(HAVE_SYS_PRCTL_H)
 345   if (prctl(PR_SET_DUMPABLE, 1) == -1) {
 346     cerr << "warning: unable to set dumpable flag: " << cpp_strerror(errno) << std::endl;
 347   }
 348 #  if defined(PR_SET_THP_DISABLE)
 349   if (!g_conf().get_val<bool>("thp") && prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0) == -1) {
 350     cerr << "warning: unable to disable THP: " << cpp_strerror(errno) << std::endl;
 351   }
 352 #  endif
 353 #endif
 354
 355   //
 356   // Utterly important to run first network connection after setuid().
 357   // In case of rdma transport uverbs kernel module starts returning
 358   // -EACCESS on each operation if credentials has been changed, see
 359   // callers of ib_safe_file_access() for details.
 360   //
 361   // fork() syscall also matters, so daemonization won't work in case
 362   // of rdma.
 363   //
 364   if (!g_conf()->no_mon_config) {
 365     // make sure our mini-session gets legacy values
 366     g_conf().apply_changes(nullptr);
 367
 368     ceph::async::io_context_pool cp(1);
 369     MonClient mc_bootstrap(g_ceph_context, cp);
 370     if (mc_bootstrap.get_monmap_and_config() < 0) {
 371       cp.stop();
 372       g_ceph_context->_log->flush();
 373       cerr << "failed to fetch mon config (--no-mon-config to skip)"
 374            << std::endl;
 375       _exit(1);
 376     }
 377     cp.stop();
 378   }
 379
 380   // Expand metavariables. Invoke configuration observers. Open log file.
 381   g_conf().apply_changes(nullptr);
 382
 383   if (g_conf()->run_dir.length() &&
 384       code_env == CODE_ENVIRONMENT_DAEMON &&
 385       !(flags & CINIT_FLAG_NO_DAEMON_ACTIONS)) {
 386
 387     if (!fs::exists(g_conf()->run_dir.c_str())) {
 388       std::error_code ec;
 389       if (!fs::create_directory(g_conf()->run_dir, ec)) {
 390        cerr << "warning: unable to create " << g_conf()->run_dir
 391             << ec.message() << std::endl;
 392       }
 393       fs::permissions(
 394         g_conf()->run_dir.c_str(),
 395         fs::perms::owner_all |
 396         fs::perms::group_read | fs::perms::group_exec |
 397         fs::perms::others_read | fs::perms::others_exec);
 398     }
 399   }
 400
 401   // call all observers now.  this has the side-effect of configuring
 402   // and opening the log file immediately.
 403   g_conf().call_all_observers();
 404
 405   if (priv_ss.str().length()) {
 406     dout(0) << priv_ss.str() << dendl;
 407   }
 408
 409   if ((flags & CINIT_FLAG_DEFER_DROP_PRIVILEGES) &&
 410       (g_ceph_context->get_set_uid() || g_ceph_context->get_set_gid())) {
 411     // Fix ownership on log files and run directories if needed.
 412     // Admin socket files are chown()'d during the common init path _after_
 413     // the service thread has been started. This is sadly a bit of a hack :(
 414     chown_path(g_conf()->run_dir,
 415                g_ceph_context->get_set_uid(),
 416                g_ceph_context->get_set_gid(),
 417                g_ceph_context->get_set_uid_string(),
 418                g_ceph_context->get_set_gid_string());
 419     g_ceph_context->_log->chown_log_file(
 420       g_ceph_context->get_set_uid(),
 421       g_ceph_context->get_set_gid());
 422   }
 423
 424   // Now we're ready to complain about config file parse errors
 425   g_conf().complain_about_parse_error(g_ceph_context);
 426
 427   // test leak checking
 428   if (g_conf()->debug_deliberately_leak_memory) {
 429     derr << "deliberately leaking some memory" << dendl;
 430     char *s = new char[1234567];
 431     (void)s;
 432     // cppcheck-suppress memleak
 433   }
 434
 435   if (code_env == CODE_ENVIRONMENT_DAEMON && !(flags & CINIT_FLAG_NO_DAEMON_ACTIONS))
 436     output_ceph_version();
 437
 438   if (g_ceph_context->crush_location.init_on_startup()) {
 439     cerr << " failed to init_on_startup : " << cpp_strerror(errno) << std::endl;
 440     exit(1);
 441   }
 442
 443   return boost::intrusive_ptr<CephContext>{g_ceph_context, false};
 444 }
 445
 446 void global_print_banner(void)
 447 {
 448   output_ceph_version();
 449 }
 450
 451 int global_init_prefork(CephContext *cct)
 452 {
 453   if (g_code_env != CODE_ENVIRONMENT_DAEMON)
 454     return -1;
 455
 456   const auto& conf = cct->_conf;
 457   if (!conf->daemonize) {
 458
 459     if (pidfile_write(conf->pid_file) < 0)
 460       exit(1);
 461
 462     if ((cct->get_init_flags() & CINIT_FLAG_DEFER_DROP_PRIVILEGES) &&
 463         (cct->get_set_uid() || cct->get_set_gid())) {
 464       chown_path(conf->pid_file, cct->get_set_uid(), cct->get_set_gid(),
 465                  cct->get_set_uid_string(), cct->get_set_gid_string());
 466     }
 467
 468     return -1;
 469   }
 470
 471   cct->notify_pre_fork();
 472   // stop log thread
 473   cct->_log->flush();
 474   cct->_log->stop();
 475   return 0;
 476 }
 477
 478 void global_init_daemonize(CephContext *cct)
 479 {
 480   if (global_init_prefork(cct) < 0)
 481     return;
 482
 483 #if !defined(_AIX) && !defined(_WIN32)
 484   int ret = daemon(1, 1);
 485   if (ret) {
 486     ret = errno;
 487     derr << "global_init_daemonize: BUG: daemon error: "
 488          << cpp_strerror(ret) << dendl;
 489     exit(1);
 490   }
 491
 492   global_init_postfork_start(cct);
 493   global_init_postfork_finish(cct);
 494 #else
 495 # warning daemon not supported on aix
 496 #endif
 497 }
 498
 499 /* Make file descriptors 0, 1, and possibly 2 point to /dev/null.
 500  *
 501  * Instead of just closing fd, we redirect it to /dev/null with dup2().
 502  * We have to do this because otherwise some arbitrary call to open() later
 503  * in the program might get back one of these file descriptors. It's hard to
 504  * guarantee that nobody ever writes to stdout, even though they're not
 505  * supposed to.
 506  */
 507 int reopen_as_null(CephContext *cct, int fd)
 508 {
 509   int newfd = open(DEV_NULL, O_RDWR | O_CLOEXEC);
 510   if (newfd < 0) {
 511     int err = errno;
 512     lderr(cct) << __func__ << " failed to open /dev/null: " << cpp_strerror(err)
 513                << dendl;
 514     return -1;
 515   }
 516   // atomically dup newfd to target fd.  target fd is implicitly closed if
 517   // open and atomically replaced; see man dup2
 518   int r = dup2(newfd, fd);
 519   if (r < 0) {
 520     int err = errno;
 521     lderr(cct) << __func__ << " failed to dup2 " << fd << ": "
 522                << cpp_strerror(err) << dendl;
 523     return -1;
 524   }
 525   // close newfd (we cloned it to target fd)
 526   VOID_TEMP_FAILURE_RETRY(close(newfd));
 527   // N.B. FD_CLOEXEC is cleared on fd (see dup2(2))
 528   return 0;
 529 }
 530
 531 void global_init_postfork_start(CephContext *cct)
 532 {
 533   // reexpand the meta in child process
 534   cct->_conf.finalize_reexpand_meta();
 535
 536   // restart log thread
 537   cct->_log->start();
 538   cct->notify_post_fork();
 539
 540   reopen_as_null(cct, STDIN_FILENO);
 541
 542   const auto& conf = cct->_conf;
 543   if (pidfile_write(conf->pid_file) < 0)
 544     exit(1);
 545
 546   if ((cct->get_init_flags() & CINIT_FLAG_DEFER_DROP_PRIVILEGES) &&
 547       (cct->get_set_uid() || cct->get_set_gid())) {
 548     chown_path(conf->pid_file, cct->get_set_uid(), cct->get_set_gid(),
 549                cct->get_set_uid_string(), cct->get_set_gid_string());
 550   }
 551 }
 552
 553 void global_init_postfork_finish(CephContext *cct)
 554 {
 555   /* We only close stdout+stderr once the caller decides the daemonization
 556    * process is finished.  This way we can allow error or other messages to be
 557    * propagated in a manner that the user is able to see.
 558    */
 559   if (!(cct->get_init_flags() & CINIT_FLAG_NO_CLOSE_STDERR)) {
 560     int ret = global_init_shutdown_stderr(cct);
 561     if (ret) {
 562       derr << "global_init_daemonize: global_init_shutdown_stderr failed with "
 563            << "error code " << ret << dendl;
 564       exit(1);
 565     }
 566   }
 567
 568   reopen_as_null(cct, STDOUT_FILENO);
 569
 570   ldout(cct, 1) << "finished global_init_daemonize" << dendl;
 571 }
 572
 573
 574 void global_init_chdir(const CephContext *cct)
 575 {
 576   const auto& conf = cct->_conf;
 577   if (conf->chdir.empty())
 578     return;
 579   if (::chdir(conf->chdir.c_str())) {
 580     int err = errno;
 581     derr << "global_init_chdir: failed to chdir to directory: '"
 582          << conf->chdir << "': " << cpp_strerror(err) << dendl;
 583   }
 584 }
 585
 586 int global_init_shutdown_stderr(CephContext *cct)
 587 {
 588   reopen_as_null(cct, STDERR_FILENO);
 589   cct->_log->set_stderr_level(-2, -2);
 590   return 0;
 591 }
 592
 593 int global_init_preload_erasure_code(const CephContext *cct)
 594 {
 595   const auto& conf = cct->_conf;
 596   string plugins = conf->osd_erasure_code_plugins;
 597
 598   // validate that this is a not a legacy plugin
 599   std::list<string> plugins_list;
 600   get_str_list(plugins, plugins_list);
 601   for (auto i = plugins_list.begin(); i != plugins_list.end(); ++i) {
 602         string plugin_name = *i;
 603         string replacement = "";
 604
 605         if (plugin_name == "jerasure_generic" ||
 606             plugin_name == "jerasure_sse3" ||
 607             plugin_name == "jerasure_sse4" ||
 608             plugin_name == "jerasure_neon") {
 609           replacement = "jerasure";
 610         }
 611         else if (plugin_name == "shec_generic" ||
 612                  plugin_name == "shec_sse3" ||
 613                  plugin_name == "shec_sse4" ||
 614                  plugin_name == "shec_neon") {
 615           replacement = "shec";
 616         }
 617
 618         if (replacement != "") {
 619           dout(0) << "WARNING: osd_erasure_code_plugins contains plugin "
 620                   << plugin_name << " that is now deprecated. Please modify the value "
 621                   << "for osd_erasure_code_plugins to use "  << replacement << " instead." << dendl;
 622         }
 623   }
 624
 625   std::stringstream ss;
 626   int r = ceph::ErasureCodePluginRegistry::instance().preload(
 627     plugins,
 628     conf.get_val<std::string>("erasure_code_dir"),
 629     &ss);
 630   if (r)
 631     derr << ss.str() << dendl;
 632   else
 633     dout(0) << ss.str() << dendl;
 634   return r;
 635 }