[ceph.git] / ceph / src / common / pick_address.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2012 Inktank
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include "common/pick_address.h"

#include <bitset>
#include <netdb.h>
#include <netinet/in.h>
#include <string>
#include <string.h>
#include <vector>

#include <boost/algorithm/string/predicate.hpp>
#include <fmt/format.h>

#include "include/ipaddr.h"
#include "include/str_list.h"
#include "common/ceph_context.h"
#ifndef WITH_SEASTAR
#include "common/config.h"
#include "common/config_obs.h"
#endif
#include "common/debug.h"
#include "common/errno.h"
#include "common/numa.h"

#ifndef HAVE_IN_ADDR_T
typedef uint32_t in_addr_t;
#endif

#ifndef IN_LOOPBACKNET
#define IN_LOOPBACKNET 127
#endif

#define dout_subsys ceph_subsys_

using std::string;
using std::vector;

namespace {

bool matches_with_name(const ifaddrs& ifa, const std::string& if_name)
{
  return if_name.compare(ifa.ifa_name) == 0;
}

static int is_loopback_addr(sockaddr* addr)
{
  if (addr->sa_family == AF_INET) {
    const sockaddr_in* sin = (struct sockaddr_in *)(addr);
    const in_addr_t net = ntohl(sin->sin_addr.s_addr) >> IN_CLASSA_NSHIFT;
    return net == IN_LOOPBACKNET ? 1 : 0;
  } else if (addr->sa_family == AF_INET6) {
    sockaddr_in6* sin6 = (struct sockaddr_in6 *)(addr);
    return IN6_IS_ADDR_LOOPBACK(&sin6->sin6_addr) ? 1 : 0;
  } else {
    return -1;
  }
}

static int grade_addr(const ifaddrs& ifa)
{
  if (ifa.ifa_addr == nullptr) {
    return -1;
  }
  int score = 0;
  if (ifa.ifa_flags & IFF_UP) {
    score += 4;
  }
  switch (is_loopback_addr(ifa.ifa_addr)) {
  case 0:
    // prefer non-loopback addresses
    score += 2;
    break;
  case 1:
    score += 0;
    break;
  default:
    score = -1;
    break;
  }
  return score;
}

bool matches_with_net(const ifaddrs& ifa,
                      const sockaddr* net,
                      unsigned int prefix_len,
                      unsigned ipv)
{
  switch (net->sa_family) {
  case AF_INET:
    if (ipv & CEPH_PICK_ADDRESS_IPV4) {
      return matches_ipv4_in_subnet(ifa, (struct sockaddr_in*)net, prefix_len);
    }
    break;
  case AF_INET6:
    if (ipv & CEPH_PICK_ADDRESS_IPV6) {
      return matches_ipv6_in_subnet(ifa, (struct sockaddr_in6*)net, prefix_len);
    }
    break;
  }
  return false;
}

bool matches_with_net(CephContext *cct,
                      const ifaddrs& ifa,
                      const std::string& s,
                      unsigned ipv)
{
  struct sockaddr_storage net;
  unsigned int prefix_len;
  if (!parse_network(s.c_str(), &net, &prefix_len)) {
    lderr(cct) << "unable to parse network: " << s << dendl;
    exit(1);
  }
  return matches_with_net(ifa, (sockaddr*)&net, prefix_len, ipv);
}

int grade_with_numa_node(const ifaddrs& ifa, int numa_node)
{
#if defined(WITH_SEASTAR) || defined(_WIN32)
  return 0;
#else
  if (numa_node < 0) {
    return 0;
  }
  int if_node = -1;
  int r = get_iface_numa_node(ifa.ifa_name, &if_node);
  if (r < 0) {
    return 0;
  }
  return if_node == numa_node ? 1 : 0;
#endif
}
}

const struct sockaddr *find_ip_in_subnet_list(
  CephContext *cct,
  const struct ifaddrs *ifa,
  unsigned ipv,
  const std::string &networks,
  const std::string &interfaces,
  int numa_node)
{
  const auto ifs = get_str_list(interfaces);
  const auto nets = get_str_list(networks);
  if (!ifs.empty() && nets.empty()) {
      lderr(cct) << "interface names specified but not network names" << dendl;
      exit(1);
  }

  int best_score = 0;
  const sockaddr* best_addr = nullptr;
  for (const auto* addr = ifa; addr != nullptr; addr = addr->ifa_next) {
    if (!ifs.empty() &&
	std::none_of(std::begin(ifs), std::end(ifs),
                     [&](const auto& if_name) {
                       return matches_with_name(*addr, if_name);
                     })) {
      continue;
    }
    if (!nets.empty() &&
	std::none_of(std::begin(nets), std::end(nets),
                     [&](const auto& net) {
                       return matches_with_net(cct, *addr, net, ipv);
                     })) {
      continue;
    }
    int score = grade_addr(*addr);
    if (score < 0) {
      continue;
    }
    score += grade_with_numa_node(*addr, numa_node);
    if (score > best_score) {
      best_score = score;
      best_addr = addr->ifa_addr;
    }
  }
  return best_addr;
}

#ifndef WITH_SEASTAR
// observe this change
struct Observer : public md_config_obs_t {
  const char *keys[2];
  explicit Observer(const char *c) {
    keys[0] = c;
    keys[1] = NULL;
  }

  const char** get_tracked_conf_keys() const override {
    return (const char **)keys;
  }
  void handle_conf_change(const ConfigProxy& conf,
			  const std::set <std::string> &changed) override {
    // do nothing.
  }
};

static void fill_in_one_address(CephContext *cct,
				const struct ifaddrs *ifa,
				const string &networks,
				const string &interfaces,
				const char *conf_var,
				int numa_node = -1)
{
  const struct sockaddr *found = find_ip_in_subnet_list(
    cct,
    ifa,
    CEPH_PICK_ADDRESS_IPV4|CEPH_PICK_ADDRESS_IPV6,
    networks,
    interfaces,
    numa_node);
  if (!found) {
    lderr(cct) << "unable to find any IP address in networks '" << networks
	       << "' interfaces '" << interfaces << "'" << dendl;
    exit(1);
  }

  char buf[INET6_ADDRSTRLEN];
  int err;

  err = getnameinfo(found,
		    (found->sa_family == AF_INET)
		    ? sizeof(struct sockaddr_in)
		    : sizeof(struct sockaddr_in6),

		    buf, sizeof(buf),
		    nullptr, 0,
		    NI_NUMERICHOST);
  if (err != 0) {
    lderr(cct) << "unable to convert chosen address to string: " << gai_strerror(err) << dendl;
    exit(1);
  }

  Observer obs(conf_var);

  cct->_conf.add_observer(&obs);

  cct->_conf.set_val_or_die(conf_var, buf);
  cct->_conf.apply_changes(nullptr);

  cct->_conf.remove_observer(&obs);
}

void pick_addresses(CephContext *cct, int needs)
{
  auto public_addr = cct->_conf.get_val<entity_addr_t>("public_addr");
  auto public_network = cct->_conf.get_val<std::string>("public_network");
  auto public_network_interface =
    cct->_conf.get_val<std::string>("public_network_interface");
  auto cluster_addr = cct->_conf.get_val<entity_addr_t>("cluster_addr");
  auto cluster_network = cct->_conf.get_val<std::string>("cluster_network");
  auto cluster_network_interface =
    cct->_conf.get_val<std::string>("cluster_network_interface");

  struct ifaddrs *ifa;
  int r = getifaddrs(&ifa);
  if (r < 0) {
    string err = cpp_strerror(errno);
    lderr(cct) << "unable to fetch interfaces and addresses: " << err << dendl;
    exit(1);
  }
  auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
  if ((needs & CEPH_PICK_ADDRESS_PUBLIC) &&
    public_addr.is_blank_ip() && !public_network.empty()) {
    fill_in_one_address(cct, ifa, public_network, public_network_interface,
			"public_addr");
  }

  if ((needs & CEPH_PICK_ADDRESS_CLUSTER) && cluster_addr.is_blank_ip()) {
    if (!cluster_network.empty()) {
      fill_in_one_address(cct, ifa, cluster_network, cluster_network_interface,
			  "cluster_addr");
    } else {
      if (!public_network.empty()) {
        lderr(cct) << "Public network was set, but cluster network was not set " << dendl;
        lderr(cct) << "    Using public network also for cluster network" << dendl;
        fill_in_one_address(cct, ifa, public_network, public_network_interface,
			    "cluster_addr");
      }
    }
  }
}
#endif	// !WITH_SEASTAR

static std::optional<entity_addr_t> get_one_address(
  CephContext *cct,
  const struct ifaddrs *ifa,
  unsigned ipv,
  const string &networks,
  const string &interfaces,
  int numa_node = -1)
{
  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv,
							networks,
							interfaces,
							numa_node);
  if (!found) {
    std::string_view ip_type;
    if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) {
      ip_type = "IPv4 or IPv6";
    } else if (ipv & CEPH_PICK_ADDRESS_IPV4) {
      ip_type = "IPv4";
    } else {
      ip_type = "IPv6";
    }
    lderr(cct) << "unable to find any " << ip_type << " address in networks '"
               << networks << "' interfaces '" << interfaces << "'" << dendl;
    return {};
  }

  char buf[INET6_ADDRSTRLEN];
  int err;

  err = getnameinfo(found,
		    (found->sa_family == AF_INET)
		    ? sizeof(struct sockaddr_in)
		    : sizeof(struct sockaddr_in6),

		    buf, sizeof(buf),
		    nullptr, 0,
		    NI_NUMERICHOST);
  if (err != 0) {
    lderr(cct) << "unable to convert chosen address to string: " << gai_strerror(err) << dendl;
    return {};
  }

  entity_addr_t addr;
  if (addr.parse(buf)) {
    return addr;
  } else {
    return {};
  }
}

int pick_addresses(
  CephContext *cct,
  unsigned flags,
  struct ifaddrs *ifa,
  entity_addrvec_t *addrs,
  int preferred_numa_node)
{
  addrs->v.clear();

  unsigned addrt = (flags & (CEPH_PICK_ADDRESS_PUBLIC |
			     CEPH_PICK_ADDRESS_PUBLIC_BIND |
			     CEPH_PICK_ADDRESS_CLUSTER));
  // TODO: move to std::popcount when it's available for all release lines
  // we are interested in (quincy was a blocker at the time of writing)
  if (std::bitset<sizeof(addrt)*CHAR_BIT>(addrt).count() != 1) {
    // these flags are mutually exclusive and one of them must be
    // always set (in other words: it's mode selection).
    return -EINVAL;
  }
  unsigned msgrv = flags & (CEPH_PICK_ADDRESS_MSGR1 |
			    CEPH_PICK_ADDRESS_MSGR2);
  if (msgrv == 0) {
    if (cct->_conf.get_val<bool>("ms_bind_msgr1")) {
      msgrv |= CEPH_PICK_ADDRESS_MSGR1;
    }
    if (cct->_conf.get_val<bool>("ms_bind_msgr2")) {
      msgrv |= CEPH_PICK_ADDRESS_MSGR2;
    }
    if (msgrv == 0) {
      return -EINVAL;
    }
  }
  unsigned ipv = flags & (CEPH_PICK_ADDRESS_IPV4 |
			  CEPH_PICK_ADDRESS_IPV6);
  if (ipv == 0) {
    if (cct->_conf.get_val<bool>("ms_bind_ipv4")) {
      ipv |= CEPH_PICK_ADDRESS_IPV4;
    }
    if (cct->_conf.get_val<bool>("ms_bind_ipv6")) {
      ipv |= CEPH_PICK_ADDRESS_IPV6;
    }
    if (ipv == 0) {
      return -EINVAL;
    }
    if (cct->_conf.get_val<bool>("ms_bind_prefer_ipv4")) {
      flags |= CEPH_PICK_ADDRESS_PREFER_IPV4;
    } else {
      flags &= ~CEPH_PICK_ADDRESS_PREFER_IPV4;
    }
  }

  entity_addr_t addr;
  string networks;
  string interfaces;
  if (addrt & CEPH_PICK_ADDRESS_PUBLIC) {
    addr = cct->_conf.get_val<entity_addr_t>("public_addr");
    networks = cct->_conf.get_val<std::string>("public_network");
    interfaces =
      cct->_conf.get_val<std::string>("public_network_interface");
  } else if (addrt & CEPH_PICK_ADDRESS_PUBLIC_BIND) {
    addr = cct->_conf.get_val<entity_addr_t>("public_bind_addr");
    // XXX: we don't support _network nor _network_interface for
    // the public_bind addrs yet.
    if (addr.is_blank_ip()) {
      return -ENOENT;
    }
  } else {
    addr = cct->_conf.get_val<entity_addr_t>("cluster_addr");
    networks = cct->_conf.get_val<std::string>("cluster_network");
    interfaces =
      cct->_conf.get_val<std::string>("cluster_network_interface");
    if (networks.empty()) {
      lderr(cct) << "Falling back to public interface" << dendl;
      // fall back to public_ network and interface if cluster is not set
      networks = cct->_conf.get_val<std::string>("public_network");
      interfaces =
	cct->_conf.get_val<std::string>("public_network_interface");
    }
  }
  if (addr.is_blank_ip() &&
      !networks.empty()) {
    // note: pass in ipv to filter the matching addresses
    for (auto pick_mask :  {CEPH_PICK_ADDRESS_IPV4, CEPH_PICK_ADDRESS_IPV6}) {
      if (ipv & pick_mask) {
        auto ip_addr = get_one_address(cct, ifa, pick_mask,
                                       networks, interfaces,
                                       preferred_numa_node);
        if (ip_addr) {
          addrs->v.push_back(*ip_addr);
        } else {
          // picked but not found
          return -1;
        }
      }
    }
  }

  // note: we may have a blank addr here

  // ipv4 and/or ipv6?
  if (addrs->v.empty()) {
    addr.set_type(entity_addr_t::TYPE_MSGR2);
    for (auto pick_mask : {CEPH_PICK_ADDRESS_IPV4, CEPH_PICK_ADDRESS_IPV6}) {
      if (ipv & pick_mask) {
        addr.set_family(pick_mask == CEPH_PICK_ADDRESS_IPV4 ? AF_INET : AF_INET6);
        addrs->v.push_back(addr);
      }
    }
  }

  std::sort(addrs->v.begin(), addrs->v.end(),
	    [flags] (entity_addr_t& lhs, entity_addr_t& rhs) {
	      if (flags & CEPH_PICK_ADDRESS_PREFER_IPV4) {
		return lhs.is_ipv4() && rhs.is_ipv6();
	      } else {
		return lhs.is_ipv6() && rhs.is_ipv4();
	      }
	    });

  // msgr2 or legacy or both?
  if (msgrv == (CEPH_PICK_ADDRESS_MSGR1 | CEPH_PICK_ADDRESS_MSGR2)) {
    vector<entity_addr_t> v;
    v.swap(addrs->v);
    for (auto a : v) {
      a.set_type(entity_addr_t::TYPE_MSGR2);
      if (flags & CEPH_PICK_ADDRESS_DEFAULT_MON_PORTS) {
	a.set_port(CEPH_MON_PORT_IANA);
      }
      addrs->v.push_back(a);
      a.set_type(entity_addr_t::TYPE_LEGACY);
      if (flags & CEPH_PICK_ADDRESS_DEFAULT_MON_PORTS) {
	a.set_port(CEPH_MON_PORT_LEGACY);
      }
      addrs->v.push_back(a);
    }
  } else if (msgrv == CEPH_PICK_ADDRESS_MSGR1) {
    for (auto& a : addrs->v) {
      a.set_type(entity_addr_t::TYPE_LEGACY);
    }
  } else {
    for (auto& a : addrs->v) {
      a.set_type(entity_addr_t::TYPE_MSGR2);
    }
  }

  return 0;
}

int pick_addresses(
  CephContext *cct,
  unsigned flags,
  entity_addrvec_t *addrs,
  int preferred_numa_node)
{
  struct ifaddrs *ifa;
  int r = getifaddrs(&ifa);
  if (r < 0) {
    r = -errno;
    string err = cpp_strerror(r);
    lderr(cct) << "unable to fetch interfaces and addresses: "
	       <<  cpp_strerror(r) << dendl;
    return r;
  }
  r = pick_addresses(cct, flags, ifa, addrs, preferred_numa_node);
  freeifaddrs(ifa);
  return r;
}

std::string pick_iface(CephContext *cct, const struct sockaddr_storage &network)
{
  struct ifaddrs *ifa;
  int r = getifaddrs(&ifa);
  if (r < 0) {
    string err = cpp_strerror(errno);
    lderr(cct) << "unable to fetch interfaces and addresses: " << err << dendl;
    return {};
  }
  auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
  const unsigned int prefix_len = std::max(sizeof(in_addr::s_addr), sizeof(in6_addr::s6_addr)) * CHAR_BIT;
  for (auto addr = ifa; addr != nullptr; addr = addr->ifa_next) {
    if (matches_with_net(*ifa, (const struct sockaddr *) &network, prefix_len,
			 CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6)) {
      return addr->ifa_name;
    }
  }
  return {};
}


bool have_local_addr(CephContext *cct, const std::list<entity_addr_t>& ls, entity_addr_t *match)
{
  struct ifaddrs *ifa;
  int r = getifaddrs(&ifa);
  if (r < 0) {
    lderr(cct) << "unable to fetch interfaces and addresses: " << cpp_strerror(errno) << dendl;
    exit(1);
  }
  auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });

  for (struct ifaddrs *addrs = ifa; addrs != nullptr; addrs = addrs->ifa_next) {
    if (addrs->ifa_addr) {
      entity_addr_t a;
      a.set_sockaddr(addrs->ifa_addr);
      for (auto& p : ls) {
        if (a.is_same_host(p)) {
          *match = p;
          return true;
        }
      }
    }
  }
  return false;
}

int get_iface_numa_node(
  const std::string& iface,
  int *node)
{
  enum class iface_t {
    PHY_PORT,
    BOND_PORT
  } ifatype = iface_t::PHY_PORT;
  std::string_view ifa{iface};
  if (auto pos = ifa.find(":"); pos != ifa.npos) {
    ifa.remove_suffix(ifa.size() - pos);
  }
  string fn = fmt::format("/sys/class/net/{}/device/numa_node", ifa);
  int fd = ::open(fn.c_str(), O_RDONLY);
  if (fd < 0) {
    fn = fmt::format("/sys/class/net/{}/bonding/slaves", ifa);
    fd = ::open(fn.c_str(), O_RDONLY);
    if (fd < 0) {
      return -errno;
    }
    ifatype = iface_t::BOND_PORT;
  }

  int r = 0;
  char buf[1024];
  char *endptr = 0;
  r = safe_read(fd, &buf, sizeof(buf));
  if (r < 0) {
    goto out;
  }
  buf[r] = 0;
  while (r > 0 && ::isspace(buf[--r])) {
    buf[r] = 0;
  }

  switch (ifatype) {
  case iface_t::PHY_PORT:
    *node = strtoll(buf, &endptr, 10);
    if (endptr != buf + strlen(buf)) {
      r = -EINVAL;
      goto out;
    }
    r = 0;
    break;
  case iface_t::BOND_PORT:
    int bond_node = -1;
    std::vector<std::string> sv;
    std::string ifacestr = buf;
    get_str_vec(ifacestr, " ", sv);
    for (auto& iter : sv) {
      int bn = -1;
      r = get_iface_numa_node(iter, &bn);
      if (r >= 0) {
        if (bond_node == -1 || bn == bond_node) {
          bond_node = bn;
        } else {
          *node = -2;
          goto out;
        }
      } else {
        goto out;
      }
    }
    *node = bond_node;
    break;
  }

  out:
  ::close(fd);
  return r;
}
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph - scalable distributed file system
	5	*
	6	* Copyright (C) 2004-2012 Inktank
	7	*
	8	* This is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License version 2.1, as published by the Free Software
	11	* Foundation. See file COPYING.
	12	*
	13	*/
	14
	15	#include "common/pick_address.h"
9f95a23c	16
39ae355f	17	#include <bitset>
9f95a23c	18	#include <netdb.h>
522d829b	19	#include <netinet/in.h>
9f95a23c TL	20	#include <string>
	21	#include <string.h>
	22	#include <vector>
	23
522d829b	24	#include <boost/algorithm/string/predicate.hpp>
9f95a23c TL	25	#include <fmt/format.h>
9f95a23c TL	26
7c673cae FG	27	#include "include/ipaddr.h"
7c673cae FG	28	#include "include/str_list.h"
11fdf7f2 TL	29	#include "common/ceph_context.h"
	30	#ifndef WITH_SEASTAR
	31	#include "common/config.h"
	32	#include "common/config_obs.h"
	33	#endif
7c673cae FG	34	#include "common/debug.h"
7c673cae FG	35	#include "common/errno.h"
11fdf7f2	36	#include "common/numa.h"
7c673cae	37
522d829b TL	38	#ifndef HAVE_IN_ADDR_T
	39	typedef uint32_t in_addr_t;
	40	#endif
	41
	42	#ifndef IN_LOOPBACKNET
	43	#define IN_LOOPBACKNET 127
	44	#endif
	45
7c673cae FG	46	#define dout_subsys ceph_subsys_
7c673cae FG	47
f67539c2 TL	48	using std::string;
	49	using std::vector;
	50
522d829b TL	51	namespace {
	52
	53	bool matches_with_name(const ifaddrs& ifa, const std::string& if_name)
	54	{
	55	return if_name.compare(ifa.ifa_name) == 0;
	56	}
	57
	58	static int is_loopback_addr(sockaddr* addr)
	59	{
	60	if (addr->sa_family == AF_INET) {
	61	const sockaddr_in* sin = (struct sockaddr_in *)(addr);
	62	const in_addr_t net = ntohl(sin->sin_addr.s_addr) >> IN_CLASSA_NSHIFT;
	63	return net == IN_LOOPBACKNET ? 1 : 0;
	64	} else if (addr->sa_family == AF_INET6) {
	65	sockaddr_in6* sin6 = (struct sockaddr_in6 *)(addr);
	66	return IN6_IS_ADDR_LOOPBACK(&sin6->sin6_addr) ? 1 : 0;
	67	} else {
	68	return -1;
	69	}
	70	}
	71
	72	static int grade_addr(const ifaddrs& ifa)
	73	{
	74	if (ifa.ifa_addr == nullptr) {
	75	return -1;
	76	}
	77	int score = 0;
	78	if (ifa.ifa_flags & IFF_UP) {
	79	score += 4;
	80	}
	81	switch (is_loopback_addr(ifa.ifa_addr)) {
	82	case 0:
	83	// prefer non-loopback addresses
	84	score += 2;
	85	break;
	86	case 1:
	87	score += 0;
	88	break;
	89	default:
	90	score = -1;
	91	break;
	92	}
	93	return score;
	94	}
	95
	96	bool matches_with_net(const ifaddrs& ifa,
	97	const sockaddr* net,
	98	unsigned int prefix_len,
	99	unsigned ipv)
	100	{
	101	switch (net->sa_family) {
	102	case AF_INET:
	103	if (ipv & CEPH_PICK_ADDRESS_IPV4) {
	104	return matches_ipv4_in_subnet(ifa, (struct sockaddr_in*)net, prefix_len);
	105	}
	106	break;
	107	case AF_INET6:
	108	if (ipv & CEPH_PICK_ADDRESS_IPV6) {
	109	return matches_ipv6_in_subnet(ifa, (struct sockaddr_in6*)net, prefix_len);
	110	}
	111	break;
	112	}
	113	return false;
	114	}
115
116	bool matches_with_net(CephContext *cct,
117	const ifaddrs& ifa,
118	const std::string& s,
119	unsigned ipv)
120	{
121	struct sockaddr_storage net;
122	unsigned int prefix_len;
123	if (!parse_network(s.c_str(), &net, &prefix_len)) {
124	lderr(cct) << "unable to parse network: " << s << dendl;
125	exit(1);
126	}
127	return matches_with_net(ifa, (sockaddr*)&net, prefix_len, ipv);
128	}
129
130	int grade_with_numa_node(const ifaddrs& ifa, int numa_node)
131	{
132	#if defined(WITH_SEASTAR) \|\| defined(_WIN32)
133	return 0;
134	#else
135	if (numa_node < 0) {
136	return 0;
137	}
138	int if_node = -1;
139	int r = get_iface_numa_node(ifa.ifa_name, &if_node);
140	if (r < 0) {
141	return 0;
142	}
143	return if_node == numa_node ? 1 : 0;
144	#endif
145	}
146	}
147
3efd9988 FG	148	const struct sockaddr *find_ip_in_subnet_list(
	149	CephContext *cct,
	150	const struct ifaddrs *ifa,
11fdf7f2	151	unsigned ipv,
3efd9988	152	const std::string &networks,
11fdf7f2 TL	153	const std::string &interfaces,
11fdf7f2 TL	154	int numa_node)
7c673cae	155	{
522d829b TL	156	const auto ifs = get_str_list(interfaces);
	157	const auto nets = get_str_list(networks);
	158	if (!ifs.empty() && nets.empty()) {
3efd9988 FG	159	lderr(cct) << "interface names specified but not network names" << dendl;
3efd9988 FG	160	exit(1);
3efd9988	161	}
7c673cae	162
522d829b TL	163	int best_score = 0;
	164	const sockaddr* best_addr = nullptr;
	165	for (const auto* addr = ifa; addr != nullptr; addr = addr->ifa_next) {
	166	if (!ifs.empty() &&
	167	std::none_of(std::begin(ifs), std::end(ifs),
	168	[&](const auto& if_name) {
	169	return matches_with_name(*addr, if_name);
	170	})) {
	171	continue;
3efd9988	172	}
522d829b TL	173	if (!nets.empty() &&
	174	std::none_of(std::begin(nets), std::end(nets),
	175	[&](const auto& net) {
	176	return matches_with_net(cct, *addr, net, ipv);
	177	})) {
	178	continue;
11fdf7f2	179	}
522d829b TL	180	int score = grade_addr(*addr);
	181	if (score < 0) {
	182	continue;
3efd9988	183	}
522d829b TL	184	score += grade_with_numa_node(*addr, numa_node);
	185	if (score > best_score) {
	186	best_score = score;
	187	best_addr = addr->ifa_addr;
7c673cae	188	}
3efd9988	189	}
522d829b	190	return best_addr;
7c673cae FG	191	}
7c673cae FG	192
11fdf7f2	193	#ifndef WITH_SEASTAR
7c673cae FG	194	// observe this change
	195	struct Observer : public md_config_obs_t {
	196	const char *keys[2];
	197	explicit Observer(const char *c) {
	198	keys[0] = c;
	199	keys[1] = NULL;
	200	}
	201
	202	const char** get_tracked_conf_keys() const override {
	203	return (const char **)keys;
	204	}
11fdf7f2	205	void handle_conf_change(const ConfigProxy& conf,
7c673cae FG	206	const std::set <std::string> &changed) override {
	207	// do nothing.
	208	}
	209	};
	210
	211	static void fill_in_one_address(CephContext *cct,
	212	const struct ifaddrs *ifa,
522d829b TL	213	const string &networks,
522d829b TL	214	const string &interfaces,
11fdf7f2 TL	215	const char *conf_var,
11fdf7f2 TL	216	int numa_node = -1)
7c673cae	217	{
11fdf7f2 TL	218	const struct sockaddr *found = find_ip_in_subnet_list(
	219	cct,
	220	ifa,
	221	CEPH_PICK_ADDRESS_IPV4\|CEPH_PICK_ADDRESS_IPV6,
	222	networks,
	223	interfaces,
	224	numa_node);
7c673cae	225	if (!found) {
3efd9988 FG	226	lderr(cct) << "unable to find any IP address in networks '" << networks
3efd9988 FG	227	<< "' interfaces '" << interfaces << "'" << dendl;
7c673cae FG	228	exit(1);
	229	}
	230
	231	char buf[INET6_ADDRSTRLEN];
	232	int err;
	233
	234	err = getnameinfo(found,
	235	(found->sa_family == AF_INET)
	236	? sizeof(struct sockaddr_in)
	237	: sizeof(struct sockaddr_in6),
	238
	239	buf, sizeof(buf),
11fdf7f2	240	nullptr, 0,
7c673cae FG	241	NI_NUMERICHOST);
	242	if (err != 0) {
	243	lderr(cct) << "unable to convert chosen address to string: " << gai_strerror(err) << dendl;
	244	exit(1);
	245	}
	246
	247	Observer obs(conf_var);
	248
11fdf7f2	249	cct->_conf.add_observer(&obs);
7c673cae	250
11fdf7f2 TL	251	cct->_conf.set_val_or_die(conf_var, buf);
11fdf7f2 TL	252	cct->_conf.apply_changes(nullptr);
7c673cae	253
11fdf7f2	254	cct->_conf.remove_observer(&obs);
7c673cae FG	255	}
	256
	257	void pick_addresses(CephContext *cct, int needs)
	258	{
11fdf7f2 TL	259	auto public_addr = cct->_conf.get_val<entity_addr_t>("public_addr");
	260	auto public_network = cct->_conf.get_val<std::string>("public_network");
	261	auto public_network_interface =
	262	cct->_conf.get_val<std::string>("public_network_interface");
	263	auto cluster_addr = cct->_conf.get_val<entity_addr_t>("cluster_addr");
	264	auto cluster_network = cct->_conf.get_val<std::string>("cluster_network");
	265	auto cluster_network_interface =
	266	cct->_conf.get_val<std::string>("cluster_network_interface");
	267
522d829b TL	268	struct ifaddrs *ifa;
522d829b TL	269	int r = getifaddrs(&ifa);
11fdf7f2	270	if (r < 0) {
7c673cae FG	271	string err = cpp_strerror(errno);
	272	lderr(cct) << "unable to fetch interfaces and addresses: " << err << dendl;
	273	exit(1);
	274	}
522d829b	275	auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
11fdf7f2 TL	276	if ((needs & CEPH_PICK_ADDRESS_PUBLIC) &&
	277	public_addr.is_blank_ip() && !public_network.empty()) {
	278	fill_in_one_address(cct, ifa, public_network, public_network_interface,
522d829b	279	"public_addr");
7c673cae FG	280	}
7c673cae FG	281
11fdf7f2 TL	282	if ((needs & CEPH_PICK_ADDRESS_CLUSTER) && cluster_addr.is_blank_ip()) {
	283	if (!cluster_network.empty()) {
	284	fill_in_one_address(cct, ifa, cluster_network, cluster_network_interface,
522d829b	285	"cluster_addr");
7c673cae	286	} else {
11fdf7f2	287	if (!public_network.empty()) {
7c673cae FG	288	lderr(cct) << "Public network was set, but cluster network was not set " << dendl;
7c673cae FG	289	lderr(cct) << " Using public network also for cluster network" << dendl;
11fdf7f2	290	fill_in_one_address(cct, ifa, public_network, public_network_interface,
522d829b	291	"cluster_addr");
7c673cae FG	292	}
	293	}
	294	}
7c673cae	295	}
11fdf7f2 TL	296	#endif // !WITH_SEASTAR
11fdf7f2 TL	297
20effc67	298	static std::optional<entity_addr_t> get_one_address(
11fdf7f2 TL	299	CephContext *cct,
	300	const struct ifaddrs *ifa,
	301	unsigned ipv,
522d829b TL	302	const string &networks,
522d829b TL	303	const string &interfaces,
11fdf7f2 TL	304	int numa_node = -1)
11fdf7f2 TL	305	{
522d829b TL	306	const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv,
	307	networks,
	308	interfaces,
	309	numa_node);
11fdf7f2	310	if (!found) {
20effc67	311	std::string_view ip_type;
11fdf7f2 TL	312	if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) {
	313	ip_type = "IPv4 or IPv6";
	314	} else if (ipv & CEPH_PICK_ADDRESS_IPV4) {
	315	ip_type = "IPv4";
	316	} else {
	317	ip_type = "IPv6";
	318	}
	319	lderr(cct) << "unable to find any " << ip_type << " address in networks '"
	320	<< networks << "' interfaces '" << interfaces << "'" << dendl;
20effc67	321	return {};
11fdf7f2 TL	322	}
	323
	324	char buf[INET6_ADDRSTRLEN];
	325	int err;
7c673cae	326
11fdf7f2 TL	327	err = getnameinfo(found,
	328	(found->sa_family == AF_INET)
	329	? sizeof(struct sockaddr_in)
	330	: sizeof(struct sockaddr_in6),
	331
	332	buf, sizeof(buf),
	333	nullptr, 0,
	334	NI_NUMERICHOST);
	335	if (err != 0) {
	336	lderr(cct) << "unable to convert chosen address to string: " << gai_strerror(err) << dendl;
20effc67	337	return {};
11fdf7f2 TL	338	}
	339
	340	entity_addr_t addr;
20effc67 TL	341	if (addr.parse(buf)) {
	342	return addr;
	343	} else {
	344	return {};
11fdf7f2	345	}
11fdf7f2 TL	346	}
	347
	348	int pick_addresses(
	349	CephContext *cct,
	350	unsigned flags,
	351	struct ifaddrs *ifa,
	352	entity_addrvec_t *addrs,
	353	int preferred_numa_node)
	354	{
	355	addrs->v.clear();
	356
	357	unsigned addrt = (flags & (CEPH_PICK_ADDRESS_PUBLIC \|
39ae355f	358	CEPH_PICK_ADDRESS_PUBLIC_BIND \|
11fdf7f2	359	CEPH_PICK_ADDRESS_CLUSTER));
39ae355f TL	360	// TODO: move to std::popcount when it's available for all release lines
	361	// we are interested in (quincy was a blocker at the time of writing)
	362	if (std::bitset<sizeof(addrt)*CHAR_BIT>(addrt).count() != 1) {
	363	// these flags are mutually exclusive and one of them must be
	364	// always set (in other words: it's mode selection).
11fdf7f2 TL	365	return -EINVAL;
	366	}
	367	unsigned msgrv = flags & (CEPH_PICK_ADDRESS_MSGR1 \|
	368	CEPH_PICK_ADDRESS_MSGR2);
	369	if (msgrv == 0) {
	370	if (cct->_conf.get_val<bool>("ms_bind_msgr1")) {
	371	msgrv \|= CEPH_PICK_ADDRESS_MSGR1;
	372	}
	373	if (cct->_conf.get_val<bool>("ms_bind_msgr2")) {
	374	msgrv \|= CEPH_PICK_ADDRESS_MSGR2;
	375	}
	376	if (msgrv == 0) {
	377	return -EINVAL;
	378	}
	379	}
	380	unsigned ipv = flags & (CEPH_PICK_ADDRESS_IPV4 \|
	381	CEPH_PICK_ADDRESS_IPV6);
	382	if (ipv == 0) {
	383	if (cct->_conf.get_val<bool>("ms_bind_ipv4")) {
	384	ipv \|= CEPH_PICK_ADDRESS_IPV4;
	385	}
	386	if (cct->_conf.get_val<bool>("ms_bind_ipv6")) {
	387	ipv \|= CEPH_PICK_ADDRESS_IPV6;
	388	}
	389	if (ipv == 0) {
	390	return -EINVAL;
	391	}
	392	if (cct->_conf.get_val<bool>("ms_bind_prefer_ipv4")) {
	393	flags \|= CEPH_PICK_ADDRESS_PREFER_IPV4;
	394	} else {
	395	flags &= ~CEPH_PICK_ADDRESS_PREFER_IPV4;
	396	}
	397	}
	398
	399	entity_addr_t addr;
	400	string networks;
	401	string interfaces;
	402	if (addrt & CEPH_PICK_ADDRESS_PUBLIC) {
	403	addr = cct->_conf.get_val<entity_addr_t>("public_addr");
	404	networks = cct->_conf.get_val<std::string>("public_network");
	405	interfaces =
	406	cct->_conf.get_val<std::string>("public_network_interface");
39ae355f TL	407	} else if (addrt & CEPH_PICK_ADDRESS_PUBLIC_BIND) {
	408	addr = cct->_conf.get_val<entity_addr_t>("public_bind_addr");
	409	// XXX: we don't support _network nor _network_interface for
	410	// the public_bind addrs yet.
	411	if (addr.is_blank_ip()) {
	412	return -ENOENT;
	413	}
11fdf7f2 TL	414	} else {
	415	addr = cct->_conf.get_val<entity_addr_t>("cluster_addr");
	416	networks = cct->_conf.get_val<std::string>("cluster_network");
	417	interfaces =
	418	cct->_conf.get_val<std::string>("cluster_network_interface");
	419	if (networks.empty()) {
	420	lderr(cct) << "Falling back to public interface" << dendl;
	421	// fall back to public_ network and interface if cluster is not set
	422	networks = cct->_conf.get_val<std::string>("public_network");
	423	interfaces =
	424	cct->_conf.get_val<std::string>("public_network_interface");
	425	}
	426	}
	427	if (addr.is_blank_ip() &&
	428	!networks.empty()) {
522d829b	429	// note: pass in ipv to filter the matching addresses
20effc67 TL	430	for (auto pick_mask : {CEPH_PICK_ADDRESS_IPV4, CEPH_PICK_ADDRESS_IPV6}) {
	431	if (ipv & pick_mask) {
	432	auto ip_addr = get_one_address(cct, ifa, pick_mask,
	433	networks, interfaces,
	434	preferred_numa_node);
	435	if (ip_addr) {
	436	addrs->v.push_back(*ip_addr);
	437	} else {
	438	// picked but not found
	439	return -1;
	440	}
	441	}
11fdf7f2 TL	442	}
	443	}
	444
	445	// note: we may have a blank addr here
	446
	447	// ipv4 and/or ipv6?
	448	if (addrs->v.empty()) {
11fdf7f2	449	addr.set_type(entity_addr_t::TYPE_MSGR2);
20effc67 TL	450	for (auto pick_mask : {CEPH_PICK_ADDRESS_IPV4, CEPH_PICK_ADDRESS_IPV6}) {
	451	if (ipv & pick_mask) {
	452	addr.set_family(pick_mask == CEPH_PICK_ADDRESS_IPV4 ? AF_INET : AF_INET6);
	453	addrs->v.push_back(addr);
	454	}
11fdf7f2 TL	455	}
	456	}
	457
20effc67 TL	458	std::sort(addrs->v.begin(), addrs->v.end(),
	459	[flags] (entity_addr_t& lhs, entity_addr_t& rhs) {
	460	if (flags & CEPH_PICK_ADDRESS_PREFER_IPV4) {
	461	return lhs.is_ipv4() && rhs.is_ipv6();
	462	} else {
	463	return lhs.is_ipv6() && rhs.is_ipv4();
	464	}
	465	});
	466
11fdf7f2 TL	467	// msgr2 or legacy or both?
	468	if (msgrv == (CEPH_PICK_ADDRESS_MSGR1 \| CEPH_PICK_ADDRESS_MSGR2)) {
	469	vector<entity_addr_t> v;
	470	v.swap(addrs->v);
	471	for (auto a : v) {
	472	a.set_type(entity_addr_t::TYPE_MSGR2);
	473	if (flags & CEPH_PICK_ADDRESS_DEFAULT_MON_PORTS) {
	474	a.set_port(CEPH_MON_PORT_IANA);
	475	}
	476	addrs->v.push_back(a);
	477	a.set_type(entity_addr_t::TYPE_LEGACY);
	478	if (flags & CEPH_PICK_ADDRESS_DEFAULT_MON_PORTS) {
	479	a.set_port(CEPH_MON_PORT_LEGACY);
	480	}
	481	addrs->v.push_back(a);
	482	}
	483	} else if (msgrv == CEPH_PICK_ADDRESS_MSGR1) {
	484	for (auto& a : addrs->v) {
	485	a.set_type(entity_addr_t::TYPE_LEGACY);
	486	}
	487	} else {
	488	for (auto& a : addrs->v) {
	489	a.set_type(entity_addr_t::TYPE_MSGR2);
	490	}
	491	}
	492
	493	return 0;
	494	}
	495
	496	int pick_addresses(
	497	CephContext *cct,
	498	unsigned flags,
	499	entity_addrvec_t *addrs,
	500	int preferred_numa_node)
	501	{
	502	struct ifaddrs *ifa;
	503	int r = getifaddrs(&ifa);
	504	if (r < 0) {
	505	r = -errno;
	506	string err = cpp_strerror(r);
	507	lderr(cct) << "unable to fetch interfaces and addresses: "
	508	<< cpp_strerror(r) << dendl;
	509	return r;
	510	}
	511	r = pick_addresses(cct, flags, ifa, addrs, preferred_numa_node);
	512	freeifaddrs(ifa);
	513	return r;
	514	}
b5b8bbf5 FG	515
	516	std::string pick_iface(CephContext *cct, const struct sockaddr_storage &network)
	517	{
	518	struct ifaddrs *ifa;
	519	int r = getifaddrs(&ifa);
	520	if (r < 0) {
	521	string err = cpp_strerror(errno);
	522	lderr(cct) << "unable to fetch interfaces and addresses: " << err << dendl;
	523	return {};
	524	}
522d829b	525	auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
f67539c2	526	const unsigned int prefix_len = std::max(sizeof(in_addr::s_addr), sizeof(in6_addr::s6_addr)) * CHAR_BIT;
522d829b TL	527	for (auto addr = ifa; addr != nullptr; addr = addr->ifa_next) {
	528	if (matches_with_net(ifa, (const struct sockaddr ) &network, prefix_len,
	529	CEPH_PICK_ADDRESS_IPV4 \| CEPH_PICK_ADDRESS_IPV6)) {
	530	return addr->ifa_name;
	531	}
b5b8bbf5	532	}
522d829b	533	return {};
b5b8bbf5 FG	534	}
	535
	536
f67539c2	537	bool have_local_addr(CephContext cct, const std::list<entity_addr_t>& ls, entity_addr_t match)
7c673cae FG	538	{
	539	struct ifaddrs *ifa;
	540	int r = getifaddrs(&ifa);
	541	if (r < 0) {
	542	lderr(cct) << "unable to fetch interfaces and addresses: " << cpp_strerror(errno) << dendl;
	543	exit(1);
	544	}
522d829b	545	auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
7c673cae	546
11fdf7f2	547	for (struct ifaddrs *addrs = ifa; addrs != nullptr; addrs = addrs->ifa_next) {
7c673cae FG	548	if (addrs->ifa_addr) {
	549	entity_addr_t a;
	550	a.set_sockaddr(addrs->ifa_addr);
11fdf7f2 TL	551	for (auto& p : ls) {
	552	if (a.is_same_host(p)) {
	553	*match = p;
522d829b	554	return true;
7c673cae FG	555	}
	556	}
	557	}
	558	}
522d829b	559	return false;
7c673cae	560	}
11fdf7f2 TL	561
	562	int get_iface_numa_node(
	563	const std::string& iface,
	564	int *node)
	565	{
9f95a23c TL	566	enum class iface_t {
	567	PHY_PORT,
	568	BOND_PORT
	569	} ifatype = iface_t::PHY_PORT;
f67539c2	570	std::string_view ifa{iface};
9f95a23c TL	571	if (auto pos = ifa.find(":"); pos != ifa.npos) {
9f95a23c TL	572	ifa.remove_suffix(ifa.size() - pos);
92f5a8d4	573	}
9f95a23c	574	string fn = fmt::format("/sys/class/net/{}/device/numa_node", ifa);
92f5a8d4 TL	575	int fd = ::open(fn.c_str(), O_RDONLY);
92f5a8d4 TL	576	if (fd < 0) {
9f95a23c	577	fn = fmt::format("/sys/class/net/{}/bonding/slaves", ifa);
92f5a8d4 TL	578	fd = ::open(fn.c_str(), O_RDONLY);
	579	if (fd < 0) {
	580	return -errno;
	581	}
9f95a23c	582	ifatype = iface_t::BOND_PORT;
92f5a8d4	583	}
11fdf7f2 TL	584
	585	int r = 0;
	586	char buf[1024];
	587	char *endptr = 0;
11fdf7f2 TL	588	r = safe_read(fd, &buf, sizeof(buf));
	589	if (r < 0) {
	590	goto out;
	591	}
	592	buf[r] = 0;
	593	while (r > 0 && ::isspace(buf[--r])) {
	594	buf[r] = 0;
	595	}
92f5a8d4 TL	596
92f5a8d4 TL	597	switch (ifatype) {
9f95a23c	598	case iface_t::PHY_PORT:
92f5a8d4 TL	599	*node = strtoll(buf, &endptr, 10);
	600	if (endptr != buf + strlen(buf)) {
	601	r = -EINVAL;
	602	goto out;
	603	}
	604	r = 0;
	605	break;
9f95a23c TL	606	case iface_t::BOND_PORT:
9f95a23c TL	607	int bond_node = -1;
92f5a8d4	608	std::vector<std::string> sv;
9f95a23c TL	609	std::string ifacestr = buf;
9f95a23c TL	610	get_str_vec(ifacestr, " ", sv);
92f5a8d4 TL	611	for (auto& iter : sv) {
	612	int bn = -1;
	613	r = get_iface_numa_node(iter, &bn);
	614	if (r >= 0) {
	615	if (bond_node == -1 \|\| bn == bond_node) {
	616	bond_node = bn;
	617	} else {
	618	*node = -2;
	619	goto out;
	620	}
	621	} else {
	622	goto out;
	623	}
	624	}
	625	*node = bond_node;
	626	break;
11fdf7f2	627	}
92f5a8d4 TL	628
92f5a8d4 TL	629	out:
11fdf7f2 TL	630	::close(fd);
	631	return r;
	632	}
92f5a8d4	633