*/
#include "common/pick_address.h"
+
+#include <netdb.h>
+#include <string>
+#include <string.h>
+#include <vector>
+
+#include <fmt/format.h>
+
#include "include/ipaddr.h"
+#include "include/scope_guard.h"
#include "include/str_list.h"
#include "common/ceph_context.h"
#ifndef WITH_SEASTAR
#include "common/errno.h"
#include "common/numa.h"
-#include <netdb.h>
-#include <string>
-#include <string.h>
-#include <vector>
+#include <net/if.h>
+#include <netinet/in.h>
#define dout_subsys ceph_subsys_
+namespace {
+
+bool matches_with_name(const ifaddrs& ifa, const std::string& if_name)
+{
+ return if_name.compare(ifa.ifa_name) == 0;
+}
+
+static int is_loopback_addr(sockaddr* addr)
+{
+ if (addr->sa_family == AF_INET) {
+ const sockaddr_in* sin = (struct sockaddr_in *)(addr);
+ const in_addr_t net = ntohl(sin->sin_addr.s_addr) >> IN_CLASSA_NSHIFT;
+ return net == IN_LOOPBACKNET ? 1 : 0;
+ } else if (addr->sa_family == AF_INET6) {
+ sockaddr_in6* sin6 = (struct sockaddr_in6 *)(addr);
+ return IN6_IS_ADDR_LOOPBACK(&sin6->sin6_addr) ? 1 : 0;
+ } else {
+ return -1;
+ }
+}
+
+static int grade_addr(const ifaddrs& ifa)
+{
+ if (ifa.ifa_addr == nullptr) {
+ return -1;
+ }
+ int score = 0;
+ if (ifa.ifa_flags & IFF_UP) {
+ score += 4;
+ }
+ switch (is_loopback_addr(ifa.ifa_addr)) {
+ case 0:
+ // prefer non-loopback addresses
+ score += 2;
+ break;
+ case 1:
+ score += 0;
+ break;
+ default:
+ score = -1;
+ break;
+ }
+ return score;
+}
+
+bool matches_with_net(const ifaddrs& ifa,
+ const sockaddr* net,
+ unsigned int prefix_len,
+ unsigned ipv)
+{
+ switch (net->sa_family) {
+ case AF_INET:
+ if (ipv & CEPH_PICK_ADDRESS_IPV4) {
+ return matches_ipv4_in_subnet(ifa, (struct sockaddr_in*)net, prefix_len);
+ }
+ break;
+ case AF_INET6:
+ if (ipv & CEPH_PICK_ADDRESS_IPV6) {
+ return matches_ipv6_in_subnet(ifa, (struct sockaddr_in6*)net, prefix_len);
+ }
+ break;
+ }
+ return false;
+}
+
+bool matches_with_net(CephContext *cct,
+ const ifaddrs& ifa,
+ const std::string& s,
+ unsigned ipv)
+{
+ struct sockaddr_storage net;
+ unsigned int prefix_len;
+ if (!parse_network(s.c_str(), &net, &prefix_len)) {
+ lderr(cct) << "unable to parse network: " << s << dendl;
+ exit(1);
+ }
+ return matches_with_net(ifa, (sockaddr*)&net, prefix_len, ipv);
+}
+
+int grade_with_numa_node(const ifaddrs& ifa, int numa_node)
+{
+#if defined(WITH_SEASTAR) || defined(_WIN32)
+ return 0;
+#else
+ if (numa_node < 0) {
+ return 0;
+ }
+ int if_node = -1;
+ int r = get_iface_numa_node(ifa.ifa_name, &if_node);
+ if (r < 0) {
+ return 0;
+ }
+ return if_node == numa_node ? 1 : 0;
+#endif
+}
+}
+
const struct sockaddr *find_ip_in_subnet_list(
CephContext *cct,
const struct ifaddrs *ifa,
const std::string &interfaces,
int numa_node)
{
- std::list<string> nets;
- get_str_list(networks, nets);
- std::list<string> ifs;
- get_str_list(interfaces, ifs);
-
- // filter interfaces by name
- const struct ifaddrs *filtered = nullptr;
- if (ifs.empty()) {
- filtered = ifa;
- } else {
- if (nets.empty()) {
+ const auto ifs = get_str_list(interfaces);
+ const auto nets = get_str_list(networks);
+ if (!ifs.empty() && nets.empty()) {
lderr(cct) << "interface names specified but not network names" << dendl;
exit(1);
- }
- const struct ifaddrs *t = ifa;
- struct ifaddrs *head = 0;
- while (t) {
- bool match = false;
- for (auto& i : ifs) {
- if (strcmp(i.c_str(), t->ifa_name) == 0) {
- match = true;
- break;
- }
- }
- if (match) {
- struct ifaddrs *n = new ifaddrs;
- memcpy(n, t, sizeof(*t));
- n->ifa_next = head;
- head = n;
- }
- t = t->ifa_next;
- }
- if (!head) {
- lderr(cct) << "no interfaces matching " << ifs << dendl;
- exit(1);
- }
- filtered = head;
}
- struct sockaddr *r = nullptr;
- for (auto& s : nets) {
- struct sockaddr_storage net;
- unsigned int prefix_len;
-
- if (!parse_network(s.c_str(), &net, &prefix_len)) {
- lderr(cct) << "unable to parse network: " << s << dendl;
- exit(1);
+ int best_score = 0;
+ const sockaddr* best_addr = nullptr;
+ for (const auto* addr = ifa; addr != nullptr; addr = addr->ifa_next) {
+ if (!ifs.empty() &&
+ std::none_of(std::begin(ifs), std::end(ifs),
+ [&](const auto& if_name) {
+ return matches_with_name(*addr, if_name);
+ })) {
+ continue;
}
-
- switch (net.ss_family) {
- case AF_INET:
- if (!(ipv & CEPH_PICK_ADDRESS_IPV4)) {
- continue;
- }
- break;
- case AF_INET6:
- if (!(ipv & CEPH_PICK_ADDRESS_IPV6)) {
- continue;
- }
- break;
+ if (!nets.empty() &&
+ std::none_of(std::begin(nets), std::end(nets),
+ [&](const auto& net) {
+ return matches_with_net(cct, *addr, net, ipv);
+ })) {
+ continue;
}
-
- const struct ifaddrs *found = find_ip_in_subnet(
- filtered,
- (struct sockaddr *) &net, prefix_len, numa_node);
- if (found) {
- r = found->ifa_addr;
- break;
+ int score = grade_addr(*addr);
+ if (score < 0) {
+ continue;
}
- }
-
- if (filtered != ifa) {
- while (filtered) {
- struct ifaddrs *t = filtered->ifa_next;
- delete filtered;
- filtered = t;
+ score += grade_with_numa_node(*addr, numa_node);
+ if (score > best_score) {
+ best_score = score;
+ best_addr = addr->ifa_addr;
}
}
-
- return r;
+ return best_addr;
}
#ifndef WITH_SEASTAR
static void fill_in_one_address(CephContext *cct,
const struct ifaddrs *ifa,
- const string networks,
- const string interfaces,
+ const string &networks,
+ const string &interfaces,
const char *conf_var,
int numa_node = -1)
{
void pick_addresses(CephContext *cct, int needs)
{
- struct ifaddrs *ifa;
- int r = getifaddrs(&ifa);
auto public_addr = cct->_conf.get_val<entity_addr_t>("public_addr");
auto public_network = cct->_conf.get_val<std::string>("public_network");
auto public_network_interface =
auto cluster_network_interface =
cct->_conf.get_val<std::string>("cluster_network_interface");
+ struct ifaddrs *ifa;
+ int r = getifaddrs(&ifa);
if (r < 0) {
string err = cpp_strerror(errno);
lderr(cct) << "unable to fetch interfaces and addresses: " << err << dendl;
exit(1);
}
-
+ auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
if ((needs & CEPH_PICK_ADDRESS_PUBLIC) &&
public_addr.is_blank_ip() && !public_network.empty()) {
fill_in_one_address(cct, ifa, public_network, public_network_interface,
- "public_addr");
+ "public_addr");
}
if ((needs & CEPH_PICK_ADDRESS_CLUSTER) && cluster_addr.is_blank_ip()) {
if (!cluster_network.empty()) {
fill_in_one_address(cct, ifa, cluster_network, cluster_network_interface,
- "cluster_addr");
+ "cluster_addr");
} else {
if (!public_network.empty()) {
lderr(cct) << "Public network was set, but cluster network was not set " << dendl;
lderr(cct) << " Using public network also for cluster network" << dendl;
fill_in_one_address(cct, ifa, public_network, public_network_interface,
- "cluster_addr");
+ "cluster_addr");
}
}
}
-
- freeifaddrs(ifa);
}
#endif // !WITH_SEASTAR
CephContext *cct,
const struct ifaddrs *ifa,
unsigned ipv,
- const string networks,
- const string interfaces,
+ const string &networks,
+ const string &interfaces,
entity_addrvec_t *addrs,
int numa_node = -1)
{
- const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks,
- interfaces, numa_node);
+ const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv,
+ networks,
+ interfaces,
+ numa_node);
if (!found) {
std::string ip_type = "";
if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) {
!networks.empty()) {
int ipv4_r = !(ipv & CEPH_PICK_ADDRESS_IPV4) ? 0 : -1;
int ipv6_r = !(ipv & CEPH_PICK_ADDRESS_IPV6) ? 0 : -1;
- // first try on preferred numa node (if >= 0), then anywhere.
- while (true) {
- // note: pass in ipv to filter the matching addresses
- if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
- (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
- ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
- networks, interfaces, addrs,
- preferred_numa_node);
- }
- if (ipv & CEPH_PICK_ADDRESS_IPV6) {
- ipv6_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6,
- networks, interfaces, addrs,
- preferred_numa_node);
- }
- if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
- !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
- ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
- networks, interfaces, addrs,
- preferred_numa_node);
- }
- if (ipv4_r >= 0 && ipv6_r >= 0) {
- break;
- }
- if (preferred_numa_node < 0) {
- return ipv4_r >= 0 && ipv6_r >= 0 ? 0 : -1;
- }
- preferred_numa_node = -1; // try any numa node
+ // note: pass in ipv to filter the matching addresses
+ if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
+ (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
+ ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
+ networks, interfaces,
+ addrs,
+ preferred_numa_node);
+ }
+ if (ipv & CEPH_PICK_ADDRESS_IPV6) {
+ ipv6_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6,
+ networks, interfaces,
+ addrs,
+ preferred_numa_node);
+ }
+ if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
+ !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
+ ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
+ networks, interfaces,
+ addrs,
+ preferred_numa_node);
+ }
+ if (ipv4_r < 0 || ipv6_r < 0) {
+ return -1;
}
}
lderr(cct) << "unable to fetch interfaces and addresses: " << err << dendl;
return {};
}
-
- const unsigned int prefix_len = max(sizeof(in_addr::s_addr), sizeof(in6_addr::s6_addr)) * CHAR_BIT;
- const struct ifaddrs *found = find_ip_in_subnet(
- ifa,
- (const struct sockaddr *) &network, prefix_len);
-
- std::string result;
- if (found) {
- result = found->ifa_name;
+ auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
+ const unsigned int prefix_len = std::max(sizeof(in_addr::s_addr), sizeof(in6_addr::s6_addr)) * CHAR_BIT;
+ for (auto addr = ifa; addr != nullptr; addr = addr->ifa_next) {
+ if (matches_with_net(*ifa, (const struct sockaddr *) &network, prefix_len,
+ CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6)) {
+ return addr->ifa_name;
+ }
}
-
- freeifaddrs(ifa);
-
- return result;
+ return {};
}
lderr(cct) << "unable to fetch interfaces and addresses: " << cpp_strerror(errno) << dendl;
exit(1);
}
+ auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
- bool found = false;
for (struct ifaddrs *addrs = ifa; addrs != nullptr; addrs = addrs->ifa_next) {
if (addrs->ifa_addr) {
entity_addr_t a;
for (auto& p : ls) {
if (a.is_same_host(p)) {
*match = p;
- found = true;
- goto out;
+ return true;
}
}
}
}
-
- out:
- freeifaddrs(ifa);
- return found;
+ return false;
}
int get_iface_numa_node(
const std::string& iface,
int *node)
{
- int ifatype = IFACE_DEFAULT;
- string ifa = iface;
- int pos = ifa.find(":");
- if (pos != string::npos) {
- ifa.erase(pos);
+ enum class iface_t {
+ PHY_PORT,
+ BOND_PORT
+ } ifatype = iface_t::PHY_PORT;
+ string_view ifa{iface};
+ if (auto pos = ifa.find(":"); pos != ifa.npos) {
+ ifa.remove_suffix(ifa.size() - pos);
}
- string fn = std::string("/sys/class/net/") + ifa + "/device/numa_node";
+ string fn = fmt::format("/sys/class/net/{}/device/numa_node", ifa);
int fd = ::open(fn.c_str(), O_RDONLY);
if (fd < 0) {
- fn = std::string("/sys/class/net/") + ifa + "/bonding/slaves";
+ fn = fmt::format("/sys/class/net/{}/bonding/slaves", ifa);
fd = ::open(fn.c_str(), O_RDONLY);
if (fd < 0) {
return -errno;
}
- ifatype = IFACE_BOND_PORT;
- } else {
- ifatype = IFACE_PHY_PORT;
+ ifatype = iface_t::BOND_PORT;
}
int r = 0;
char buf[1024];
char *endptr = 0;
- int bond_node = -1;
r = safe_read(fd, &buf, sizeof(buf));
if (r < 0) {
goto out;
}
switch (ifatype) {
- case IFACE_PHY_PORT:
+ case iface_t::PHY_PORT:
*node = strtoll(buf, &endptr, 10);
if (endptr != buf + strlen(buf)) {
r = -EINVAL;
}
r = 0;
break;
- case IFACE_BOND_PORT:
+ case iface_t::BOND_PORT:
+ int bond_node = -1;
std::vector<std::string> sv;
- char *q, *p = strtok_r(buf, " ", &q);
- while (p != NULL) {
- sv.push_back(p);
- p = strtok_r(NULL, " ", &q);
- }
+ std::string ifacestr = buf;
+ get_str_vec(ifacestr, " ", sv);
for (auto& iter : sv) {
int bn = -1;
r = get_iface_numa_node(iter, &bn);
projects / ceph.git / blobdiff