Revert "Fix #2020: use /sys to map nvmeXnY to nvmeX"
[pve-storage.git] / PVE / CephConfig.pm
1 package PVE::CephConfig;
2
3 use strict;
4 use warnings;
5 use Net::IP;
6 use PVE::Tools qw(run_command);
7 use PVE::Cluster qw(cfs_register_file);
8
9 cfs_register_file('ceph.conf',
10 \&parse_ceph_config,
11 \&write_ceph_config);
12
13 sub parse_ceph_config {
14 my ($filename, $raw) = @_;
15
16 my $cfg = {};
17 return $cfg if !defined($raw);
18
19 my @lines = split /\n/, $raw;
20
21 my $section;
22
23 foreach my $line (@lines) {
24 $line =~ s/#.*$//;
25 $line =~ s/^\s+//;
26 $line =~ s/^;.*$//;
27 $line =~ s/\s+$//;
28 next if !$line;
29
30 $section = $1 if $line =~ m/^\[(\S+)\]$/;
31 if (!$section) {
32 warn "no section - skip: $line\n";
33 next;
34 }
35
36 if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
37 my ($key, $val) = ($1, $2);
38 # ceph treats ' ', '_' and '-' in keys the same, so lets do too
39 $key =~ s/[-\ ]/_/g;
40 $cfg->{$section}->{$key} = $val;
41 }
42
43 }
44
45 return $cfg;
46 }
47
48 my $parse_ceph_file = sub {
49 my ($filename) = @_;
50
51 my $cfg = {};
52
53 return $cfg if ! -f $filename;
54
55 my $content = PVE::Tools::file_get_contents($filename);
56
57 return parse_ceph_config($filename, $content);
58 };
59
60 sub write_ceph_config {
61 my ($filename, $cfg) = @_;
62
63 my $out = '';
64
65 my $cond_write_sec = sub {
66 my $re = shift;
67
68 foreach my $section (sort keys %$cfg) {
69 next if $section !~ m/^$re$/;
70 $out .= "[$section]\n";
71 foreach my $key (sort keys %{$cfg->{$section}}) {
72 $out .= "\t $key = $cfg->{$section}->{$key}\n";
73 }
74 $out .= "\n";
75 }
76 };
77
78 &$cond_write_sec('global');
79 &$cond_write_sec('client');
80
81 &$cond_write_sec('mds');
82 &$cond_write_sec('mon');
83 &$cond_write_sec('osd');
84 &$cond_write_sec('mgr');
85
86 &$cond_write_sec('mds\..*');
87 &$cond_write_sec('mon\..*');
88 &$cond_write_sec('osd\..*');
89 &$cond_write_sec('mgr\..*');
90
91 return $out;
92 }
93
94 my $ceph_get_key = sub {
95 my ($keyfile, $username) = @_;
96
97 my $key = $parse_ceph_file->($keyfile);
98 my $secret = $key->{"client.$username"}->{key};
99
100 return $secret;
101 };
102
103 my $get_host = sub {
104 my ($hostport) = @_;
105 my ($host, $port) = PVE::Tools::parse_host_and_port($hostport);
106 if (!defined($host)) {
107 return "";
108 }
109 $port = defined($port) ? ":$port" : '';
110 $host = "[$host]" if Net::IP::ip_is_ipv6($host);
111 return "${host}${port}";
112 };
113
114 sub get_monaddr_list {
115 my ($configfile) = shift;
116
117 if (!defined($configfile)) {
118 warn "No ceph config specified\n";
119 return;
120 }
121
122 my $config = $parse_ceph_file->($configfile);
123
124 my $monhostlist = {};
125
126 # get all ip addresses from mon_host
127 my $monhosts = [ split (/[ ,;]+/, $config->{global}->{mon_host} // "") ];
128
129 foreach my $monhost (@$monhosts) {
130 $monhost =~ s/^\[?v\d\://; # remove beginning of vector
131 $monhost =~ s|/\d+\]?||; # remove end of vector
132 my $host = $get_host->($monhost);
133 if ($host ne "") {
134 $monhostlist->{$host} = 1;
135 }
136 }
137
138 # then get all addrs from mon. sections
139 for my $section ( keys %$config ) {
140 next if $section !~ m/^mon\./;
141
142 if (my $addr = $config->{$section}->{mon_addr}) {
143 $monhostlist->{$addr} = 1;
144 }
145 }
146
147 return join(',', sort keys %$monhostlist);
148 }
149
150 sub hostlist {
151 my ($list_text, $separator) = @_;
152
153 my @monhostlist = PVE::Tools::split_list($list_text);
154 return join($separator, map { $get_host->($_) } @monhostlist);
155 }
156
157 my $ceph_check_keyfile = sub {
158 my ($filename, $type) = @_;
159
160 return if ! -f $filename;
161
162 my $content = PVE::Tools::file_get_contents($filename);
163 eval {
164 die if !$content;
165
166 if ($type eq 'rbd') {
167 die if $content !~ /\s*\[\S+\]\s*key\s*=\s*\S+==\s*$/m;
168 } elsif ($type eq 'cephfs') {
169 die if $content !~ /\S+==\s*$/;
170 }
171 };
172 die "Not a proper $type authentication file: $filename\n" if $@;
173
174 return undef;
175 };
176
177 sub ceph_connect_option {
178 my ($scfg, $storeid, %options) = @_;
179
180 my $cmd_option = {};
181 my $ceph_storeid_conf = "/etc/pve/priv/ceph/${storeid}.conf";
182 my $pveceph_config = '/etc/pve/ceph.conf';
183 my $keyfile = "/etc/pve/priv/ceph/${storeid}.keyring";
184 $keyfile = "/etc/pve/priv/ceph/${storeid}.secret" if ($scfg->{type} eq 'cephfs');
185 my $pveceph_managed = !defined($scfg->{monhost});
186
187 $cmd_option->{ceph_conf} = $pveceph_config if $pveceph_managed;
188
189 $ceph_check_keyfile->($keyfile, $scfg->{type});
190
191 if (-e $ceph_storeid_conf) {
192 if ($pveceph_managed) {
193 warn "ignoring custom ceph config for storage '$storeid', 'monhost' is not set (assuming pveceph managed cluster)!\n";
194 } else {
195 $cmd_option->{ceph_conf} = $ceph_storeid_conf;
196 }
197 }
198
199 $cmd_option->{keyring} = $keyfile if (-e $keyfile);
200 $cmd_option->{auth_supported} = (defined $cmd_option->{keyring}) ? 'cephx' : 'none';
201 $cmd_option->{userid} = $scfg->{username} ? $scfg->{username} : 'admin';
202 $cmd_option->{mon_host} = hostlist($scfg->{monhost}, ',') if (defined($scfg->{monhost}));
203
204 if (%options) {
205 foreach my $k (keys %options) {
206 $cmd_option->{$k} = $options{$k};
207 }
208 }
209
210 return $cmd_option;
211
212 }
213
214 sub ceph_create_keyfile {
215 my ($type, $storeid, $secret) = @_;
216
217 my $extension = 'keyring';
218 $extension = 'secret' if ($type eq 'cephfs');
219
220 my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring';
221 my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension";
222
223 die "ceph authx keyring file for storage '$storeid' already exists!\n"
224 if -e $ceph_storage_keyring && !defined($secret);
225
226 if (-e $ceph_admin_keyring || defined($secret)) {
227 eval {
228 if (defined($secret)) {
229 mkdir '/etc/pve/priv/ceph';
230 chomp $secret;
231 PVE::Tools::file_set_contents($ceph_storage_keyring, "${secret}\n", 0400);
232 } elsif ($type eq 'rbd') {
233 mkdir '/etc/pve/priv/ceph';
234 PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring);
235 } elsif ($type eq 'cephfs') {
236 my $cephfs_secret = $ceph_get_key->($ceph_admin_keyring, 'admin');
237 mkdir '/etc/pve/priv/ceph';
238 chomp $cephfs_secret;
239 PVE::Tools::file_set_contents($ceph_storage_keyring, "${cephfs_secret}\n", 0400);
240 }
241 };
242 if (my $err = $@) {
243 unlink $ceph_storage_keyring;
244 die "failed to copy ceph authx $extension for storage '$storeid': $err\n";
245 }
246 } else {
247 warn "$ceph_admin_keyring not found, authentication is disabled.\n";
248 }
249 }
250
251 sub ceph_remove_keyfile {
252 my ($type, $storeid) = @_;
253
254 my $extension = 'keyring';
255 $extension = 'secret' if ($type eq 'cephfs');
256 my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension";
257
258 if (-f $ceph_storage_keyring) {
259 unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n";
260 }
261 }
262
263 my $ceph_version_parser = sub {
264 my $ceph_version = shift;
265 # FIXME this is the same as pve-manager PVE::Ceph::Tools get_local_version
266 if ($ceph_version =~ /^ceph.*\sv?(\d+(?:\.\d+)+(?:-pve\d+)?)\s+(?:\(([a-zA-Z0-9]+)\))?/) {
267 my ($version, $buildcommit) = ($1, $2);
268 my $subversions = [ split(/\.|-/, $version) ];
269
270 return ($subversions, $version, $buildcommit);
271 }
272 warn "Could not parse Ceph version: '$ceph_version'\n";
273 };
274
275 sub local_ceph_version {
276 my ($cache) = @_;
277
278 my $version_string = $cache;
279 if (!defined($version_string)) {
280 run_command('ceph --version', outfunc => sub {
281 $version_string = shift;
282 });
283 }
284 return undef if !defined($version_string);
285 # subversion is an array ref. with the version parts from major to minor
286 # version is the filtered version string
287 my ($subversions, $version) = $ceph_version_parser->($version_string);
288
289 return wantarray ? ($subversions, $version) : $version;
290 }
291
292 1;