]>
Commit | Line | Data |
---|---|---|
6fb08cb9 | 1 | package PVE::Ceph::Tools; |
a34866f0 DM |
2 | |
3 | use strict; | |
4 | use warnings; | |
f8346b52 | 5 | |
a34866f0 | 6 | use File::Path; |
b436dca8 | 7 | use File::Basename; |
f8346b52 | 8 | use IO::File; |
48e8a06d | 9 | use JSON; |
a34866f0 | 10 | |
f8346b52 | 11 | use PVE::Tools qw(run_command dir_glob_foreach); |
48e8a06d | 12 | use PVE::Cluster qw(cfs_read_file); |
f96d7012 | 13 | use PVE::RADOS; |
91dfa228 AA |
14 | use PVE::Ceph::Services; |
15 | use PVE::CephConfig; | |
a34866f0 DM |
16 | |
17 | my $ccname = 'ceph'; # ceph cluster name | |
18 | my $ceph_cfgdir = "/etc/ceph"; | |
19 | my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf"; | |
20 | my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf"; | |
21 | ||
22 | my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring"; | |
23 | my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring"; | |
17709566 | 24 | my $ckeyring_path = "/etc/ceph/ceph.client.admin.keyring"; |
a34866f0 DM |
25 | my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring"; |
26 | my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring"; | |
b82649cc | 27 | my $ceph_mds_data_dir = '/var/lib/ceph/mds'; |
a34866f0 | 28 | |
c64c04dd AA |
29 | my $ceph_service = { |
30 | ceph_bin => "/usr/bin/ceph", | |
31 | ceph_mon => "/usr/bin/ceph-mon", | |
32 | ceph_mgr => "/usr/bin/ceph-mgr", | |
b82649cc TL |
33 | ceph_osd => "/usr/bin/ceph-osd", |
34 | ceph_mds => "/usr/bin/ceph-mds", | |
48e8a06d | 35 | ceph_volume => '/usr/sbin/ceph-volume', |
c64c04dd | 36 | }; |
a34866f0 DM |
37 | |
38 | my $config_hash = { | |
39 | ccname => $ccname, | |
40 | pve_ceph_cfgpath => $pve_ceph_cfgpath, | |
41 | pve_mon_key_path => $pve_mon_key_path, | |
42 | pve_ckeyring_path => $pve_ckeyring_path, | |
43 | ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring, | |
44 | ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring, | |
b82649cc | 45 | ceph_mds_data_dir => $ceph_mds_data_dir, |
7d4fc5ef | 46 | long_rados_timeout => 60, |
91dfa228 | 47 | ceph_cfgpath => $ceph_cfgpath, |
a34866f0 DM |
48 | }; |
49 | ||
c64c04dd AA |
50 | sub get_local_version { |
51 | my ($noerr) = @_; | |
52 | ||
6fb08cb9 | 53 | if (check_ceph_installed('ceph_bin', $noerr)) { |
c64c04dd | 54 | my $ceph_version; |
3248590d TL |
55 | run_command( |
56 | [ $ceph_service->{ceph_bin}, '--version' ], | |
57 | noerr => $noerr, | |
58 | outfunc => sub { $ceph_version = shift if !defined $ceph_version }, | |
59 | ); | |
60 | return undef if !defined $ceph_version; | |
61 | ||
2a080be1 | 62 | if ($ceph_version =~ /^ceph.*\sv?(\d+(?:\.\d+)+(?:-pve\d+)?)\s+(?:\(([a-zA-Z0-9]+)\))?/) { |
3248590d TL |
63 | my ($version, $buildcommit) = ($1, $2); |
64 | my $subversions = [ split(/\.|-/, $version) ]; | |
65 | ||
66 | # return (version, buildid, major, minor, ...) : major; | |
67 | return wantarray | |
68 | ? ($version, $buildcommit, $subversions) | |
69 | : $subversions->[0]; | |
c64c04dd AA |
70 | } |
71 | } | |
72 | ||
73 | return undef; | |
74 | } | |
75 | ||
8ba0d0a0 FG |
76 | sub get_cluster_versions { |
77 | my ($service, $noerr) = @_; | |
78 | ||
79 | my $rados = PVE::RADOS->new(); | |
80 | my $cmd = $service ? "$service versions" : 'versions'; | |
81 | return $rados->mon_command({ prefix => $cmd }); | |
82 | } | |
83 | ||
a34866f0 DM |
84 | sub get_config { |
85 | my $key = shift; | |
86 | ||
87 | my $value = $config_hash->{$key}; | |
88 | ||
6f8bf83d | 89 | die "no such ceph config '$key'" if !$value; |
a34866f0 DM |
90 | |
91 | return $value; | |
92 | } | |
93 | ||
a34866f0 | 94 | sub purge_all_ceph_files { |
91dfa228 AA |
95 | my ($services) = @_; |
96 | my $is_local_mon; | |
97 | my $monlist = [ split(',', PVE::CephConfig::get_monaddr_list($pve_ceph_cfgpath)) ]; | |
98 | ||
99 | foreach my $service (keys %$services) { | |
100 | my $type = $services->{$service}; | |
101 | next if (!%$type); | |
102 | ||
103 | foreach my $name (keys %$type) { | |
104 | my $dir_exists = $type->{$name}->{direxists}; | |
105 | ||
106 | $is_local_mon = grep($type->{$name}->{addr}, @$monlist) | |
107 | if $service eq 'mon'; | |
108 | ||
109 | my $path = "/var/lib/ceph/$service"; | |
110 | $path = '/var/log/ceph' if $service eq 'logs'; | |
111 | if ($dir_exists) { | |
112 | my $err; | |
113 | File::Path::remove_tree($path, { | |
114 | keep_root => 1, | |
115 | error => \$err, | |
116 | }); | |
117 | warn "Error removing path, '$path'\n" if @$err; | |
118 | } | |
119 | } | |
120 | } | |
a34866f0 | 121 | |
91dfa228 AA |
122 | if (scalar @$monlist > 0 && !$is_local_mon) { |
123 | warn "Foreign MON address in ceph.conf. Keeping config & keyrings\n" | |
124 | } else { | |
125 | print "Removing config & keyring files\n"; | |
126 | foreach my $file (%$config_hash) { | |
127 | unlink $file if (-e $file); | |
128 | } | |
129 | } | |
130 | } | |
a34866f0 | 131 | |
91dfa228 AA |
132 | sub purge_all_ceph_services { |
133 | my ($services) = @_; | |
134 | ||
135 | foreach my $service (keys %$services) { | |
136 | my $type = $services->{$service}; | |
137 | next if (!%$type); | |
138 | ||
139 | foreach my $name (keys %$type) { | |
140 | my $service_exists = $type->{$name}->{service}; | |
141 | ||
142 | if ($service_exists) { | |
75cac279 TL |
143 | eval { PVE::Ceph::Services::ceph_service_cmd('disable', "$service.$name") }; |
144 | warn "Could not disable ceph-$service\@$name, error: $@\n" if $@; | |
145 | ||
146 | eval { PVE::Ceph::Services::ceph_service_cmd('stop', "$service.$name") }; | |
147 | warn "Could not stop ceph-$service\@$name, error: $@\n" if $@; | |
91dfa228 AA |
148 | } |
149 | } | |
150 | } | |
a34866f0 DM |
151 | } |
152 | ||
153 | sub check_ceph_installed { | |
c64c04dd AA |
154 | my ($service, $noerr) = @_; |
155 | ||
156 | $service = 'ceph_bin' if !defined($service); | |
a34866f0 | 157 | |
c64c04dd AA |
158 | if (! -x $ceph_service->{$service}) { |
159 | die "binary not installed: $ceph_service->{$service}\n" if !$noerr; | |
a34866f0 DM |
160 | return undef; |
161 | } | |
162 | ||
163 | return 1; | |
164 | } | |
165 | ||
7ef69f33 TL |
166 | |
167 | sub check_ceph_configured { | |
168 | ||
169 | check_ceph_inited(); | |
170 | ||
171 | die "ceph not fully configured - missing '$pve_ckeyring_path'\n" | |
172 | if ! -f $pve_ckeyring_path; | |
173 | ||
174 | return 1; | |
175 | } | |
176 | ||
a34866f0 DM |
177 | sub check_ceph_inited { |
178 | my ($noerr) = @_; | |
179 | ||
315304f3 | 180 | return undef if !check_ceph_installed('ceph_mon', $noerr); |
6f8bf83d | 181 | |
a34866f0 DM |
182 | if (! -f $pve_ceph_cfgpath) { |
183 | die "pveceph configuration not initialized\n" if !$noerr; | |
184 | return undef; | |
185 | } | |
186 | ||
187 | return 1; | |
188 | } | |
189 | ||
190 | sub check_ceph_enabled { | |
191 | my ($noerr) = @_; | |
192 | ||
193 | return undef if !check_ceph_inited($noerr); | |
194 | ||
195 | if (! -f $ceph_cfgpath) { | |
196 | die "pveceph configuration not enabled\n" if !$noerr; | |
197 | return undef; | |
198 | } | |
199 | ||
200 | return 1; | |
201 | } | |
202 | ||
11d74274 AA |
203 | my $set_pool_setting = sub { |
204 | my ($pool, $setting, $value) = @_; | |
205 | ||
206 | my $command; | |
207 | if ($setting eq 'application') { | |
208 | $command = { | |
209 | prefix => "osd pool application enable", | |
210 | pool => "$pool", | |
211 | app => "$value", | |
212 | }; | |
213 | } else { | |
214 | $command = { | |
215 | prefix => "osd pool set", | |
216 | pool => "$pool", | |
217 | var => "$setting", | |
218 | val => "$value", | |
219 | format => 'plain', | |
220 | }; | |
221 | } | |
222 | ||
223 | my $rados = PVE::RADOS->new(); | |
224 | eval { $rados->mon_command($command); }; | |
225 | return $@ ? $@ : undef; | |
226 | }; | |
227 | ||
50adb131 AA |
228 | sub set_pool { |
229 | my ($pool, $param) = @_; | |
230 | ||
69c0ff3e | 231 | # by default, pool size always resets min_size, so set it as first item |
11d74274 | 232 | # https://tracker.ceph.com/issues/44862 |
69c0ff3e TL |
233 | my $keys = [ grep { $_ ne 'size' } sort keys %$param ]; |
234 | unshift @$keys, 'size' if exists $param->{size}; | |
11d74274 | 235 | |
69c0ff3e | 236 | for my $setting (@$keys) { |
11d74274 | 237 | my $value = $param->{$setting}; |
50adb131 | 238 | |
cb83113d | 239 | print "pool $pool: applying $setting = $value\n"; |
11d74274 AA |
240 | if (my $err = $set_pool_setting->($pool, $setting, $value)) { |
241 | print "$err"; | |
50adb131 AA |
242 | } else { |
243 | delete $param->{$setting}; | |
244 | } | |
245 | } | |
246 | ||
68f94af8 TL |
247 | if (scalar(keys %$param) > 0) { |
248 | my $missing = join(', ', sort keys %$param ); | |
249 | die "Could not set: $missing\n"; | |
50adb131 AA |
250 | } |
251 | ||
252 | } | |
253 | ||
f96d7012 TL |
254 | sub create_pool { |
255 | my ($pool, $param, $rados) = @_; | |
256 | ||
257 | if (!defined($rados)) { | |
258 | $rados = PVE::RADOS->new(); | |
259 | } | |
260 | ||
6ad70a2b | 261 | my $pg_num = $param->{pg_num} || 128; |
f96d7012 TL |
262 | |
263 | $rados->mon_command({ | |
264 | prefix => "osd pool create", | |
265 | pool => $pool, | |
266 | pg_num => int($pg_num), | |
267 | format => 'plain', | |
268 | }); | |
269 | ||
50adb131 | 270 | set_pool($pool, $param); |
f96d7012 TL |
271 | |
272 | } | |
273 | ||
7e1a9d25 TL |
274 | sub ls_pools { |
275 | my ($pool, $rados) = @_; | |
276 | ||
277 | if (!defined($rados)) { | |
278 | $rados = PVE::RADOS->new(); | |
279 | } | |
280 | ||
281 | my $res = $rados->mon_command({ prefix => "osd lspools" }); | |
282 | ||
283 | return $res; | |
284 | } | |
285 | ||
f96d7012 TL |
286 | sub destroy_pool { |
287 | my ($pool, $rados) = @_; | |
288 | ||
289 | if (!defined($rados)) { | |
290 | $rados = PVE::RADOS->new(); | |
291 | } | |
292 | ||
293 | # fixme: '--yes-i-really-really-mean-it' | |
294 | $rados->mon_command({ | |
295 | prefix => "osd pool delete", | |
296 | pool => $pool, | |
297 | pool2 => $pool, | |
f8eade23 | 298 | 'yes_i_really_really_mean_it' => JSON::true, |
f96d7012 TL |
299 | format => 'plain', |
300 | }); | |
301 | } | |
302 | ||
a34866f0 DM |
303 | sub setup_pve_symlinks { |
304 | # fail if we find a real file instead of a link | |
305 | if (-f $ceph_cfgpath) { | |
306 | my $lnk = readlink($ceph_cfgpath); | |
e881cf2a | 307 | die "file '$ceph_cfgpath' already exists and is not a symlink to $pve_ceph_cfgpath\n" |
a34866f0 DM |
308 | if !$lnk || $lnk ne $pve_ceph_cfgpath; |
309 | } else { | |
c5a673ed | 310 | mkdir $ceph_cfgdir; |
a34866f0 DM |
311 | symlink($pve_ceph_cfgpath, $ceph_cfgpath) || |
312 | die "unable to create symlink '$ceph_cfgpath' - $!\n"; | |
313 | } | |
c18db15b TL |
314 | my $ceph_uid = getpwnam('ceph'); |
315 | my $ceph_gid = getgrnam('ceph'); | |
316 | chown $ceph_uid, $ceph_gid, $ceph_cfgdir; | |
a34866f0 DM |
317 | } |
318 | ||
d558d296 DC |
319 | sub get_or_create_admin_keyring { |
320 | if (! -f $pve_ckeyring_path) { | |
321 | run_command("ceph-authtool --create-keyring $pve_ckeyring_path " . | |
322 | "--gen-key -n client.admin " . | |
323 | "--cap mon 'allow *' " . | |
324 | "--cap osd 'allow *' " . | |
325 | "--cap mds 'allow *' " . | |
326 | "--cap mgr 'allow *' "); | |
327 | # we do not want to overwrite it | |
328 | if (! -f $ckeyring_path) { | |
329 | run_command("cp $pve_ckeyring_path $ckeyring_path"); | |
ea60e3b7 | 330 | run_command("chown ceph:ceph $ckeyring_path"); |
d558d296 DC |
331 | } |
332 | } | |
333 | return $pve_ckeyring_path; | |
334 | } | |
335 | ||
48e8a06d DC |
336 | # get ceph-volume managed osds |
337 | sub ceph_volume_list { | |
338 | my $result = {}; | |
48e8a06d DC |
339 | |
340 | if (!check_ceph_installed('ceph_volume', 1)) { | |
341 | return $result; | |
342 | } | |
343 | ||
d79e9eb5 TL |
344 | my $output = ''; |
345 | my $cmd = [ $ceph_service->{ceph_volume}, 'lvm', 'list', '--format', 'json' ]; | |
346 | run_command($cmd, outfunc => sub { $output .= shift }); | |
48e8a06d DC |
347 | |
348 | $result = eval { decode_json($output) }; | |
349 | warn $@ if $@; | |
350 | return $result; | |
351 | } | |
352 | ||
353 | sub ceph_volume_zap { | |
354 | my ($osdid, $destroy) = @_; | |
355 | ||
356 | die "no osdid given\n" if !defined($osdid); | |
357 | ||
d79e9eb5 | 358 | my $cmd = [ $ceph_service->{ceph_volume}, 'lvm', 'zap', '--osd-id', $osdid ]; |
48e8a06d DC |
359 | push @$cmd, '--destroy' if $destroy; |
360 | ||
361 | run_command($cmd); | |
362 | } | |
363 | ||
d4e7f1bf DC |
364 | sub get_db_wal_sizes { |
365 | my $res = {}; | |
366 | ||
367 | my $rados = PVE::RADOS->new(); | |
368 | my $db_config = $rados->mon_command({ prefix => 'config-key dump', key => 'config/' }); | |
369 | ||
370 | $res->{db} = $db_config->{"config/osd/bluestore_block_db_size"} // | |
371 | $db_config->{"config/global/bluestore_block_db_size"}; | |
372 | ||
373 | $res->{wal} = $db_config->{"config/osd/bluestore_block_wal_size"} // | |
374 | $db_config->{"config/global/bluestore_block_wal_size"}; | |
375 | ||
376 | if (!$res->{db} || !$res->{wal}) { | |
377 | my $cfg = cfs_read_file('ceph.conf'); | |
378 | if (!$res->{db}) { | |
379 | $res->{db} = $cfg->{osd}->{bluestore_block_db_size} // | |
380 | $cfg->{global}->{bluestore_block_db_size}; | |
381 | } | |
382 | ||
383 | if (!$res->{wal}) { | |
384 | $res->{wal} = $cfg->{osd}->{bluestore_block_wal_size} // | |
385 | $cfg->{global}->{bluestore_block_wal_size}; | |
386 | } | |
387 | } | |
388 | ||
389 | return $res; | |
390 | } | |
735f24eb TL |
391 | sub get_possible_osd_flags { |
392 | my $possible_flags = { | |
393 | pause => { | |
394 | description => 'Pauses read and writes.', | |
395 | type => 'boolean', | |
396 | optional=> 1, | |
397 | }, | |
398 | noup => { | |
399 | description => 'OSDs are not allowed to start.', | |
400 | type => 'boolean', | |
401 | optional=> 1, | |
402 | }, | |
403 | nodown => { | |
404 | description => 'OSD failure reports are being ignored, such that the monitors will not mark OSDs down.', | |
405 | type => 'boolean', | |
406 | optional=> 1, | |
407 | }, | |
408 | noout => { | |
409 | description => 'OSDs will not automatically be marked out after the configured interval.', | |
410 | type => 'boolean', | |
411 | optional=> 1, | |
412 | }, | |
413 | noin => { | |
414 | description => 'OSDs that were previously marked out will not be marked back in when they start.', | |
415 | type => 'boolean', | |
416 | optional=> 1, | |
417 | }, | |
418 | nobackfill => { | |
419 | description => 'Backfilling of PGs is suspended.', | |
420 | type => 'boolean', | |
421 | optional=> 1, | |
422 | }, | |
423 | norebalance => { | |
424 | description => 'Rebalancing of PGs is suspended.', | |
425 | type => 'boolean', | |
426 | optional=> 1, | |
427 | }, | |
428 | norecover => { | |
429 | description => 'Recovery of PGs is suspended.', | |
430 | type => 'boolean', | |
431 | optional=> 1, | |
432 | }, | |
433 | noscrub => { | |
434 | description => 'Scrubbing is disabled.', | |
435 | type => 'boolean', | |
436 | optional=> 1, | |
437 | }, | |
438 | 'nodeep-scrub' => { | |
439 | description => 'Deep Scrubbing is disabled.', | |
440 | type => 'boolean', | |
441 | optional=> 1, | |
442 | }, | |
443 | notieragent => { | |
444 | description => 'Cache tiering activity is suspended.', | |
445 | type => 'boolean', | |
446 | optional=> 1, | |
447 | }, | |
448 | }; | |
449 | return $possible_flags; | |
450 | } | |
451 | ||
452 | sub get_real_flag_name { | |
453 | my ($flag) = @_; | |
454 | ||
455 | # the 'pause' flag gets always set to both 'pauserd' and 'pausewr' | |
456 | # so decide that the 'pause' flag is set if we detect 'pauserd' | |
457 | my $flagmap = { | |
458 | 'pause' => 'pauserd', | |
459 | }; | |
460 | ||
461 | return $flagmap->{$flag} // $flag; | |
462 | } | |
d4e7f1bf | 463 | |
e25dda25 AA |
464 | sub ceph_cluster_status { |
465 | my ($rados) = @_; | |
466 | $rados = PVE::RADOS->new() if !$rados; | |
467 | ||
e25dda25 | 468 | my $status = $rados->mon_command({ prefix => 'status' }); |
e25dda25 AA |
469 | $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' }); |
470 | ||
fdf79b4e | 471 | if (!exists $status->{monmap}->{mons}) { # octopus moved most info out of status, re-add |
e25dda25 AA |
472 | $status->{monmap} = $rados->mon_command({ prefix => 'mon dump' }); |
473 | $status->{mgrmap} = $rados->mon_command({ prefix => 'mgr dump' }); | |
474 | } | |
475 | ||
476 | return $status; | |
477 | } | |
478 | ||
a34866f0 | 479 | 1; |