]> git.proxmox.com Git - pve-storage.git/blob - PVE/Storage/RBDPlugin.pm
rbd: refactor cmdline helpers
[pve-storage.git] / PVE / Storage / RBDPlugin.pm
1 package PVE::Storage::RBDPlugin;
2
3 use strict;
4 use warnings;
5 use IO::File;
6 use Net::IP;
7 use PVE::Tools qw(run_command trim);
8 use PVE::Storage::Plugin;
9 use PVE::JSONSchema qw(get_standard_option);
10
11 use base qw(PVE::Storage::Plugin);
12
13 my $rbd_unittobytes = {
14 "k" => 1024,
15 "M" => 1024*1024,
16 "G" => 1024*1024*1024,
17 "T" => 1024*1024*1024*1024,
18 };
19
20 my $add_pool_to_disk = sub {
21 my ($scfg, $disk) = @_;
22
23 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
24
25 return "$pool/$disk";
26 };
27
28 my $hostlist = sub {
29 my ($list_text, $separator) = @_;
30
31 my @monhostlist = PVE::Tools::split_list($list_text);
32 return join($separator, map {
33 my ($host, $port) = PVE::Tools::parse_host_and_port($_);
34 $port = defined($port) ? ":$port" : '';
35 $host = "[$host]" if Net::IP::ip_is_ipv6($host);
36 "${host}${port}"
37 } @monhostlist);
38 };
39
40 my $build_cmd = sub {
41 my ($binary, $scfg, $storeid, $op, @options) = @_;
42
43 my $monhost = &$hostlist($scfg->{monhost}, ',');
44
45 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
46 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
47 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
48
49 my $cmd = [$binary, '-p', $pool, '-m', $monhost];
50
51 if (-e $keyring) {
52 push @$cmd, '-n', "client.$username";
53 push @$cmd, '--keyring', $keyring;
54 push @$cmd, '--auth_supported', 'cephx';
55 } else {
56 push @$cmd, '--auth_supported', 'none';
57 }
58
59 my $cephconfig = "/etc/pve/priv/ceph/${storeid}.conf";
60
61 if (-e $cephconfig) {
62 push @$cmd, '-c', $cephconfig;
63 }
64
65 push @$cmd, $op;
66
67 push @$cmd, @options if scalar(@options);
68
69 return $cmd;
70 };
71
72 my $rbd_cmd = sub {
73 my ($scfg, $storeid, $op, @options) = @_;
74
75 return $build_cmd->('/usr/bin/rbd', $scfg, $storeid, $op, @options);
76 };
77
78 my $rados_cmd = sub {
79 my ($scfg, $storeid, $op, @options) = @_;
80
81 return $build_cmd->('/usr/bin/rados', $scfg, $storeid, $op, @options);
82 };
83
84 # needed for volumes created using ceph jewel (or higher)
85 my $krdb_feature_disable = sub {
86 my ($scfg, $storeid, $name) = @_;
87
88 return 1 if !$scfg->{krbd};
89
90 my ($major, undef, undef, undef) = ceph_version();
91 return 1 if $major < 10;
92
93 my $feature_cmd = &$rbd_cmd($scfg, $storeid, 'feature', 'disable', $name, 'deep-flatten,fast-diff,object-map,exclusive-lock');
94 run_rbd_command($feature_cmd, errmsg => "could not disable krbd-incompatible image features of rbd volume $name");
95 };
96
97 my $ceph_version_parser = sub {
98 my $line = shift;
99 if ($line =~ m/^ceph version ((\d+)\.(\d+)\.(\d+))(?: \([a-fA-F0-9]+\))/) {
100 return ($2, $3, $4, $1);
101 } else {
102 warn "Could not parse Ceph version: '$line'\n";
103 }
104 };
105
106 sub ceph_version {
107 my ($cache) = @_;
108
109 my $version_string = $cache;
110
111 my $major;
112 my $minor;
113 my $bugfix;
114
115 if (defined($version_string)) {
116 ($major, $minor, $bugfix, $version_string) = &$ceph_version_parser($version_string);
117 } else {
118 run_command('ceph --version', outfunc => sub {
119 my $line = shift;
120 ($major, $minor, $bugfix, $version_string) = &$ceph_version_parser($line);
121 });
122 }
123 return undef if !defined($version_string);
124 return wantarray ? ($major, $minor, $bugfix, $version_string) : $version_string;
125 }
126
127 sub run_rbd_command {
128 my ($cmd, %args) = @_;
129
130 my $lasterr;
131 my $errmsg = $args{errmsg} . ": " || "";
132 if (!exists($args{errfunc})) {
133 # ' error: 2014-02-06 11:51:59.839135 7f09f94d0760 -1 librbd: snap_unprotect: can't unprotect;
134 # at least 1 child(ren) in pool cephstor1
135 $args{errfunc} = sub {
136 my $line = shift;
137 if ($line =~ m/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ [0-9a-f]+ [\-\d]+ librbd: (.*)$/) {
138 $lasterr = "$1\n";
139 } else {
140 $lasterr = $line;
141 }
142 print STDERR $lasterr;
143 *STDERR->flush();
144 };
145 }
146
147 eval { run_command($cmd, %args); };
148 if (my $err = $@) {
149 die $errmsg . $lasterr if length($lasterr);
150 die $err;
151 }
152
153 return undef;
154 }
155
156 sub rbd_ls {
157 my ($scfg, $storeid) = @_;
158
159 my $cmd = &$rbd_cmd($scfg, $storeid, 'ls', '-l');
160 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
161
162 my $list = {};
163
164 my $parser = sub {
165 my $line = shift;
166
167 if ($line =~ m/^((vm|base)-(\d+)-\S+)\s+(\d+)(k|M|G|T)\s((\S+)\/((vm|base)-\d+-\S+@\S+))?/) {
168 my ($image, $owner, $size, $unit, $parent) = ($1, $3, $4, $5, $8);
169 return if $image =~ /@/; #skip snapshots
170
171 $list->{$pool}->{$image} = {
172 name => $image,
173 size => $size*$rbd_unittobytes->{$unit},
174 parent => $parent,
175 vmid => $owner
176 };
177 }
178 };
179
180 eval {
181 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
182 };
183 my $err = $@;
184
185 die $err if $err && $err !~ m/doesn't contain rbd images/ ;
186
187 return $list;
188 }
189
190 sub rbd_volume_info {
191 my ($scfg, $storeid, $volname, $snap) = @_;
192
193 my $cmd = undef;
194
195 if($snap){
196 $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname, '--snap', $snap);
197 }else{
198 $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname);
199 }
200
201 my $size = undef;
202 my $parent = undef;
203 my $format = undef;
204 my $protected = undef;
205
206 my $parser = sub {
207 my $line = shift;
208
209 if ($line =~ m/size (\d+) (k|M|G|T)B in (\d+) objects/) {
210 $size = $1 * $rbd_unittobytes->{$2} if ($1);
211 } elsif ($line =~ m/parent:\s(\S+)\/(\S+)/) {
212 $parent = $2;
213 } elsif ($line =~ m/format:\s(\d+)/) {
214 $format = $1;
215 } elsif ($line =~ m/protected:\s(\S+)/) {
216 $protected = 1 if $1 eq "True";
217 }
218
219 };
220
221 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
222
223 return ($size, $parent, $format, $protected);
224 }
225
226 # Configuration
227
228 sub type {
229 return 'rbd';
230 }
231
232 sub plugindata {
233 return {
234 content => [ {images => 1, rootdir => 1}, { images => 1 }],
235 };
236 }
237
238 sub properties {
239 return {
240 monhost => {
241 description => "IP addresses of monitors (for external clusters).",
242 type => 'string', format => 'pve-storage-portal-dns-list',
243 },
244 pool => {
245 description => "Pool.",
246 type => 'string',
247 },
248 username => {
249 description => "RBD Id.",
250 type => 'string',
251 },
252 authsupported => {
253 description => "Authsupported.",
254 type => 'string',
255 },
256 krbd => {
257 description => "Access rbd through krbd kernel module.",
258 type => 'boolean',
259 },
260 };
261 }
262
263 sub options {
264 return {
265 nodes => { optional => 1 },
266 disable => { optional => 1 },
267 monhost => { optional => 1},
268 pool => { optional => 1 },
269 username => { optional => 1 },
270 content => { optional => 1 },
271 krbd => { optional => 1 },
272 };
273 }
274
275 # Storage implementation
276
277 sub parse_volname {
278 my ($class, $volname) = @_;
279
280 if ($volname =~ m/^((base-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
281 return ('images', $4, $7, $2, $3, $5, 'raw');
282 }
283
284 die "unable to parse rbd volume name '$volname'\n";
285 }
286
287 sub path {
288 my ($class, $scfg, $volname, $storeid, $snapname) = @_;
289
290 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
291 $name .= '@'.$snapname if $snapname;
292
293 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
294 return ("/dev/rbd/$pool/$name", $vmid, $vtype) if $scfg->{krbd};
295
296 my $monhost = &$hostlist($scfg->{monhost}, ';');
297 $monhost =~ s/:/\\:/g;
298
299 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
300
301 my $path = "rbd:$pool/$name:mon_host=$monhost";
302 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
303
304 if (-e $keyring) {
305 $path .= ":id=$username:auth_supported=cephx:keyring=$keyring";
306 } else {
307 $path .= ":auth_supported=none";
308 }
309
310 my $cephconfig = "/etc/pve/priv/ceph/${storeid}.conf";
311
312 if (-e $cephconfig) {
313 $path .= ":conf=$cephconfig";
314 }
315
316 return ($path, $vmid, $vtype);
317 }
318
319 my $find_free_diskname = sub {
320 my ($storeid, $scfg, $vmid) = @_;
321
322 my $cmd = &$rbd_cmd($scfg, $storeid, 'ls');
323 my $disk_ids = {};
324
325 my $parser = sub {
326 my $line = shift;
327
328 if ($line =~ m/^(vm|base)-\Q$vmid\E+-disk-(\d+)$/) {
329 $disk_ids->{$2} = 1;
330 }
331 };
332
333 eval {
334 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
335 };
336 my $err = $@;
337
338 die $err if $err && $err !~ m/doesn't contain rbd images/;
339
340 #fix: can we search in $rbd hash key with a regex to find (vm|base) ?
341 for (my $i = 1; $i < 100; $i++) {
342 if (!$disk_ids->{$i}) {
343 return "vm-$vmid-disk-$i";
344 }
345 }
346
347 die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
348 };
349
350 sub create_base {
351 my ($class, $storeid, $scfg, $volname) = @_;
352
353 my $snap = '__base__';
354
355 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
356 $class->parse_volname($volname);
357
358 die "create_base not possible with base image\n" if $isBase;
359
360 my ($size, $parent, $format, undef) = rbd_volume_info($scfg, $storeid, $name);
361 die "rbd volume info on '$name' failed\n" if !($size);
362
363 die "rbd image must be at format V2" if $format ne "2";
364
365 die "volname '$volname' contains wrong information about parent $parent $basename\n"
366 if $basename && (!$parent || $parent ne $basename."@".$snap);
367
368 my $newname = $name;
369 $newname =~ s/^vm-/base-/;
370
371 my $newvolname = $basename ? "$basename/$newname" : "$newname";
372
373 my $cmd = &$rbd_cmd($scfg, $storeid, 'rename', &$add_pool_to_disk($scfg, $name), &$add_pool_to_disk($scfg, $newname));
374 run_rbd_command($cmd, errmsg => "rbd rename '$name' error");
375
376 my $running = undef; #fixme : is create_base always offline ?
377
378 $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
379
380 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $newname, $snap);
381
382 if (!$protected){
383 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'protect', $newname, '--snap', $snap);
384 run_rbd_command($cmd, errmsg => "rbd protect $newname snap '$snap' error");
385 }
386
387 return $newvolname;
388
389 }
390
391 sub clone_image {
392 my ($class, $scfg, $storeid, $volname, $vmid, $snapname) = @_;
393
394 my $snap = '__base__';
395 $snap = $snapname if length $snapname;
396
397 my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
398 $class->parse_volname($volname);
399
400 die "$volname is not a base image and snapname is not provided\n"
401 if !$isBase && !length($snapname);
402
403 my $name = &$find_free_diskname($storeid, $scfg, $vmid);
404
405 warn "clone $volname: $basename snapname $snap to $name\n";
406
407 if (length($snapname)) {
408 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $volname, $snapname);
409
410 if (!$protected) {
411 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'protect', $volname, '--snap', $snapname);
412 run_rbd_command($cmd, errmsg => "rbd protect $volname snap $snapname error");
413 }
414 }
415
416 my $newvol = "$basename/$name";
417 $newvol = $name if length($snapname);
418
419 my $cmd = &$rbd_cmd($scfg, $storeid, 'clone', &$add_pool_to_disk($scfg, $basename),
420 '--snap', $snap, &$add_pool_to_disk($scfg, $name));
421
422 run_rbd_command($cmd, errmsg => "rbd clone '$basename' error");
423
424 &$krdb_feature_disable($scfg, $storeid, $name);
425
426 return $newvol;
427 }
428
429 sub alloc_image {
430 my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
431
432
433 die "illegal name '$name' - should be 'vm-$vmid-*'\n"
434 if $name && $name !~ m/^vm-$vmid-/;
435
436 $name = &$find_free_diskname($storeid, $scfg, $vmid) if !$name;
437
438 my $cmd = &$rbd_cmd($scfg, $storeid, 'create', '--image-format' , 2, '--size', int(($size+1023)/1024), $name);
439 run_rbd_command($cmd, errmsg => "rbd create $name' error");
440
441 &$krdb_feature_disable($scfg, $storeid, $name);
442
443 return $name;
444 }
445
446 sub free_image {
447 my ($class, $storeid, $scfg, $volname, $isBase) = @_;
448
449 my ($vtype, $name, $vmid, undef, undef, undef) =
450 $class->parse_volname($volname);
451
452 if ($isBase) {
453 my $snap = '__base__';
454 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $name, $snap);
455 if ($protected){
456 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'unprotect', $name, '--snap', $snap);
457 run_rbd_command($cmd, errmsg => "rbd unprotect $name snap '$snap' error");
458 }
459 }
460
461 $class->deactivate_volume($storeid, $scfg, $volname);
462
463 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'purge', $name);
464 run_rbd_command($cmd, errmsg => "rbd snap purge '$volname' error");
465
466 $cmd = &$rbd_cmd($scfg, $storeid, 'rm', $name);
467 run_rbd_command($cmd, errmsg => "rbd rm '$volname' error");
468
469 return undef;
470 }
471
472 sub list_images {
473 my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
474
475 $cache->{rbd} = rbd_ls($scfg, $storeid) if !$cache->{rbd};
476 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
477
478 my $res = [];
479
480 if (my $dat = $cache->{rbd}->{$pool}) {
481 foreach my $image (keys %$dat) {
482
483 my $info = $dat->{$image};
484
485 my $volname = $info->{name};
486 my $parent = $info->{parent};
487 my $owner = $info->{vmid};
488
489 if ($parent && $parent =~ m/^(base-\d+-\S+)\@__base__$/) {
490 $info->{volid} = "$storeid:$1/$volname";
491 } else {
492 $info->{volid} = "$storeid:$volname";
493 }
494
495 if ($vollist) {
496 my $found = grep { $_ eq $info->{volid} } @$vollist;
497 next if !$found;
498 } else {
499 next if defined ($vmid) && ($owner ne $vmid);
500 }
501
502 $info->{format} = 'raw';
503
504 push @$res, $info;
505 }
506 }
507
508 return $res;
509 }
510
511 sub status {
512 my ($class, $storeid, $scfg, $cache) = @_;
513
514 my $cmd = &$rados_cmd($scfg, $storeid, 'df');
515
516 my $stats = {};
517
518 my $parser = sub {
519 my $line = shift;
520 if ($line =~ m/^\s*total(?:\s|_)(\S+)\s+(\d+)(k|M|G|T)?/) {
521 $stats->{$1} = $2;
522 # luminous has units here..
523 if ($3) {
524 $stats->{$1} *= $rbd_unittobytes->{$3}/1024;
525 }
526 }
527 };
528
529 eval {
530 run_rbd_command($cmd, errmsg => "rados error", errfunc => sub {}, outfunc => $parser);
531 };
532
533 my $total = $stats->{space} ? $stats->{space}*1024 : 0;
534 my $free = $stats->{avail} ? $stats->{avail}*1024 : 0;
535 my $used = $stats->{used} ? $stats->{used}*1024: 0;
536 my $active = 1;
537
538 return ($total, $free, $used, $active);
539 }
540
541 sub activate_storage {
542 my ($class, $storeid, $scfg, $cache) = @_;
543 return 1;
544 }
545
546 sub deactivate_storage {
547 my ($class, $storeid, $scfg, $cache) = @_;
548 return 1;
549 }
550
551 sub activate_volume {
552 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
553
554 return 1 if !$scfg->{krbd};
555
556 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
557 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
558
559 my $path = "/dev/rbd/$pool/$name";
560 $path .= '@'.$snapname if $snapname;
561 return if -b $path;
562
563 $name .= '@'.$snapname if $snapname;
564 my $cmd = &$rbd_cmd($scfg, $storeid, 'map', $name);
565 run_rbd_command($cmd, errmsg => "can't mount rbd volume $name");
566
567 return 1;
568 }
569
570 sub deactivate_volume {
571 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
572
573 return 1 if !$scfg->{krbd};
574
575 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
576 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
577
578 my $path = "/dev/rbd/$pool/$name";
579 $path .= '@'.$snapname if $snapname;
580 return if ! -b $path;
581
582 my $cmd = &$rbd_cmd($scfg, $storeid, 'unmap', $path);
583 run_rbd_command($cmd, errmsg => "can't unmap rbd volume $name");
584
585 return 1;
586 }
587
588 sub volume_size_info {
589 my ($class, $scfg, $storeid, $volname, $timeout) = @_;
590
591 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
592 my ($size, undef) = rbd_volume_info($scfg, $storeid, $name);
593 return $size;
594 }
595
596 sub volume_resize {
597 my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
598
599 return 1 if $running && !$scfg->{krbd};
600
601 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
602
603 my $cmd = &$rbd_cmd($scfg, $storeid, 'resize', '--allow-shrink', '--size', ($size/1024/1024), $name);
604 run_rbd_command($cmd, errmsg => "rbd resize '$volname' error");
605 return undef;
606 }
607
608 sub volume_snapshot {
609 my ($class, $scfg, $storeid, $volname, $snap) = @_;
610
611 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
612
613 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'create', '--snap', $snap, $name);
614 run_rbd_command($cmd, errmsg => "rbd snapshot '$volname' error");
615 return undef;
616 }
617
618 sub volume_snapshot_rollback {
619 my ($class, $scfg, $storeid, $volname, $snap) = @_;
620
621 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
622
623 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'rollback', '--snap', $snap, $name);
624 run_rbd_command($cmd, errmsg => "rbd snapshot $volname to '$snap' error");
625 }
626
627 sub volume_snapshot_delete {
628 my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
629
630 return 1 if $running && !$scfg->{krbd};
631
632 $class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
633
634 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
635
636 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $name, $snap);
637 if ($protected){
638 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'unprotect', $name, '--snap', $snap);
639 run_rbd_command($cmd, errmsg => "rbd unprotect $name snap '$snap' error");
640 }
641
642 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'rm', '--snap', $snap, $name);
643
644 run_rbd_command($cmd, errmsg => "rbd snapshot '$volname' error");
645
646 return undef;
647 }
648
649 sub volume_has_feature {
650 my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
651
652 my $features = {
653 snapshot => { current => 1, snap => 1},
654 clone => { base => 1, snap => 1},
655 template => { current => 1},
656 copy => { base => 1, current => 1, snap => 1},
657 sparseinit => { base => 1, current => 1},
658 };
659
660 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
661 $class->parse_volname($volname);
662
663 my $key = undef;
664 if($snapname){
665 $key = 'snap';
666 }else{
667 $key = $isBase ? 'base' : 'current';
668 }
669 return 1 if $features->{$feature}->{$key};
670
671 return undef;
672 }
673
674 1;