]> git.proxmox.com Git - pve-storage.git/blob - PVE/Storage/RBDPlugin.pm
rbd: use pve-storage-portal-dns-list for monhost
[pve-storage.git] / PVE / Storage / RBDPlugin.pm
1 package PVE::Storage::RBDPlugin;
2
3 use strict;
4 use warnings;
5 use IO::File;
6 use Net::IP;
7 use PVE::Tools qw(run_command trim);
8 use PVE::Storage::Plugin;
9 use PVE::JSONSchema qw(get_standard_option);
10
11 use base qw(PVE::Storage::Plugin);
12
13 sub rbd_unittobytes {
14 {
15 "M" => 1024*1024,
16 "G" => 1024*1024*1024,
17 "T" => 1024*1024*1024*1024,
18 }
19 }
20
21 my $add_pool_to_disk = sub {
22 my ($scfg, $disk) = @_;
23
24 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
25
26 return "$pool/$disk";
27 };
28
29 my $hostlist = sub {
30 my ($list_text, $separator) = @_;
31 my @monhostlist = PVE::Tools::split_list($list_text);
32 return join($separator, map {
33 my ($host, $port) = PVE::Tools::parse_host_and_port($_);
34 $port = defined($port) ? ":$port" : '';
35 $host = "[$host]" if Net::IP::ip_is_ipv6($host);
36 "${host}${port}"
37 } @monhostlist);
38 };
39
40 my $rbd_cmd = sub {
41 my ($scfg, $storeid, $op, @options) = @_;
42
43 my $monhost = &$hostlist($scfg->{monhost}, ',');
44
45 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
46 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
47 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
48
49 my $cmd = ['/usr/bin/rbd', '-p', $pool, '-m', $monhost];
50
51 if(-e $keyring){
52 push @$cmd, '-n', "client.$username";
53 push @$cmd, '--keyring', $keyring;
54 push @$cmd, '--auth_supported', 'cephx';
55 }else{
56 push @$cmd, '--auth_supported', 'none';
57 }
58
59 push @$cmd, $op;
60
61 push @$cmd, @options if scalar(@options);
62
63 return $cmd;
64 };
65
66 my $rados_cmd = sub {
67 my ($scfg, $storeid, $op, @options) = @_;
68
69 my $monhost = &$hostlist($scfg->{monhost}, ',');
70
71 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
72 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
73 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
74
75 my $cmd = ['/usr/bin/rados', '-p', $pool, '-m', $monhost];
76
77 if(-e $keyring){
78 push @$cmd, '-n', "client.$username";
79 push @$cmd, '--keyring', $keyring;
80 push @$cmd, '--auth_supported', 'cephx';
81 }else{
82 push @$cmd, '--auth_supported', 'none';
83 }
84
85 push @$cmd, $op;
86
87 push @$cmd, @options if scalar(@options);
88
89 return $cmd;
90 };
91
92 # needed for volumes created using ceph jewel (or higher)
93 my $krdb_feature_disable = sub {
94 my ($scfg, $storeid, $name) = @_;
95
96 return 1 if !$scfg->{krbd};
97
98 my ($major, undef, undef, undef) = ceph_version();
99 return 1 if $major < 10;
100
101 my $feature_cmd = &$rbd_cmd($scfg, $storeid, 'feature', 'disable', $name, 'deep-flatten,fast-diff,object-map,exclusive-lock');
102 run_rbd_command($feature_cmd, errmsg => "could not disable krbd-incompatible image features of rbd volume $name");
103 };
104
105 my $ceph_version_parser = sub {
106 my $line = shift;
107 if ($line =~ m/^ceph version ((\d+)\.(\d+)\.(\d+))(?: \([a-fA-F0-9]+\))?$/) {
108 return ($2, $3, $4, $1);
109 } else {
110 warn "Could not parse Ceph version: '$line'\n";
111 }
112 };
113
114 sub ceph_version {
115 my ($cache) = @_;
116
117 my $version_string = $cache;
118
119 my $major;
120 my $minor;
121 my $bugfix;
122
123 if (defined($version_string)) {
124 ($major, $minor, $bugfix, $version_string) = &$ceph_version_parser($version_string);
125 } else {
126 run_command('ceph --version', outfunc => sub {
127 my $line = shift;
128 ($major, $minor, $bugfix, $version_string) = &$ceph_version_parser($line);
129 });
130 }
131 return undef if !defined($version_string);
132 return wantarray ? ($major, $minor, $bugfix, $version_string) : $version_string;
133 }
134
135 sub run_rbd_command {
136 my ($cmd, %args) = @_;
137
138 my $lasterr;
139 my $errmsg = $args{errmsg} . ": " || "";
140 if (!exists($args{errfunc})) {
141 # ' error: 2014-02-06 11:51:59.839135 7f09f94d0760 -1 librbd: snap_unprotect: can't unprotect;
142 # at least 1 child(ren) in pool cephstor1
143 $args{errfunc} = sub {
144 my $line = shift;
145 if ($line =~ m/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ [0-9a-f]+ [\-\d]+ librbd: (.*)$/) {
146 $lasterr = "$1\n";
147 } else {
148 $lasterr = $line;
149 }
150 print STDERR $lasterr;
151 *STDERR->flush();
152 };
153 }
154
155 eval { run_command($cmd, %args); };
156 if (my $err = $@) {
157 die $errmsg . $lasterr if length($lasterr);
158 die $err;
159 }
160
161 return undef;
162 }
163
164 sub rbd_ls {
165 my ($scfg, $storeid) = @_;
166
167 my $cmd = &$rbd_cmd($scfg, $storeid, 'ls', '-l');
168 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
169
170 my $list = {};
171
172 my $parser = sub {
173 my $line = shift;
174
175 if ($line =~ m/^((vm|base)-(\d+)-disk-\d+)\s+(\d+)(M|G|T)\s((\S+)\/((vm|base)-\d+-\S+@\S+))?/) {
176 my ($image, $owner, $size, $unit, $parent) = ($1, $3, $4, $5, $8);
177
178 $list->{$pool}->{$image} = {
179 name => $image,
180 size => $size*rbd_unittobytes()->{$unit},
181 parent => $parent,
182 vmid => $owner
183 };
184 }
185 };
186
187 eval {
188 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
189 };
190 my $err = $@;
191
192 die $err if $err && $err !~ m/doesn't contain rbd images/ ;
193
194 return $list;
195 }
196
197 sub rbd_volume_info {
198 my ($scfg, $storeid, $volname, $snap) = @_;
199
200 my $cmd = undef;
201
202 if($snap){
203 $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname, '--snap', $snap);
204 }else{
205 $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname);
206 }
207
208 my $size = undef;
209 my $parent = undef;
210 my $format = undef;
211 my $protected = undef;
212
213 my $parser = sub {
214 my $line = shift;
215
216 if ($line =~ m/size (\d+) (M|G|T)B in (\d+) objects/) {
217 $size = $1 * rbd_unittobytes()->{$2} if ($1);
218 } elsif ($line =~ m/parent:\s(\S+)\/(\S+)/) {
219 $parent = $2;
220 } elsif ($line =~ m/format:\s(\d+)/) {
221 $format = $1;
222 } elsif ($line =~ m/protected:\s(\S+)/) {
223 $protected = 1 if $1 eq "True";
224 }
225
226 };
227
228 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
229
230 return ($size, $parent, $format, $protected);
231 }
232
233 # Configuration
234
235 PVE::JSONSchema::register_format('pve-storage-monhost', \&parse_monhost);
236 sub parse_monhost {
237 my ($name, $noerr) = @_;
238
239 if ($name !~ m/^[a-z][a-z0-9\-\_\.]*[a-z0-9]$/i) {
240 return undef if $noerr;
241 die "lvm name '$name' contains illegal characters\n";
242 }
243
244 return $name;
245 }
246
247 sub type {
248 return 'rbd';
249 }
250
251 sub plugindata {
252 return {
253 content => [ {images => 1, rootdir => 1}, { images => 1 }],
254 };
255 }
256
257 sub properties {
258 return {
259 monhost => {
260 description => "Monitors daemon ips.",
261 type => 'string', format => 'pve-storage-portal-dns-list',
262 },
263 pool => {
264 description => "Pool.",
265 type => 'string',
266 },
267 username => {
268 description => "RBD Id.",
269 type => 'string',
270 },
271 authsupported => {
272 description => "Authsupported.",
273 type => 'string',
274 },
275 krbd => {
276 description => "Access rbd through krbd kernel module.",
277 type => 'boolean',
278 },
279 };
280 }
281
282 sub options {
283 return {
284 nodes => { optional => 1 },
285 disable => { optional => 1 },
286 monhost => { fixed => 1 },
287 pool => { optional => 1 },
288 username => { optional => 1 },
289 content => { optional => 1 },
290 krbd => { optional => 1 },
291 };
292 }
293
294 # Storage implementation
295
296 sub parse_volname {
297 my ($class, $volname) = @_;
298
299 if ($volname =~ m/^((base-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
300 return ('images', $4, $7, $2, $3, $5, 'raw');
301 }
302
303 die "unable to parse rbd volume name '$volname'\n";
304 }
305
306 sub path {
307 my ($class, $scfg, $volname, $storeid, $snapname) = @_;
308
309 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
310 $name .= '@'.$snapname if $snapname;
311
312 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
313 return ("/dev/rbd/$pool/$name", $vmid, $vtype) if $scfg->{krbd};
314
315 my $monhost = &$hostlist($scfg->{monhost}, ';');
316 $monhost =~ s/:/\\:/g;
317
318 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
319
320 my $path = "rbd:$pool/$name:mon_host=$monhost";
321 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
322
323 if(-e $keyring ){
324 $path .= ":id=$username:auth_supported=cephx:keyring=$keyring";
325 }else{
326 $path .= ":auth_supported=none";
327 }
328
329 return ($path, $vmid, $vtype);
330 }
331
332 my $find_free_diskname = sub {
333 my ($storeid, $scfg, $vmid) = @_;
334
335 my $rbd = rbd_ls($scfg, $storeid);
336 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
337 my $disk_ids = {};
338 my $dat = $rbd->{$pool};
339
340 foreach my $image (keys %$dat) {
341 my $volname = $dat->{$image}->{name};
342 if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
343 $disk_ids->{$2} = 1;
344 }
345 }
346 #fix: can we search in $rbd hash key with a regex to find (vm|base) ?
347 for (my $i = 1; $i < 100; $i++) {
348 if (!$disk_ids->{$i}) {
349 return "vm-$vmid-disk-$i";
350 }
351 }
352
353 die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
354 };
355
356 sub create_base {
357 my ($class, $storeid, $scfg, $volname) = @_;
358
359 my $snap = '__base__';
360
361 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
362 $class->parse_volname($volname);
363
364 die "create_base not possible with base image\n" if $isBase;
365
366 my ($size, $parent, $format, undef) = rbd_volume_info($scfg, $storeid, $name);
367 die "rbd volume info on '$name' failed\n" if !($size);
368
369 die "rbd image must be at format V2" if $format ne "2";
370
371 die "volname '$volname' contains wrong information about parent $parent $basename\n"
372 if $basename && (!$parent || $parent ne $basename."@".$snap);
373
374 my $newname = $name;
375 $newname =~ s/^vm-/base-/;
376
377 my $newvolname = $basename ? "$basename/$newname" : "$newname";
378
379 my $cmd = &$rbd_cmd($scfg, $storeid, 'rename', &$add_pool_to_disk($scfg, $name), &$add_pool_to_disk($scfg, $newname));
380 run_rbd_command($cmd, errmsg => "rbd rename '$name' error");
381
382 my $running = undef; #fixme : is create_base always offline ?
383
384 $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
385
386 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $newname, $snap);
387
388 if (!$protected){
389 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'protect', $newname, '--snap', $snap);
390 run_rbd_command($cmd, errmsg => "rbd protect $newname snap '$snap' error");
391 }
392
393 return $newvolname;
394
395 }
396
397 sub clone_image {
398 my ($class, $scfg, $storeid, $volname, $vmid, $snapname) = @_;
399
400 my $snap = '__base__';
401 $snap = $snapname if length $snapname;
402
403 my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
404 $class->parse_volname($volname);
405
406 die "$volname is not a base image and snapname is not provided\n"
407 if !$isBase && !length($snapname);
408
409 my $name = &$find_free_diskname($storeid, $scfg, $vmid);
410
411 warn "clone $volname: $basename snapname $snap to $name\n";
412
413 if (length($snapname)) {
414 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $volname, $snapname);
415
416 if (!$protected) {
417 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'protect', $volname, '--snap', $snapname);
418 run_rbd_command($cmd, errmsg => "rbd protect $volname snap $snapname error");
419 }
420 }
421
422 my $newvol = "$basename/$name";
423 $newvol = $name if length($snapname);
424
425 my $cmd = &$rbd_cmd($scfg, $storeid, 'clone', &$add_pool_to_disk($scfg, $basename),
426 '--snap', $snap, &$add_pool_to_disk($scfg, $name));
427
428 run_rbd_command($cmd, errmsg => "rbd clone '$basename' error");
429
430 &$krdb_feature_disable($scfg, $storeid, $name);
431
432 return $newvol;
433 }
434
435 sub alloc_image {
436 my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
437
438
439 die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
440 if $name && $name !~ m/^vm-$vmid-/;
441
442 $name = &$find_free_diskname($storeid, $scfg, $vmid) if !$name;
443
444 my $cmd = &$rbd_cmd($scfg, $storeid, 'create', '--image-format' , 2, '--size', int(($size+1023)/1024), $name);
445 run_rbd_command($cmd, errmsg => "rbd create $name' error");
446
447 &$krdb_feature_disable($scfg, $storeid, $name);
448
449 return $name;
450 }
451
452 sub free_image {
453 my ($class, $storeid, $scfg, $volname, $isBase) = @_;
454
455 my ($vtype, $name, $vmid, undef, undef, undef) =
456 $class->parse_volname($volname);
457
458 if ($isBase) {
459 my $snap = '__base__';
460 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $name, $snap);
461 if ($protected){
462 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'unprotect', $name, '--snap', $snap);
463 run_rbd_command($cmd, errmsg => "rbd unprotect $name snap '$snap' error");
464 }
465 }
466
467 $class->deactivate_volume($storeid, $scfg, $volname);
468
469 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'purge', $name);
470 run_rbd_command($cmd, errmsg => "rbd snap purge '$volname' error");
471
472 $cmd = &$rbd_cmd($scfg, $storeid, 'rm', $name);
473 run_rbd_command($cmd, errmsg => "rbd rm '$volname' error");
474
475 return undef;
476 }
477
478 sub list_images {
479 my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
480
481 $cache->{rbd} = rbd_ls($scfg, $storeid) if !$cache->{rbd};
482 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
483
484 my $res = [];
485
486 if (my $dat = $cache->{rbd}->{$pool}) {
487 foreach my $image (keys %$dat) {
488
489 my $volname = $dat->{$image}->{name};
490
491 my $volid = "$storeid:$volname";
492
493 my $owner = $dat->{$volname}->{vmid};
494 if ($vollist) {
495 my $found = grep { $_ eq $volid } @$vollist;
496 next if !$found;
497 } else {
498 next if defined ($vmid) && ($owner ne $vmid);
499 }
500
501 my $info = $dat->{$volname};
502 $info->{volid} = $volid;
503 $info->{format} = 'raw';
504
505 push @$res, $info;
506 }
507 }
508
509 return $res;
510 }
511
512 sub status {
513 my ($class, $storeid, $scfg, $cache) = @_;
514
515 my $cmd = &$rados_cmd($scfg, $storeid, 'df');
516
517 my $stats = {};
518
519 my $parser = sub {
520 my $line = shift;
521 if ($line =~ m/^\s+total\s(\S+)\s+(\d+)/) {
522 $stats->{$1} = $2;
523 }
524 };
525
526 eval {
527 run_rbd_command($cmd, errmsg => "rados error", errfunc => sub {}, outfunc => $parser);
528 };
529
530 my $total = $stats->{space} ? $stats->{space}*1024 : 0;
531 my $free = $stats->{avail} ? $stats->{avail}*1024 : 0;
532 my $used = $stats->{used} ? $stats->{used}*1024: 0;
533 my $active = 1;
534
535 return ($total, $free, $used, $active);
536 }
537
538 sub activate_storage {
539 my ($class, $storeid, $scfg, $cache) = @_;
540 return 1;
541 }
542
543 sub deactivate_storage {
544 my ($class, $storeid, $scfg, $cache) = @_;
545 return 1;
546 }
547
548 sub activate_volume {
549 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
550
551 return 1 if !$scfg->{krbd};
552
553 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
554 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
555
556 my $path = "/dev/rbd/$pool/$name";
557 $path .= '@'.$snapname if $snapname;
558 return if -b $path;
559
560 $name .= '@'.$snapname if $snapname;
561 my $cmd = &$rbd_cmd($scfg, $storeid, 'map', $name);
562 run_rbd_command($cmd, errmsg => "can't mount rbd volume $name");
563
564 return 1;
565 }
566
567 sub deactivate_volume {
568 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
569
570 return 1 if !$scfg->{krbd};
571
572 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
573 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
574
575 my $path = "/dev/rbd/$pool/$name";
576 $path .= '@'.$snapname if $snapname;
577 return if ! -b $path;
578
579 my $cmd = &$rbd_cmd($scfg, $storeid, 'unmap', $path);
580 run_rbd_command($cmd, errmsg => "can't unmap rbd volume $name");
581
582 return 1;
583 }
584
585 sub volume_size_info {
586 my ($class, $scfg, $storeid, $volname, $timeout) = @_;
587
588 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
589 my ($size, undef) = rbd_volume_info($scfg, $storeid, $name);
590 return $size;
591 }
592
593 sub volume_resize {
594 my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
595
596 return 1 if $running;
597
598 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
599
600 my $cmd = &$rbd_cmd($scfg, $storeid, 'resize', '--size', ($size/1024/1024), $name);
601 run_rbd_command($cmd, errmsg => "rbd resize '$volname' error");
602 return undef;
603 }
604
605 sub volume_snapshot {
606 my ($class, $scfg, $storeid, $volname, $snap) = @_;
607
608 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
609
610 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'create', '--snap', $snap, $name);
611 run_rbd_command($cmd, errmsg => "rbd snapshot '$volname' error");
612 return undef;
613 }
614
615 sub volume_snapshot_rollback {
616 my ($class, $scfg, $storeid, $volname, $snap) = @_;
617
618 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
619
620 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'rollback', '--snap', $snap, $name);
621 run_rbd_command($cmd, errmsg => "rbd snapshot $volname to '$snap' error");
622 }
623
624 sub volume_snapshot_delete {
625 my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
626
627 return 1 if $running;
628
629 $class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
630
631 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
632
633 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $name, $snap);
634 if ($protected){
635 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'unprotect', $name, '--snap', $snap);
636 run_rbd_command($cmd, errmsg => "rbd unprotect $name snap '$snap' error");
637 }
638
639 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'rm', '--snap', $snap, $name);
640
641 run_rbd_command($cmd, errmsg => "rbd snapshot '$volname' error");
642
643 return undef;
644 }
645
646 sub volume_has_feature {
647 my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
648
649 my $features = {
650 snapshot => { current => 1, snap => 1},
651 clone => { base => 1, snap => 1},
652 template => { current => 1},
653 copy => { base => 1, current => 1, snap => 1},
654 sparseinit => { base => 1, current => 1},
655 };
656
657 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
658 $class->parse_volname($volname);
659
660 my $key = undef;
661 if($snapname){
662 $key = 'snap';
663 }else{
664 $key = $isBase ? 'base' : 'current';
665 }
666 return 1 if $features->{$feature}->{$key};
667
668 return undef;
669 }
670
671 1;