]> git.proxmox.com Git - pve-storage.git/blob - PVE/Storage/RBDPlugin.pm
white space cleanups
[pve-storage.git] / PVE / Storage / RBDPlugin.pm
1 package PVE::Storage::RBDPlugin;
2
3 use strict;
4 use warnings;
5 use IO::File;
6 use Net::IP;
7 use PVE::Tools qw(run_command trim);
8 use PVE::Storage::Plugin;
9 use PVE::JSONSchema qw(get_standard_option);
10
11 use base qw(PVE::Storage::Plugin);
12
13 my $rbd_unittobytes = {
14 "M" => 1024*1024,
15 "G" => 1024*1024*1024,
16 "T" => 1024*1024*1024*1024,
17 };
18
19 my $add_pool_to_disk = sub {
20 my ($scfg, $disk) = @_;
21
22 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
23
24 return "$pool/$disk";
25 };
26
27 my $hostlist = sub {
28 my ($list_text, $separator) = @_;
29
30 my @monhostlist = PVE::Tools::split_list($list_text);
31 return join($separator, map {
32 my ($host, $port) = PVE::Tools::parse_host_and_port($_);
33 $port = defined($port) ? ":$port" : '';
34 $host = "[$host]" if Net::IP::ip_is_ipv6($host);
35 "${host}${port}"
36 } @monhostlist);
37 };
38
39 my $rbd_cmd = sub {
40 my ($scfg, $storeid, $op, @options) = @_;
41
42 my $monhost = &$hostlist($scfg->{monhost}, ',');
43
44 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
45 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
46 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
47
48 my $cmd = ['/usr/bin/rbd', '-p', $pool, '-m', $monhost];
49
50 if (-e $keyring) {
51 push @$cmd, '-n', "client.$username";
52 push @$cmd, '--keyring', $keyring;
53 push @$cmd, '--auth_supported', 'cephx';
54 } else {
55 push @$cmd, '--auth_supported', 'none';
56 }
57
58 my $cephconfig = "/etc/pve/priv/ceph/${storeid}.conf";
59
60 if (-e $cephconfig) {
61 push @$cmd, '-c', $cephconfig;
62 }
63
64 push @$cmd, $op;
65
66 push @$cmd, @options if scalar(@options);
67
68 return $cmd;
69 };
70
71 my $rados_cmd = sub {
72 my ($scfg, $storeid, $op, @options) = @_;
73
74 my $monhost = &$hostlist($scfg->{monhost}, ',');
75
76 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
77 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
78 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
79
80 my $cmd = ['/usr/bin/rados', '-p', $pool, '-m', $monhost];
81
82 if (-e $keyring) {
83 push @$cmd, '-n', "client.$username";
84 push @$cmd, '--keyring', $keyring;
85 push @$cmd, '--auth_supported', 'cephx';
86 } else {
87 push @$cmd, '--auth_supported', 'none';
88 }
89
90 my $cephconfig = "/etc/pve/priv/ceph/${storeid}.conf";
91
92 if (-e $cephconfig) {
93 push @$cmd, '-c', $cephconfig;
94 }
95
96 push @$cmd, $op;
97
98 push @$cmd, @options if scalar(@options);
99
100 return $cmd;
101 };
102
103 # needed for volumes created using ceph jewel (or higher)
104 my $krdb_feature_disable = sub {
105 my ($scfg, $storeid, $name) = @_;
106
107 return 1 if !$scfg->{krbd};
108
109 my ($major, undef, undef, undef) = ceph_version();
110 return 1 if $major < 10;
111
112 my $feature_cmd = &$rbd_cmd($scfg, $storeid, 'feature', 'disable', $name, 'deep-flatten,fast-diff,object-map,exclusive-lock');
113 run_rbd_command($feature_cmd, errmsg => "could not disable krbd-incompatible image features of rbd volume $name");
114 };
115
116 my $ceph_version_parser = sub {
117 my $line = shift;
118 if ($line =~ m/^ceph version ((\d+)\.(\d+)\.(\d+))(?: \([a-fA-F0-9]+\))?$/) {
119 return ($2, $3, $4, $1);
120 } else {
121 warn "Could not parse Ceph version: '$line'\n";
122 }
123 };
124
125 sub ceph_version {
126 my ($cache) = @_;
127
128 my $version_string = $cache;
129
130 my $major;
131 my $minor;
132 my $bugfix;
133
134 if (defined($version_string)) {
135 ($major, $minor, $bugfix, $version_string) = &$ceph_version_parser($version_string);
136 } else {
137 run_command('ceph --version', outfunc => sub {
138 my $line = shift;
139 ($major, $minor, $bugfix, $version_string) = &$ceph_version_parser($line);
140 });
141 }
142 return undef if !defined($version_string);
143 return wantarray ? ($major, $minor, $bugfix, $version_string) : $version_string;
144 }
145
146 sub run_rbd_command {
147 my ($cmd, %args) = @_;
148
149 my $lasterr;
150 my $errmsg = $args{errmsg} . ": " || "";
151 if (!exists($args{errfunc})) {
152 # ' error: 2014-02-06 11:51:59.839135 7f09f94d0760 -1 librbd: snap_unprotect: can't unprotect;
153 # at least 1 child(ren) in pool cephstor1
154 $args{errfunc} = sub {
155 my $line = shift;
156 if ($line =~ m/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ [0-9a-f]+ [\-\d]+ librbd: (.*)$/) {
157 $lasterr = "$1\n";
158 } else {
159 $lasterr = $line;
160 }
161 print STDERR $lasterr;
162 *STDERR->flush();
163 };
164 }
165
166 eval { run_command($cmd, %args); };
167 if (my $err = $@) {
168 die $errmsg . $lasterr if length($lasterr);
169 die $err;
170 }
171
172 return undef;
173 }
174
175 sub rbd_ls {
176 my ($scfg, $storeid) = @_;
177
178 my $cmd = &$rbd_cmd($scfg, $storeid, 'ls', '-l');
179 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
180
181 my $list = {};
182
183 my $parser = sub {
184 my $line = shift;
185
186 if ($line =~ m/^((vm|base)-(\d+)-disk-\d+)\s+(\d+)(M|G|T)\s((\S+)\/((vm|base)-\d+-\S+@\S+))?/) {
187 my ($image, $owner, $size, $unit, $parent) = ($1, $3, $4, $5, $8);
188
189 $list->{$pool}->{$image} = {
190 name => $image,
191 size => $size*$rbd_unittobytes->{$unit},
192 parent => $parent,
193 vmid => $owner
194 };
195 }
196 };
197
198 eval {
199 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
200 };
201 my $err = $@;
202
203 die $err if $err && $err !~ m/doesn't contain rbd images/ ;
204
205 return $list;
206 }
207
208 sub rbd_volume_info {
209 my ($scfg, $storeid, $volname, $snap) = @_;
210
211 my $cmd = undef;
212
213 if($snap){
214 $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname, '--snap', $snap);
215 }else{
216 $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname);
217 }
218
219 my $size = undef;
220 my $parent = undef;
221 my $format = undef;
222 my $protected = undef;
223
224 my $parser = sub {
225 my $line = shift;
226
227 if ($line =~ m/size (\d+) (M|G|T)B in (\d+) objects/) {
228 $size = $1 * $rbd_unittobytes->{$2} if ($1);
229 } elsif ($line =~ m/parent:\s(\S+)\/(\S+)/) {
230 $parent = $2;
231 } elsif ($line =~ m/format:\s(\d+)/) {
232 $format = $1;
233 } elsif ($line =~ m/protected:\s(\S+)/) {
234 $protected = 1 if $1 eq "True";
235 }
236
237 };
238
239 run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
240
241 return ($size, $parent, $format, $protected);
242 }
243
244 # Configuration
245
246 sub type {
247 return 'rbd';
248 }
249
250 sub plugindata {
251 return {
252 content => [ {images => 1, rootdir => 1}, { images => 1 }],
253 };
254 }
255
256 sub properties {
257 return {
258 monhost => {
259 description => "Monitors daemon ips.",
260 type => 'string', format => 'pve-storage-portal-dns-list',
261 },
262 pool => {
263 description => "Pool.",
264 type => 'string',
265 },
266 username => {
267 description => "RBD Id.",
268 type => 'string',
269 },
270 authsupported => {
271 description => "Authsupported.",
272 type => 'string',
273 },
274 krbd => {
275 description => "Access rbd through krbd kernel module.",
276 type => 'boolean',
277 },
278 };
279 }
280
281 sub options {
282 return {
283 nodes => { optional => 1 },
284 disable => { optional => 1 },
285 monhost => { fixed => 1 },
286 pool => { optional => 1 },
287 username => { optional => 1 },
288 content => { optional => 1 },
289 krbd => { optional => 1 },
290 };
291 }
292
293 # Storage implementation
294
295 sub parse_volname {
296 my ($class, $volname) = @_;
297
298 if ($volname =~ m/^((base-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
299 return ('images', $4, $7, $2, $3, $5, 'raw');
300 }
301
302 die "unable to parse rbd volume name '$volname'\n";
303 }
304
305 sub path {
306 my ($class, $scfg, $volname, $storeid, $snapname) = @_;
307
308 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
309 $name .= '@'.$snapname if $snapname;
310
311 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
312 return ("/dev/rbd/$pool/$name", $vmid, $vtype) if $scfg->{krbd};
313
314 my $monhost = &$hostlist($scfg->{monhost}, ';');
315 $monhost =~ s/:/\\:/g;
316
317 my $username = $scfg->{username} ? $scfg->{username} : 'admin';
318
319 my $path = "rbd:$pool/$name:mon_host=$monhost";
320 my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
321
322 if (-e $keyring) {
323 $path .= ":id=$username:auth_supported=cephx:keyring=$keyring";
324 } else {
325 $path .= ":auth_supported=none";
326 }
327
328 my $cephconfig = "/etc/pve/priv/ceph/${storeid}.conf";
329
330 if (-e $cephconfig) {
331 $path .= ":conf=$cephconfig";
332 }
333
334 return ($path, $vmid, $vtype);
335 }
336
337 my $find_free_diskname = sub {
338 my ($storeid, $scfg, $vmid) = @_;
339
340 my $rbd = rbd_ls($scfg, $storeid);
341 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
342 my $disk_ids = {};
343 my $dat = $rbd->{$pool};
344
345 foreach my $image (keys %$dat) {
346 my $volname = $dat->{$image}->{name};
347 if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
348 $disk_ids->{$2} = 1;
349 }
350 }
351 #fix: can we search in $rbd hash key with a regex to find (vm|base) ?
352 for (my $i = 1; $i < 100; $i++) {
353 if (!$disk_ids->{$i}) {
354 return "vm-$vmid-disk-$i";
355 }
356 }
357
358 die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
359 };
360
361 sub create_base {
362 my ($class, $storeid, $scfg, $volname) = @_;
363
364 my $snap = '__base__';
365
366 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
367 $class->parse_volname($volname);
368
369 die "create_base not possible with base image\n" if $isBase;
370
371 my ($size, $parent, $format, undef) = rbd_volume_info($scfg, $storeid, $name);
372 die "rbd volume info on '$name' failed\n" if !($size);
373
374 die "rbd image must be at format V2" if $format ne "2";
375
376 die "volname '$volname' contains wrong information about parent $parent $basename\n"
377 if $basename && (!$parent || $parent ne $basename."@".$snap);
378
379 my $newname = $name;
380 $newname =~ s/^vm-/base-/;
381
382 my $newvolname = $basename ? "$basename/$newname" : "$newname";
383
384 my $cmd = &$rbd_cmd($scfg, $storeid, 'rename', &$add_pool_to_disk($scfg, $name), &$add_pool_to_disk($scfg, $newname));
385 run_rbd_command($cmd, errmsg => "rbd rename '$name' error");
386
387 my $running = undef; #fixme : is create_base always offline ?
388
389 $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
390
391 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $newname, $snap);
392
393 if (!$protected){
394 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'protect', $newname, '--snap', $snap);
395 run_rbd_command($cmd, errmsg => "rbd protect $newname snap '$snap' error");
396 }
397
398 return $newvolname;
399
400 }
401
402 sub clone_image {
403 my ($class, $scfg, $storeid, $volname, $vmid, $snapname) = @_;
404
405 my $snap = '__base__';
406 $snap = $snapname if length $snapname;
407
408 my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
409 $class->parse_volname($volname);
410
411 die "$volname is not a base image and snapname is not provided\n"
412 if !$isBase && !length($snapname);
413
414 my $name = &$find_free_diskname($storeid, $scfg, $vmid);
415
416 warn "clone $volname: $basename snapname $snap to $name\n";
417
418 if (length($snapname)) {
419 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $volname, $snapname);
420
421 if (!$protected) {
422 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'protect', $volname, '--snap', $snapname);
423 run_rbd_command($cmd, errmsg => "rbd protect $volname snap $snapname error");
424 }
425 }
426
427 my $newvol = "$basename/$name";
428 $newvol = $name if length($snapname);
429
430 my $cmd = &$rbd_cmd($scfg, $storeid, 'clone', &$add_pool_to_disk($scfg, $basename),
431 '--snap', $snap, &$add_pool_to_disk($scfg, $name));
432
433 run_rbd_command($cmd, errmsg => "rbd clone '$basename' error");
434
435 &$krdb_feature_disable($scfg, $storeid, $name);
436
437 return $newvol;
438 }
439
440 sub alloc_image {
441 my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
442
443
444 die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
445 if $name && $name !~ m/^vm-$vmid-/;
446
447 $name = &$find_free_diskname($storeid, $scfg, $vmid) if !$name;
448
449 my $cmd = &$rbd_cmd($scfg, $storeid, 'create', '--image-format' , 2, '--size', int(($size+1023)/1024), $name);
450 run_rbd_command($cmd, errmsg => "rbd create $name' error");
451
452 &$krdb_feature_disable($scfg, $storeid, $name);
453
454 return $name;
455 }
456
457 sub free_image {
458 my ($class, $storeid, $scfg, $volname, $isBase) = @_;
459
460 my ($vtype, $name, $vmid, undef, undef, undef) =
461 $class->parse_volname($volname);
462
463 if ($isBase) {
464 my $snap = '__base__';
465 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $name, $snap);
466 if ($protected){
467 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'unprotect', $name, '--snap', $snap);
468 run_rbd_command($cmd, errmsg => "rbd unprotect $name snap '$snap' error");
469 }
470 }
471
472 $class->deactivate_volume($storeid, $scfg, $volname);
473
474 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'purge', $name);
475 run_rbd_command($cmd, errmsg => "rbd snap purge '$volname' error");
476
477 $cmd = &$rbd_cmd($scfg, $storeid, 'rm', $name);
478 run_rbd_command($cmd, errmsg => "rbd rm '$volname' error");
479
480 return undef;
481 }
482
483 sub list_images {
484 my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
485
486 $cache->{rbd} = rbd_ls($scfg, $storeid) if !$cache->{rbd};
487 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
488
489 my $res = [];
490
491 if (my $dat = $cache->{rbd}->{$pool}) {
492 foreach my $image (keys %$dat) {
493
494 my $volname = $dat->{$image}->{name};
495
496 my $volid = "$storeid:$volname";
497
498 my $owner = $dat->{$volname}->{vmid};
499 if ($vollist) {
500 my $found = grep { $_ eq $volid } @$vollist;
501 next if !$found;
502 } else {
503 next if defined ($vmid) && ($owner ne $vmid);
504 }
505
506 my $info = $dat->{$volname};
507 $info->{volid} = $volid;
508 $info->{format} = 'raw';
509
510 push @$res, $info;
511 }
512 }
513
514 return $res;
515 }
516
517 sub status {
518 my ($class, $storeid, $scfg, $cache) = @_;
519
520 my $cmd = &$rados_cmd($scfg, $storeid, 'df');
521
522 my $stats = {};
523
524 my $parser = sub {
525 my $line = shift;
526 if ($line =~ m/^\s+total\s(\S+)\s+(\d+)/) {
527 $stats->{$1} = $2;
528 }
529 };
530
531 eval {
532 run_rbd_command($cmd, errmsg => "rados error", errfunc => sub {}, outfunc => $parser);
533 };
534
535 my $total = $stats->{space} ? $stats->{space}*1024 : 0;
536 my $free = $stats->{avail} ? $stats->{avail}*1024 : 0;
537 my $used = $stats->{used} ? $stats->{used}*1024: 0;
538 my $active = 1;
539
540 return ($total, $free, $used, $active);
541 }
542
543 sub activate_storage {
544 my ($class, $storeid, $scfg, $cache) = @_;
545 return 1;
546 }
547
548 sub deactivate_storage {
549 my ($class, $storeid, $scfg, $cache) = @_;
550 return 1;
551 }
552
553 sub activate_volume {
554 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
555
556 return 1 if !$scfg->{krbd};
557
558 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
559 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
560
561 my $path = "/dev/rbd/$pool/$name";
562 $path .= '@'.$snapname if $snapname;
563 return if -b $path;
564
565 $name .= '@'.$snapname if $snapname;
566 my $cmd = &$rbd_cmd($scfg, $storeid, 'map', $name);
567 run_rbd_command($cmd, errmsg => "can't mount rbd volume $name");
568
569 return 1;
570 }
571
572 sub deactivate_volume {
573 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
574
575 return 1 if !$scfg->{krbd};
576
577 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
578 my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
579
580 my $path = "/dev/rbd/$pool/$name";
581 $path .= '@'.$snapname if $snapname;
582 return if ! -b $path;
583
584 my $cmd = &$rbd_cmd($scfg, $storeid, 'unmap', $path);
585 run_rbd_command($cmd, errmsg => "can't unmap rbd volume $name");
586
587 return 1;
588 }
589
590 sub volume_size_info {
591 my ($class, $scfg, $storeid, $volname, $timeout) = @_;
592
593 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
594 my ($size, undef) = rbd_volume_info($scfg, $storeid, $name);
595 return $size;
596 }
597
598 sub volume_resize {
599 my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
600
601 return 1 if $running;
602
603 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
604
605 my $cmd = &$rbd_cmd($scfg, $storeid, 'resize', '--size', ($size/1024/1024), $name);
606 run_rbd_command($cmd, errmsg => "rbd resize '$volname' error");
607 return undef;
608 }
609
610 sub volume_snapshot {
611 my ($class, $scfg, $storeid, $volname, $snap) = @_;
612
613 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
614
615 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'create', '--snap', $snap, $name);
616 run_rbd_command($cmd, errmsg => "rbd snapshot '$volname' error");
617 return undef;
618 }
619
620 sub volume_snapshot_rollback {
621 my ($class, $scfg, $storeid, $volname, $snap) = @_;
622
623 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
624
625 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'rollback', '--snap', $snap, $name);
626 run_rbd_command($cmd, errmsg => "rbd snapshot $volname to '$snap' error");
627 }
628
629 sub volume_snapshot_delete {
630 my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
631
632 return 1 if $running;
633
634 $class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
635
636 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
637
638 my (undef, undef, undef, $protected) = rbd_volume_info($scfg, $storeid, $name, $snap);
639 if ($protected){
640 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'unprotect', $name, '--snap', $snap);
641 run_rbd_command($cmd, errmsg => "rbd unprotect $name snap '$snap' error");
642 }
643
644 my $cmd = &$rbd_cmd($scfg, $storeid, 'snap', 'rm', '--snap', $snap, $name);
645
646 run_rbd_command($cmd, errmsg => "rbd snapshot '$volname' error");
647
648 return undef;
649 }
650
651 sub volume_has_feature {
652 my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
653
654 my $features = {
655 snapshot => { current => 1, snap => 1},
656 clone => { base => 1, snap => 1},
657 template => { current => 1},
658 copy => { base => 1, current => 1, snap => 1},
659 sparseinit => { base => 1, current => 1},
660 };
661
662 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
663 $class->parse_volname($volname);
664
665 my $key = undef;
666 if($snapname){
667 $key = 'snap';
668 }else{
669 $key = $isBase ? 'base' : 'current';
670 }
671 return 1 if $features->{$feature}->{$key};
672
673 return undef;
674 }
675
676 1;