PVE/Storage/DRBDPlugin.pm

   1 package PVE::Storage::DRBDPlugin;
   2
   3 use strict;
   4 use warnings;
   5 use IO::File;
   6 use Net::DBus;
   7 use Data::Dumper;
   8
   9 use PVE::Tools qw(run_command trim);
  10 use PVE::INotify;
  11 use PVE::Storage::Plugin;
  12 use PVE::JSONSchema qw(get_standard_option);
  13
  14 use base qw(PVE::Storage::Plugin);
  15
  16 # Configuration
  17
  18 my $default_redundancy = 2;
  19
  20 sub type {
  21     return 'drbd';
  22 }
  23
  24 sub plugindata {
  25     return {
  26         content => [ {images => 1}, { images => 1 }],
  27     };
  28 }
  29
  30 sub properties {
  31     return {
  32         redundancy => {
  33             description => "The redundancy count specifies the number of nodes to which the resource should be deployed. It must be at least 1 and at most the number of nodes in the cluster.",
  34             type => 'integer',
  35             minimum => 1,
  36             maximum => 16,
  37             default => $default_redundancy,
  38         },
  39     };
  40 }
  41
  42 sub options {
  43     return {
  44         redundancy => { optional => 1 },
  45         nodes => { optional => 1 },
  46         disable => { optional => 1 },
  47     };
  48 }
  49
  50 # helper
  51
  52 sub get_redundancy {
  53     my ($scfg) = @_;
  54
  55     return $scfg->{redundancy} || $default_redundancy;
  56 }
  57
  58 sub connect_drbdmanage_service {
  59
  60     my $bus = Net::DBus->system;
  61
  62     my $service = $bus->get_service("org.drbd.drbdmanaged");
  63
  64     my $hdl = $service->get_object("/interface", "org.drbd.drbdmanaged");
  65
  66     return $hdl;
  67 }
  68
  69 sub check_drbd_rc {
  70     my ($rc) = @_;
  71
  72     die "got undefined drbd rc\n" if !$rc;
  73
  74     my ($code, $msg, $details) = @$rc;
  75
  76     return undef if $code == 0;
  77
  78     $msg = "drbd error: got error code $code" if !$msg;
  79     chomp $msg;
  80
  81     # fixme: add error details?
  82     #print Dumper($details);
  83
  84     die "drbd error: $msg\n";
  85 }
  86
  87 sub drbd_list_volumes {
  88     my ($hdl) = @_;
  89
  90     $hdl = connect_drbdmanage_service() if !$hdl;
  91
  92     my ($rc, $res) = $hdl->list_volumes([], 0, {}, []);
  93     check_drbd_rc($rc->[0]);
  94
  95     my $volumes = {};
  96
  97     foreach my $entry (@$res) {
  98         my ($volname, $properties, $vol_list) = @$entry;
  99
 100         next if $volname !~ m/^vm-(\d+)-/;
 101         my $vmid = $1;
 102
 103         # fixme: we always use volid 0 ?
 104         my $size = 0;
 105         foreach my $volentry (@$vol_list) {
 106             my ($vol_id, $vol_properties) = @$volentry;
 107             next if $vol_id != 0;
 108             my $vol_size = $vol_properties->{vol_size} * 1024;
 109             $size = $vol_size if $vol_size > $size;
 110         }
 111
 112         $volumes->{$volname} = { format => 'raw', size => $size,
 113                                  vmid => $vmid };
 114     }
 115
 116     return $volumes;
 117 }
 118
 119 # Storage implementation
 120
 121 sub parse_volname {
 122     my ($class, $volname) = @_;
 123
 124     if ($volname =~ m/^(vm-(\d+)-[a-z][a-z0-9\-\_\.]*[a-z0-9]+)$/) {
 125         return ('images', $1, $2);
 126     }
 127
 128     die "unable to parse lvm volume name '$volname'\n";
 129 }
 130
 131 sub filesystem_path {
 132     my ($class, $scfg, $volname) = @_;
 133
 134     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 135
 136     # fixme: always use volid 0?
 137     my $path = "/dev/drbd/by-res/$volname/0";
 138
 139     return wantarray ? ($path, $vmid, $vtype) : $path;
 140 }
 141
 142 sub create_base {
 143     my ($class, $storeid, $scfg, $volname) = @_;
 144
 145     die "can't create base images in drbd storage\n";
 146 }
 147
 148 sub clone_image {
 149     my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
 150
 151     die "can't clone images in drbd storage\n";
 152 }
 153
 154 sub alloc_image {
 155     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 156
 157     die "unsupported format '$fmt'" if $fmt ne 'raw';
 158
 159     die "illegal name '$name' - should be 'vm-$vmid-*'\n"
 160         if defined($name) && $name !~ m/^vm-$vmid-/;
 161
 162     my $hdl = connect_drbdmanage_service();
 163     my $volumes = drbd_list_volumes($hdl);
 164
 165     die "volume '$name' already exists\n" if defined($name) && $volumes->{$name};
 166
 167     if (!defined($name)) {
 168         for (my $i = 1; $i < 100; $i++) {
 169             my $tn = "vm-$vmid-disk-$i";
 170             if (!defined ($volumes->{$tn})) {
 171                 $name = $tn;
 172                 last;
 173             }
 174         }
 175     }
 176
 177     die "unable to allocate an image name for VM $vmid in storage '$storeid'\n"
 178         if !defined($name);
 179
 180     my ($rc, $res) = $hdl->create_resource($name, {});
 181     check_drbd_rc($rc->[0]);
 182
 183     ($rc, $res) = $hdl->create_volume($name, $size, {});
 184     check_drbd_rc($rc->[0]);
 185
 186     my $redundancy = get_redundancy($scfg);;
 187
 188     ($rc, $res) = $hdl->auto_deploy($name, $redundancy, 0, 0);
 189     check_drbd_rc($rc->[0]);
 190
 191     # hack: this is required to avoid bugs with set_drbdsetup_props(),
 192     # which can result in a split brain if we do not wait.
 193     # Note: wait until all volumes have cstate:deploy
 194     my $max_wait = 15;
 195     for (my $i = 0; $i < $max_wait; $i++) {
 196         ($rc, $res) = $hdl->list_assignments([], [$name], 0, { "cstate:deploy" => "true" }, []);
 197         check_drbd_rc($rc->[0]);
 198         my $len = scalar(@$res);
 199         last if $len == $redundancy;
 200         sleep(1);
 201     }
 202     sleep(1); # another sleep - it does not work without -no idea why?
 203
 204     ($rc, $res) = $hdl->set_drbdsetup_props(
 205         {
 206             target => "resource",
 207             resource => $name,
 208             type => 'neto',
 209             'allow-two-primaries' => 'yes',
 210         });
 211     check_drbd_rc($rc->[0]);
 212
 213     return $name;
 214 }
 215
 216 sub free_image {
 217     my ($class, $storeid, $scfg, $volname, $isBase) = @_;
 218
 219     my $hdl = connect_drbdmanage_service();
 220     my ($rc, $res) = $hdl->remove_resource($volname, 0);
 221     check_drbd_rc($rc->[0]);
 222
 223     return undef;
 224 }
 225
 226 sub list_images {
 227     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 228
 229     my $vgname = $scfg->{vgname};
 230
 231     $cache->{drbd_volumes} = drbd_list_volumes() if !$cache->{drbd_volumes};
 232
 233     my $res = [];
 234
 235     my $dat =  $cache->{drbd_volumes};
 236
 237     foreach my $volname (keys %$dat) {
 238
 239         my $owner = $dat->{$volname}->{vmid};
 240
 241         my $volid = "$storeid:$volname";
 242
 243         if ($vollist) {
 244             my $found = grep { $_ eq $volid } @$vollist;
 245             next if !$found;
 246         } else {
 247             next if defined ($vmid) && ($owner ne $vmid);
 248         }
 249
 250         my $info = $dat->{$volname};
 251         $info->{volid} = $volid;
 252
 253         push @$res, $info;
 254     }
 255
 256     return $res;
 257 }
 258
 259 sub status {
 260     my ($class, $storeid, $scfg, $cache) = @_;
 261
 262     my ($total, $avail, $used);
 263
 264     eval {
 265         my $hdl = connect_drbdmanage_service();
 266         my $redundancy = get_redundancy($scfg);;
 267         my ($rc, $res) = $hdl->cluster_free_query($redundancy);
 268         check_drbd_rc($rc->[0]);
 269
 270         $avail = $res;
 271         $used = 0; # fixme
 272         $total = $used + $avail;
 273
 274     };
 275     if (my $err = $@) {
 276         # ignore error,
 277         # assume storage if offline
 278
 279         return undef;
 280     }
 281
 282     return ($total, $avail, $used, 1);
 283 }
 284
 285 sub activate_storage {
 286     my ($class, $storeid, $scfg, $cache) = @_;
 287
 288     return undef;
 289 }
 290
 291 sub deactivate_storage {
 292     my ($class, $storeid, $scfg, $cache) = @_;
 293
 294     return undef;
 295 }
 296
 297 sub activate_volume {
 298     my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
 299
 300     my $path = $class->path($scfg, $volname);
 301
 302     my $hdl = connect_drbdmanage_service();
 303     my $nodename = PVE::INotify::nodename();
 304     my ($rc, $res) = $hdl->list_assignments([$nodename], [], 0, {}, []);
 305     check_drbd_rc($rc->[0]);
 306
 307     foreach my $entry (@$res) {
 308         my ($node, $res_name, $props, $voldata) = @$entry;
 309         if (($node eq $nodename) && ($res_name eq $volname)) {
 310             return undef; # assignment already exists
 311         }
 312     }
 313
 314     # create diskless assignment
 315     ($rc, $res) = $hdl->assign($nodename, $volname, { diskless => 'true' });
 316     check_drbd_rc($rc->[0]);
 317
 318     # wait until device is acessitble
 319     my $print_warning = 1;
 320     my $max_wait_time = 20;
 321     for (my $i = 0;; $i++) {
 322         ($rc, $res) = $hdl->list_assignments([$nodename], [$volname], 0, { "cstate:deploy" => "true" }, []);
 323         check_drbd_rc($rc->[0]);
 324         my $len = scalar(@$res);
 325         last if $len > 0;
 326         die "aborting wait - device '$path' still not readable\n" if $i > $max_wait_time;
 327         print "waiting for device '$path' to become ready...\n" if $print_warning;
 328         $print_warning = 0;
 329         sleep(1);
 330     }
 331
 332     return undef;
 333 }
 334
 335 sub deactivate_volume {
 336     my ($class, $storeid, $scfg, $volname, $cache) = @_;
 337
 338     # fixme: remove diskless assdignments
 339
 340     return undef;
 341 }
 342
 343 sub volume_resize {
 344     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 345
 346     $size = ($size/1024/1024) . "M";
 347
 348     my $path = $class->path($scfg, $volname);
 349
 350     # fixme: howto implement this
 351     die "drbd volume_resize is not implemented";
 352
 353     #my $cmd = ['/sbin/lvextend', '-L', $size, $path];
 354     #run_command($cmd, errmsg => "error resizing volume '$path'");
 355
 356     return 1;
 357 }
 358
 359 sub volume_snapshot {
 360     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 361
 362     die "drbd snapshot is not implemented";
 363 }
 364
 365 sub volume_snapshot_rollback {
 366     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 367
 368     die "drbd snapshot rollback is not implemented";
 369 }
 370
 371 sub volume_snapshot_delete {
 372     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 373
 374     die "drbd snapshot delete is not implemented";
 375 }
 376
 377 sub volume_has_feature {
 378     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 379
 380     my $features = {
 381         copy => { base => 1, current => 1},
 382     };
 383
 384     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
 385         $class->parse_volname($volname);
 386
 387     my $key = undef;
 388     if($snapname){
 389         $key = 'snap';
 390     }else{
 391         $key =  $isBase ? 'base' : 'current';
 392     }
 393     return 1 if $features->{$feature}->{$key};
 394
 395     return undef;
 396 }
 397
 398 1;