test/rbd_namespace.pl

   1 #!/usr/bin/perl
   2
   3 # This script is meant to be run manually on hyperconverged PVE server with a
   4 # Ceph cluster. It tests how PVE handles RBD namespaces.
   5 #
   6 # The pool (default: rbd) must already exist. The namespace and VMs will be
   7 # created.
   8 #
   9 # Parameters like names for the pool an namespace and the VMID can be
  10 # configured.  The VMIDs for the clones is $vmid -1 and $vmid -2.
  11 #
  12 # Cleanup is done after a successful run. Cleanup can also be called manually.
  13 #
  14 # Known issues:
  15 #
  16 # * Snapshot rollback can sometimes be racy with stopping the VM and Ceph
  17 #  recognizing that the disk image is not in use anymore.
  18
  19 use strict;
  20 use warnings;
  21
  22 use Test::More;
  23 use Getopt::Long;
  24 use JSON;
  25
  26 use PVE::Tools qw(run_command);
  27
  28 my $pool = "testpool";
  29 my $use_existing= undef;
  30 my $namespace = "testspace";
  31 my $showhelp = '';
  32 my $vmid = 999999;
  33 my $cleanup = undef;
  34 my $DEBUG = 0;
  35
  36 my $helpstring = "To override default values, set them as named parameters:
  37
  38 --pool          pool name, default: ${pool}
  39 --use-existing  use existing pool, default: 0, needs --pool set
  40 --namespace     rbd namespace, default: ${namespace}
  41 --vmid          VMID of the test VM, default: ${vmid}
  42 --cleanup       Remove the storage definitions, namespaces and VMs
  43 --debug         Enable debug output\n";
  44
  45 GetOptions (
  46         "pool=s" => \$pool,
  47         "use-existing" => \$use_existing,
  48         "namespace=s" => \$namespace,
  49         "vmid=i" => \$vmid,
  50         "h|help" => \$showhelp,
  51         "cleanup" => \$cleanup,
  52         "debug" => \$DEBUG,
  53 ) or die ($helpstring);
  54
  55 if ($showhelp) {
  56     warn $helpstring;
  57     exit(0);
  58 }
  59
  60 my $storage_name = "${pool}-${namespace}";
  61
  62 my $vmid_clone = int($vmid) - 1;
  63 my $vmid_linked_clone = int($vmid) - 2;
  64
  65 sub jp {
  66     print to_json($_[0], { utf8 => 8, pretty => 1, canonical => 1 }) . "\n";
  67 }
  68 sub dbgvar {
  69     jp(@_) if $DEBUG;
  70 }
  71
  72 sub run_cmd {
  73     my ($cmd, $json, $ignore_errors) = @_;
  74
  75     my $raw = '';
  76     my $parser = sub {$raw .= shift;};
  77
  78     eval {
  79         run_command($cmd, outfunc => $parser);
  80     };
  81     if (my $err = $@) {
  82         die $err if !$ignore_errors;
  83     }
  84
  85     if ($json) {
  86         my $result;
  87         if ($raw eq '') {
  88             $result = [];
  89         } elsif ($raw =~ m/^(\[.*\])$/s) { # untaint
  90             $result = JSON::decode_json($1);
  91         } else {
  92             die "got unexpected data from command: '$cmd' -> '$raw'\n";
  93         }
  94         return $result;
  95         }
  96     return $raw;
  97 }
  98
  99 sub run_test_cmd {
 100     my ($cmd) = @_;
 101
 102     my $raw = '';
 103     my $out = sub {
 104         my $line = shift;
 105         $raw .= "${line}\n";
 106     };
 107
 108     eval {
 109         run_command($cmd, outfunc => $out);
 110     };
 111     if (my $err = $@) {
 112         print $raw;
 113         print $err;
 114         return 0;
 115     }
 116     print $raw;
 117     return 1;
 118 }
 119
 120 sub prepare {
 121     print "Preparing test environent\n";
 122
 123     my $pools = run_cmd("ceph osd pool ls --format json", 1);
 124
 125     my %poolnames = map {$_ => 1} @$pools;
 126     die "Pool '$pool' does not exist!\n"
 127         if !exists($poolnames{$pool}) && $use_existing;
 128
 129     run_cmd(['pveceph', 'pool', 'create', ${pool}, '--add_storages', 1])
 130         if !$use_existing;
 131
 132     my $namespaces = run_cmd(['rbd', '-p', ${pool}, 'namespace', 'ls', '--format', 'json'], 1);
 133     dbgvar($namespace);
 134     my $ns_found = 0;
 135     for my $i (@$namespaces) {
 136         $ns_found = 1 if $i->{name} eq $namespace;
 137     }
 138
 139     if (!$ns_found) {
 140         print "Create namespace '${namespace}' in pool '${pool}'\n";
 141         run_cmd(['rbd', 'namespace', 'create', "${pool}/${namespace}"]);
 142     }
 143
 144     my $storages = run_cmd(['pvesh', 'get', 'storage', '--output-format', 'json'], 1);
 145     dbgvar($storages);
 146     my $rbd_found = 0;
 147     my $pool_found = 0;
 148
 149     print "Create storage definition\n";
 150     for my $stor (@$storages) {
 151         $pool_found = 1 if $stor->{storage} eq $pool;
 152         $rbd_found = 1 if $stor->{storage} eq $storage_name;
 153
 154         if ($rbd_found) {
 155             run_cmd(['pvesm', 'set', ${storage_name}, '--krbd', '0']);
 156             die "Enable the storage '$stor->{storage}'!" if $stor->{disable};
 157         }
 158     }
 159     if (!$pool_found) {
 160         die "No storage for pool '${pool}' found! Must have same name as pool!\n"
 161             if $use_existing;
 162
 163         run_cmd(['pvesm', 'add', 'rbd', $pool, '--pool', $pool, '--content', 'images,rootdir']);
 164     }
 165     # create PVE storages (librbd / krbd)
 166     run_cmd(['pvesm', 'add', 'rbd', ${storage_name}, '--krbd', '0', '--pool', ${pool}, '--namespace', ${namespace}, '--content', 'images,rootdir'])
 167         if !$rbd_found;
 168
 169
 170     # create test VM
 171     print "Create test VM ${vmid}\n";
 172     my $vms = run_cmd(['pvesh', 'get', 'cluster/resources', '--type', 'vm', '--output-format', 'json'], 1);
 173     for my $vm (@$vms) {
 174         # TODO: introduce a force flag to make this behaviour configurable
 175
 176         if ($vm->{vmid} eq $vmid) {
 177             print "Test VM '${vmid}' already exists. It will be removed and recreated!\n";
 178             run_cmd(['qm', 'stop', ${vmid}], 0, 1);
 179             run_cmd(['qm', 'destroy', ${vmid}]);
 180         }
 181     }
 182     run_cmd(['qm', 'create', ${vmid}, '--bios', 'ovmf', '--efidisk0', "${storage_name}:1", '--scsi0', "${storage_name}:2"]);
 183 }
 184
 185
 186 sub cleanup {
 187     print "Cleaning up test environment!\n";
 188     print "Removing VMs\n";
 189     run_cmd(['qm', 'stop', ${vmid}], 0, 1);
 190     run_cmd(['qm', 'stop', ${vmid_linked_clone}], 0, 1);
 191     run_cmd(['qm', 'stop', ${vmid_clone}], 0, 1);
 192     run_cmd(['qm', 'destroy', ${vmid_linked_clone}], 0, 1);
 193     run_cmd(['qm', 'destroy', ${vmid_clone}], 0, 1);
 194     run_cmd(['for', 'i', 'in', "/dev/rbd/${pool}/${namespace}/*;", 'do', '/usr/bin/rbd', 'unmap', '\$i;', 'done'], 0, 1);
 195     run_cmd(['qm', 'unlock', ${vmid}], 0, 1);
 196     run_cmd(['qm', 'destroy', ${vmid}], 0, 1);
 197
 198     print "Removing Storage definition for ${storage_name}\n";
 199     run_cmd(['pvesm', 'remove', ${storage_name}], 0, 1);
 200
 201     print "Removing RBD namespace '${pool}/${namespace}'\n";
 202     run_cmd(['rbd', 'namespace', 'remove', "${pool}/${namespace}"], 0, 1);
 203
 204     if (!$use_existing) {
 205         print "Removing Storage definition for ${pool}\n";
 206         run_cmd(['pvesm', 'remove', ${pool}], 0, 1);
 207         print "Removing test pool\n";
 208         run_cmd(['pveceph', 'pool', 'destroy', $pool]);
 209     }
 210 }
 211
 212 my $tests = [
 213     # Example structure for tests
 214     # {
 215     #     name => "name of test section",
 216     #     preparations => [
 217     #         ['some', 'prep', 'command'],
 218     #     ],
 219     #     steps => [
 220     #         ['test', 'cmd', $vmid],
 221     #         ['second', 'step', $vmid],
 222     #     ],
 223     #     cleanup => [
 224     #         ['cleanup', 'command'],
 225     #     ],
 226     # },
 227     {
 228         name => 'first VM start',
 229         steps => [
 230             ['qm', 'start', $vmid],
 231         ],
 232     },
 233     {
 234         name => 'snapshot/rollback',
 235         steps => [
 236             ['qm', 'snapshot', $vmid, 'test'],
 237             ['qm', 'rollback', $vmid, 'test'],
 238         ],
 239         cleanup => [
 240             ['qm', 'unlock', $vmid],
 241         ],
 242     },
 243     {
 244         name => 'remove snapshot',
 245         steps => [
 246             ['qm', 'delsnapshot', $vmid, 'test'],
 247         ],
 248     },
 249     {
 250         name => 'moving disk between namespaces',
 251         steps => [
 252             ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1],
 253             ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1],
 254         ],
 255     },
 256     {
 257         name => 'switch to krbd',
 258         preparations => [
 259             ['qm', 'stop', $vmid],
 260             ['pvesm', 'set', $storage_name, '--krbd', 1]
 261         ],
 262     },
 263     {
 264         name => 'start VM with krbd',
 265         steps => [
 266             ['qm', 'start', $vmid],
 267         ],
 268     },
 269     {
 270         name => 'snapshot/rollback with krbd',
 271         steps => [
 272             ['qm', 'snapshot', $vmid, 'test'],
 273             ['qm', 'rollback', $vmid, 'test'],
 274         ],
 275         cleanup => [
 276             ['qm', 'unlock', $vmid],
 277         ],
 278     },
 279     {
 280         name => 'remove snapshot with krbd',
 281         steps => [
 282             ['qm', 'delsnapshot', $vmid, 'test'],
 283         ],
 284     },
 285     {
 286         name => 'moving disk between namespaces with krbd',
 287         steps => [
 288             ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1],
 289             ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1],
 290         ],
 291     },
 292     {
 293         name => 'clone VM with krbd',
 294         steps => [
 295             ['qm', 'clone', $vmid, $vmid_clone],
 296         ],
 297     },
 298     {
 299         name => 'switch to non krbd',
 300         preparations => [
 301             ['qm', 'stop', $vmid],
 302             ['qm', 'stop', $vmid_clone],
 303             ['pvesm', 'set', $storage_name, '--krbd', 0]
 304         ],
 305     },
 306     {
 307         name => 'templates and linked clone',
 308         steps => [
 309             ['qm', 'template', $vmid],
 310             ['qm', 'clone', $vmid, $vmid_linked_clone],
 311             ['qm', 'start', $vmid_linked_clone],
 312             ['qm', 'stop', $vmid_linked_clone],
 313         ],
 314     },
 315     {
 316         name => 'start linked clone with krbd',
 317         preparations => [
 318             ['pvesm', 'set', $storage_name, '--krbd', 1]
 319         ],
 320         steps => [
 321             ['qm', 'start', $vmid_linked_clone],
 322             ['qm', 'stop', $vmid_linked_clone],
 323         ],
 324     },
 325 ];
 326
 327 sub run_prep_cleanup {
 328     my ($cmds) = @_;
 329
 330     for (@$cmds) {
 331         print join(' ', @$_). "\n";
 332         run_cmd($_);
 333     }
 334 }
 335
 336 sub run_steps {
 337     my ($steps) = @_;
 338
 339     for (@$steps) {
 340         ok(run_test_cmd($_), join(' ', @$_));
 341     }
 342 }
 343
 344 sub run_tests {
 345     print "Running tests:\n";
 346
 347     my $num_tests = 0;
 348     for (@$tests) {
 349         $num_tests += scalar(@{$_->{steps}}) if defined $_->{steps};
 350     }
 351
 352     print("Tests: $num_tests\n");
 353     plan tests => $num_tests;
 354
 355     for my $test (@$tests) {
 356         print "Section: $test->{name}\n";
 357         run_prep_cleanup($test->{preparations}) if defined $test->{preparations};
 358         run_steps($test->{steps}) if defined $test->{steps};
 359         run_prep_cleanup($test->{cleanup}) if defined $test->{cleanup};
 360     }
 361
 362     done_testing();
 363
 364     if (Test::More->builder->is_passing()) {
 365         cleanup();
 366     }
 367 }
 368
 369 if ($cleanup) {
 370     cleanup();
 371 } else {
 372     prepare();
 373     run_tests();
 374 }
 375