src/test/rbd_namespace.pl

   1 #!/usr/bin/perl
   2
   3 # This script is meant to be run manually on hyperconverged PVE server with a
   4 # Ceph cluster. It tests how PVE handles RBD namespaces.
   5 #
   6 # The pool (default: rbd) must already exist. The namespace and VMs will be
   7 # created.
   8 #
   9 # Parameters like names for the pool an namespace and the VMID can be
  10 # configured.  The VMIDs for the clones is $vmid -1 and $vmid -2.
  11 #
  12 # Cleanup is done after a successful run. Cleanup can also be called manually.
  13 #
  14 # Known issues:
  15 #
  16 # * Snapshot rollback can sometimes be racy with stopping the VM and Ceph
  17 #  recognizing that the disk image is not in use anymore.
  18
  19 use strict;
  20 use warnings;
  21
  22 use Test::More;
  23 use Getopt::Long;
  24 use JSON;
  25
  26 use PVE::Tools qw(run_command);
  27
  28 my $pool = "testpool";
  29 my $use_existing= undef;
  30 my $namespace = "testspace";
  31 my $showhelp = '';
  32 my $vmid = 999999;
  33 my $cleanup = undef;
  34 my $DEBUG = 0;
  35
  36 my $helpstring = "usage: $0 [OPTIONS]
  37
  38 Known options are:
  39
  40  --pool <name>          pool name, default: ${pool}
  41  --use-existing         use existing pool, default: 0, needs --pool set
  42  --namespace <name>     rbd namespace, default: ${namespace}
  43  --vmid <id>            VMID of the test VM, default: ${vmid}
  44  --cleanup              Remove the storage definitions, namespaces and VM afterwards
  45  -d, --debug            Enable debug output
  46  -h, --help             Print this help message
  47 ";
  48
  49 GetOptions (
  50     "pool=s" => \$pool,
  51     "use-existing" => \$use_existing,
  52     "namespace=s" => \$namespace,
  53     "vmid=i" => \$vmid,
  54     "h|help" => \$showhelp,
  55     "cleanup" => \$cleanup,
  56     "d|debug" => \$DEBUG,
  57 ) or die ($helpstring);
  58
  59 if ($showhelp) {
  60     warn $helpstring;
  61     exit(0);
  62 }
  63
  64 my $storage_name = "${pool}-${namespace}";
  65
  66 my $vmid_clone = int($vmid) - 1;
  67 my $vmid_linked_clone = int($vmid) - 2;
  68
  69 sub jp {
  70     print to_json($_[0], { utf8 => 8, pretty => 1, canonical => 1 }) . "\n";
  71 }
  72 sub dbgvar {
  73     jp(@_) if $DEBUG;
  74 }
  75
  76 sub run_cmd {
  77     my ($cmd, $json, $ignore_errors) = @_;
  78
  79     my $raw = '';
  80     my $parser = sub {$raw .= shift;};
  81
  82     eval {
  83         run_command($cmd, outfunc => $parser);
  84     };
  85     if (my $err = $@) {
  86         die $err if !$ignore_errors;
  87     }
  88
  89     if ($json) {
  90         my $result;
  91         if ($raw eq '') {
  92             $result = [];
  93         } elsif ($raw =~ m/^(\[.*\])$/s) { # untaint
  94             $result = JSON::decode_json($1);
  95         } else {
  96             die "got unexpected data from command: '$cmd' -> '$raw'\n";
  97         }
  98         return $result;
  99         }
 100     return $raw;
 101 }
 102
 103 sub run_test_cmd {
 104     my ($cmd) = @_;
 105
 106     my $raw = '';
 107     my $out = sub {
 108         my $line = shift;
 109         $raw .= "${line}\n";
 110     };
 111
 112     eval {
 113         run_command($cmd, outfunc => $out);
 114     };
 115     if (my $err = $@) {
 116         print $raw;
 117         print $err;
 118         return 0;
 119     }
 120     print $raw;
 121     return 1;
 122 }
 123
 124 sub prepare {
 125     print "Preparing test environment\n";
 126
 127     my $pools = run_cmd("ceph osd pool ls --format json", 1);
 128
 129     my %poolnames = map {$_ => 1} @$pools;
 130     die "Pool '$pool' does not exist!\n"
 131         if !exists($poolnames{$pool}) && $use_existing;
 132
 133     run_cmd(['pveceph', 'pool', 'create', ${pool}, '--add_storages', 1])
 134         if !$use_existing;
 135
 136     my $namespaces = run_cmd(['rbd', '-p', ${pool}, 'namespace', 'ls', '--format', 'json'], 1);
 137     dbgvar($namespace);
 138     my $ns_found = 0;
 139     for my $i (@$namespaces) {
 140         $ns_found = 1 if $i->{name} eq $namespace;
 141     }
 142
 143     if (!$ns_found) {
 144         print "Create namespace '${namespace}' in pool '${pool}'\n";
 145         run_cmd(['rbd', 'namespace', 'create', "${pool}/${namespace}"]);
 146     }
 147
 148     my $storages = run_cmd(['pvesh', 'get', 'storage', '--output-format', 'json'], 1);
 149     dbgvar($storages);
 150     my $rbd_found = 0;
 151     my $pool_found = 0;
 152
 153     print "Create storage definition\n";
 154     for my $stor (@$storages) {
 155         $pool_found = 1 if $stor->{storage} eq $pool;
 156         $rbd_found = 1 if $stor->{storage} eq $storage_name;
 157
 158         if ($rbd_found) {
 159             run_cmd(['pvesm', 'set', ${storage_name}, '--krbd', '0']);
 160             die "Enable the storage '$stor->{storage}'!" if $stor->{disable};
 161         }
 162     }
 163     if (!$pool_found) {
 164         die "No storage for pool '${pool}' found! Must have same name as pool!\n"
 165             if $use_existing;
 166
 167         run_cmd(['pvesm', 'add', 'rbd', $pool, '--pool', $pool, '--content', 'images,rootdir']);
 168     }
 169     # create PVE storages (librbd / krbd)
 170     run_cmd(['pvesm', 'add', 'rbd', ${storage_name}, '--krbd', '0', '--pool', ${pool}, '--namespace', ${namespace}, '--content', 'images,rootdir'])
 171         if !$rbd_found;
 172
 173
 174     # create test VM
 175     print "Create test VM ${vmid}\n";
 176     my $vms = run_cmd(['pvesh', 'get', 'cluster/resources', '--type', 'vm', '--output-format', 'json'], 1);
 177     for my $vm (@$vms) {
 178         # TODO: introduce a force flag to make this behaviour configurable
 179
 180         if ($vm->{vmid} eq $vmid) {
 181             print "Test VM '${vmid}' already exists. It will be removed and recreated!\n";
 182             run_cmd(['qm', 'stop', ${vmid}], 0, 1);
 183             run_cmd(['qm', 'destroy', ${vmid}]);
 184         }
 185     }
 186     run_cmd(['qm', 'create', ${vmid}, '--bios', 'ovmf', '--efidisk0', "${storage_name}:1", '--scsi0', "${storage_name}:2"]);
 187 }
 188
 189
 190 sub cleanup {
 191     print "Cleaning up test environment!\n";
 192     print "Removing VMs\n";
 193     run_cmd(['qm', 'stop', ${vmid}], 0, 1);
 194     run_cmd(['qm', 'stop', ${vmid_linked_clone}], 0, 1);
 195     run_cmd(['qm', 'stop', ${vmid_clone}], 0, 1);
 196     run_cmd(['qm', 'destroy', ${vmid_linked_clone}], 0, 1);
 197     run_cmd(['qm', 'destroy', ${vmid_clone}], 0, 1);
 198     run_cmd(['for', 'i', 'in', "/dev/rbd/${pool}/${namespace}/*;", 'do', '/usr/bin/rbd', 'unmap', '\$i;', 'done'], 0, 1);
 199     run_cmd(['qm', 'unlock', ${vmid}], 0, 1);
 200     run_cmd(['qm', 'destroy', ${vmid}], 0, 1);
 201
 202     print "Removing Storage definition for ${storage_name}\n";
 203     run_cmd(['pvesm', 'remove', ${storage_name}], 0, 1);
 204
 205     print "Removing RBD namespace '${pool}/${namespace}'\n";
 206     run_cmd(['rbd', 'namespace', 'remove', "${pool}/${namespace}"], 0, 1);
 207
 208     if (!$use_existing) {
 209         print "Removing Storage definition for ${pool}\n";
 210         run_cmd(['pvesm', 'remove', ${pool}], 0, 1);
 211         print "Removing test pool\n";
 212         run_cmd(['pveceph', 'pool', 'destroy', $pool]);
 213     }
 214 }
 215
 216 my $tests = [
 217     # Example structure for tests
 218     # {
 219     #     name => "name of test section",
 220     #     preparations => [
 221     #         ['some', 'prep', 'command'],
 222     #     ],
 223     #     steps => [
 224     #         ['test', 'cmd', $vmid],
 225     #         ['second', 'step', $vmid],
 226     #     ],
 227     #     cleanup => [
 228     #         ['cleanup', 'command'],
 229     #     ],
 230     # },
 231     {
 232         name => 'first VM start',
 233         steps => [
 234             ['qm', 'start', $vmid],
 235         ],
 236     },
 237     {
 238         name => 'snapshot/rollback',
 239         steps => [
 240             ['qm', 'snapshot', $vmid, 'test'],
 241             ['qm', 'rollback', $vmid, 'test'],
 242         ],
 243         cleanup => [
 244             ['qm', 'unlock', $vmid],
 245         ],
 246     },
 247     {
 248         name => 'remove snapshot',
 249         steps => [
 250             ['qm', 'delsnapshot', $vmid, 'test'],
 251         ],
 252     },
 253     {
 254         name => 'moving disk between namespaces',
 255         steps => [
 256             ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1],
 257             ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1],
 258         ],
 259     },
 260     {
 261         name => 'switch to krbd',
 262         preparations => [
 263             ['qm', 'stop', $vmid],
 264             ['pvesm', 'set', $storage_name, '--krbd', 1]
 265         ],
 266     },
 267     {
 268         name => 'start VM with krbd',
 269         steps => [
 270             ['qm', 'start', $vmid],
 271         ],
 272     },
 273     {
 274         name => 'snapshot/rollback with krbd',
 275         steps => [
 276             ['qm', 'snapshot', $vmid, 'test'],
 277             ['qm', 'rollback', $vmid, 'test'],
 278         ],
 279         cleanup => [
 280             ['qm', 'unlock', $vmid],
 281         ],
 282     },
 283     {
 284         name => 'remove snapshot with krbd',
 285         steps => [
 286             ['qm', 'delsnapshot', $vmid, 'test'],
 287         ],
 288     },
 289     {
 290         name => 'moving disk between namespaces with krbd',
 291         steps => [
 292             ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1],
 293             ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1],
 294         ],
 295     },
 296     {
 297         name => 'clone VM with krbd',
 298         steps => [
 299             ['qm', 'clone', $vmid, $vmid_clone],
 300         ],
 301     },
 302     {
 303         name => 'switch to non krbd',
 304         preparations => [
 305             ['qm', 'stop', $vmid],
 306             ['qm', 'stop', $vmid_clone],
 307             ['pvesm', 'set', $storage_name, '--krbd', 0]
 308         ],
 309     },
 310     {
 311         name => 'templates and linked clone',
 312         steps => [
 313             ['qm', 'template', $vmid],
 314             ['qm', 'clone', $vmid, $vmid_linked_clone],
 315             ['qm', 'start', $vmid_linked_clone],
 316             ['qm', 'stop', $vmid_linked_clone],
 317         ],
 318     },
 319     {
 320         name => 'start linked clone with krbd',
 321         preparations => [
 322             ['pvesm', 'set', $storage_name, '--krbd', 1]
 323         ],
 324         steps => [
 325             ['qm', 'start', $vmid_linked_clone],
 326             ['qm', 'stop', $vmid_linked_clone],
 327         ],
 328     },
 329 ];
 330
 331 sub run_prep_cleanup {
 332     my ($cmds) = @_;
 333
 334     for (@$cmds) {
 335         print join(' ', @$_). "\n";
 336         run_cmd($_);
 337     }
 338 }
 339
 340 sub run_steps {
 341     my ($steps) = @_;
 342
 343     for (@$steps) {
 344         ok(run_test_cmd($_), join(' ', @$_));
 345     }
 346 }
 347
 348 sub run_tests {
 349     print "Running tests:\n";
 350
 351     my $num_tests = 0;
 352     for (@$tests) {
 353         $num_tests += scalar(@{$_->{steps}}) if defined $_->{steps};
 354     }
 355
 356     print("Tests: $num_tests\n");
 357     plan tests => $num_tests;
 358
 359     for my $test (@$tests) {
 360         print "Section: $test->{name}\n";
 361         run_prep_cleanup($test->{preparations}) if defined $test->{preparations};
 362         run_steps($test->{steps}) if defined $test->{steps};
 363         run_prep_cleanup($test->{cleanup}) if defined $test->{cleanup};
 364     }
 365
 366     done_testing();
 367
 368     if (Test::More->builder->is_passing()) {
 369         cleanup();
 370     }
 371 }
 372
 373 if ($cleanup) {
 374     cleanup();
 375 } else {
 376     prepare();
 377     run_tests();
 378 }
 379