]>
Commit | Line | Data |
---|---|---|
9f3d6f3b AL |
1 | #!/usr/bin/perl |
2 | ||
3 | # This script is meant to be run manually on hyperconverged PVE server with a | |
4 | # Ceph cluster. It tests how PVE handles RBD namespaces. | |
5 | # | |
6 | # The pool (default: rbd) must already exist. The namespace and VMs will be | |
7 | # created. | |
8 | # | |
9 | # Parameters like names for the pool an namespace and the VMID can be | |
10 | # configured. The VMIDs for the clones is $vmid -1 and $vmid -2. | |
11 | # | |
12 | # Cleanup is done after a successful run. Cleanup can also be called manually. | |
13 | # | |
14 | # Known issues: | |
15 | # | |
16 | # * Snapshot rollback can sometimes be racy with stopping the VM and Ceph | |
17 | # recognizing that the disk image is not in use anymore. | |
18 | ||
19 | use strict; | |
20 | use warnings; | |
21 | ||
22 | use Test::More; | |
23 | use Getopt::Long; | |
24 | use JSON; | |
25 | ||
26 | use PVE::Tools qw(run_command); | |
27 | ||
28 | my $pool = "testpool"; | |
29 | my $use_existing= undef; | |
30 | my $namespace = "testspace"; | |
31 | my $showhelp = ''; | |
32 | my $vmid = 999999; | |
33 | my $cleanup = undef; | |
34 | my $DEBUG = 0; | |
35 | ||
36 | my $helpstring = "To override default values, set them as named parameters: | |
37 | ||
38 | --pool pool name, default: ${pool} | |
39 | --use_existing use existing pool, default: 0, needs --pool set | |
40 | --namespace rbd namespace, default: ${namespace} | |
41 | --vmid VMID of the test VM, default: ${vmid} | |
42 | --cleanup Remove the storage definitions, namespaces and VMs | |
43 | --debug Enable debug output\n"; | |
44 | ||
45 | GetOptions ( | |
46 | "pool=s" => \$pool, | |
47 | "use_existing" => \$use_existing, | |
48 | "namespace=s" => \$namespace, | |
49 | "vmid=i" => \$vmid, | |
50 | "help" => \$showhelp, | |
51 | "cleanup" => \$cleanup, | |
52 | "debug" => \$DEBUG, | |
53 | ) or die ($helpstring); | |
54 | ||
55 | die $helpstring if $showhelp; | |
56 | ||
57 | my $storage_name = "${pool}-${namespace}"; | |
58 | ||
59 | my $vmid_clone = int($vmid) - 1; | |
60 | my $vmid_linked_clone = int($vmid) - 2; | |
61 | ||
62 | sub jp { | |
63 | return if !$DEBUG; | |
64 | print to_json($_[0], { utf8 => 8, pretty => 1, canonical => 1 }) . "\n"; | |
65 | } | |
66 | ||
67 | sub run_cmd { | |
68 | my ($cmd, $json, $ignore_errors) = @_; | |
69 | ||
70 | my $raw = ''; | |
71 | my $parser = sub {$raw .= shift;}; | |
72 | ||
73 | eval { | |
74 | run_command($cmd, outfunc => $parser); | |
75 | }; | |
76 | if (my $err = $@) { | |
77 | die $err if !$ignore_errors; | |
78 | } | |
79 | ||
80 | if ($json) { | |
81 | my $result; | |
82 | if ($raw eq '') { | |
83 | $result = []; | |
84 | } elsif ($raw =~ m/^(\[.*\])$/s) { # untaint | |
85 | $result = JSON::decode_json($1); | |
86 | } else { | |
87 | die "got unexpected data from command: '$cmd' -> '$raw'\n"; | |
88 | } | |
89 | return $result; | |
90 | } | |
91 | return $raw; | |
92 | } | |
93 | ||
94 | sub run_test_cmd { | |
95 | my ($cmd) = @_; | |
96 | ||
97 | my $raw = ''; | |
98 | my $out = sub { | |
99 | my $line = shift; | |
100 | $raw .= "${line}\n"; | |
101 | }; | |
102 | ||
103 | eval { | |
104 | run_command($cmd, outfunc => $out); | |
105 | }; | |
106 | if (my $err = $@) { | |
107 | print $raw; | |
108 | print $err; | |
109 | return 0; | |
110 | } | |
111 | print $raw; | |
112 | return 1; | |
113 | } | |
114 | ||
115 | sub prepare { | |
116 | print "Preparing test environent\n"; | |
117 | ||
118 | my $pools = run_cmd("ceph osd pool ls --format json", 1); | |
119 | ||
120 | my %poolnames = map {$_ => 1} @$pools; | |
121 | die "Pool '$pool' does not exist!\n" | |
122 | if !exists($poolnames{$pool}) && $use_existing; | |
123 | ||
124 | run_cmd(['pveceph', 'pool', 'create', ${pool}, '--add_storages', 1]) | |
125 | if !$use_existing; | |
126 | ||
127 | my $namespaces = run_cmd(['rbd', '-p', ${pool}, 'namespace', 'ls', '--format', 'json'], 1); | |
128 | my $ns_found = 0; | |
129 | for my $i (@$namespaces) { | |
130 | #print Dumper $i; | |
131 | $ns_found = 1 if $i->{name} eq $namespace; | |
132 | } | |
133 | ||
134 | if (!$ns_found) { | |
135 | print "Create namespace '${namespace}' in pool '${pool}'\n"; | |
136 | run_cmd(['rbd', 'namespace', 'create', "${pool}/${namespace}"]); | |
137 | } | |
138 | ||
139 | my $storages = run_cmd(['pvesh', 'get', 'storage', '--output-format', 'json'], 1); | |
140 | #print Dumper $storages; | |
141 | my $rbd_found = 0; | |
142 | my $pool_found = 0; | |
143 | ||
144 | print "Create storage definition\n"; | |
145 | for my $stor (@$storages) { | |
146 | $pool_found = 1 if $stor->{storage} eq $pool; | |
147 | $rbd_found = 1 if $stor->{storage} eq $storage_name; | |
148 | ||
149 | if ($rbd_found) { | |
150 | run_cmd(['pvesm', 'set', ${storage_name}, '--krbd', '0']); | |
151 | die "Enable the storage '$stor->{storage}'!" if $stor->{disable}; | |
152 | } | |
153 | } | |
154 | if (!$pool_found) { | |
155 | die "No storage for pool '${pool}' found! Must have same name as pool!\n" | |
156 | if $use_existing; | |
157 | ||
158 | run_cmd(['pvesm', 'add', 'rbd', $pool, '--pool', $pool, '--content', 'images,rootdir']); | |
159 | } | |
160 | # create PVE storages (librbd / krbd) | |
161 | run_cmd(['pvesm', 'add', 'rbd', ${storage_name}, '--krbd', '0', '--pool', ${pool}, '--namespace', ${namespace}, '--content', 'images,rootdir']) | |
162 | if !$rbd_found; | |
163 | ||
164 | ||
165 | # create test VM | |
166 | print "Create test VM ${vmid}\n"; | |
167 | my $vms = run_cmd(['pvesh', 'get', 'cluster/resources', '--type', 'vm', '--output-format', 'json'], 1); | |
168 | for my $vm (@$vms) { | |
169 | # TODO: introduce a force flag to make this behaviour configurable | |
170 | ||
171 | if ($vm->{vmid} eq $vmid) { | |
172 | print "Test VM '${vmid}' already exists. It will be removed and recreated!\n"; | |
173 | run_cmd(['qm', 'stop', ${vmid}], 0, 1); | |
174 | run_cmd(['qm', 'destroy', ${vmid}]); | |
175 | } | |
176 | } | |
177 | run_cmd(['qm', 'create', ${vmid}, '--bios', 'ovmf', '--efidisk0', "${storage_name}:1", '--scsi0', "${storage_name}:2"]); | |
178 | } | |
179 | ||
180 | ||
181 | sub cleanup { | |
182 | print "Cleaning up test environment!\n"; | |
183 | print "Removing VMs\n"; | |
184 | run_cmd(['qm', 'stop', ${vmid}], 0, 1); | |
185 | run_cmd(['qm', 'stop', ${vmid_linked_clone}], 0, 1); | |
186 | run_cmd(['qm', 'stop', ${vmid_clone}], 0, 1); | |
187 | run_cmd(['qm', 'destroy', ${vmid_linked_clone}], 0, 1); | |
188 | run_cmd(['qm', 'destroy', ${vmid_clone}], 0, 1); | |
189 | run_cmd(['for', 'i', 'in', "/dev/rbd/${pool}/${namespace}/*;", 'do', '/usr/bin/rbd', 'unmap', '\$i;', 'done'], 0, 1); | |
190 | run_cmd(['qm', 'unlock', ${vmid}], 0, 1); | |
191 | run_cmd(['qm', 'destroy', ${vmid}], 0, 1); | |
192 | ||
193 | print "Removing Storage definition for ${storage_name}\n"; | |
194 | run_cmd(['pvesm', 'remove', ${storage_name}], 0, 1); | |
195 | ||
196 | print "Removing RBD namespace '${pool}/${namespace}'\n"; | |
197 | run_cmd(['rbd', 'namespace', 'remove', "${pool}/${namespace}"], 0, 1); | |
198 | ||
199 | if (!$use_existing) { | |
200 | print "Removing Storage definition for ${pool}\n"; | |
201 | run_cmd(['pvesm', 'remove', ${pool}], 0, 1); | |
202 | print "Removing test pool\n"; | |
203 | run_cmd(['pveceph', 'pool', 'destroy', $pool]); | |
204 | } | |
205 | } | |
206 | ||
207 | my $tests = [ | |
208 | # Example structure for tests | |
209 | # { | |
210 | # name => "name of test section", | |
211 | # preparations => [ | |
212 | # ['some', 'prep', 'command'], | |
213 | # ], | |
214 | # steps => [ | |
215 | # ['test', 'cmd', $vmid], | |
216 | # ['second', 'step', $vmid], | |
217 | # ], | |
218 | # cleanup => [ | |
219 | # ['cleanup', 'command'], | |
220 | # ], | |
221 | # }, | |
222 | { | |
223 | name => 'first VM start', | |
224 | steps => [ | |
225 | ['qm', 'start', $vmid], | |
226 | ], | |
227 | }, | |
228 | { | |
229 | name => 'snapshot/rollback', | |
230 | steps => [ | |
231 | ['qm', 'snapshot', $vmid, 'test'], | |
232 | ['qm', 'rollback', $vmid, 'test'], | |
233 | ], | |
234 | cleanup => [ | |
235 | ['qm', 'unlock', $vmid], | |
236 | ], | |
237 | }, | |
238 | { | |
239 | name => 'remove snapshot', | |
240 | steps => [ | |
241 | ['qm', 'delsnapshot', $vmid, 'test'], | |
242 | ], | |
243 | }, | |
244 | { | |
245 | name => 'moving disk between namespaces', | |
246 | steps => [ | |
247 | ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1], | |
248 | ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1], | |
249 | ], | |
250 | }, | |
251 | { | |
252 | name => 'switch to krbd', | |
253 | preparations => [ | |
254 | ['qm', 'stop', $vmid], | |
255 | ['pvesm', 'set', $storage_name, '--krbd', 1] | |
256 | ], | |
257 | }, | |
258 | { | |
259 | name => 'start VM with krbd', | |
260 | steps => [ | |
261 | ['qm', 'start', $vmid], | |
262 | ], | |
263 | }, | |
264 | { | |
265 | name => 'snapshot/rollback with krbd', | |
266 | steps => [ | |
267 | ['qm', 'snapshot', $vmid, 'test'], | |
268 | ['qm', 'rollback', $vmid, 'test'], | |
269 | ], | |
270 | cleanup => [ | |
271 | ['qm', 'unlock', $vmid], | |
272 | ], | |
273 | }, | |
274 | { | |
275 | name => 'remove snapshot with krbd', | |
276 | steps => [ | |
277 | ['qm', 'delsnapshot', $vmid, 'test'], | |
278 | ], | |
279 | }, | |
280 | { | |
281 | name => 'moving disk between namespaces with krbd', | |
282 | steps => [ | |
283 | ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1], | |
284 | ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1], | |
285 | ], | |
286 | }, | |
287 | { | |
288 | name => 'clone VM with krbd', | |
289 | steps => [ | |
290 | ['qm', 'clone', $vmid, $vmid_clone], | |
291 | ], | |
292 | }, | |
293 | { | |
294 | name => 'switch to non krbd', | |
295 | preparations => [ | |
296 | ['qm', 'stop', $vmid], | |
297 | ['qm', 'stop', $vmid_clone], | |
298 | ['pvesm', 'set', $storage_name, '--krbd', 0] | |
299 | ], | |
300 | }, | |
301 | { | |
302 | name => 'templates and linked clone', | |
303 | steps => [ | |
304 | ['qm', 'template', $vmid], | |
305 | ['qm', 'clone', $vmid, $vmid_linked_clone], | |
306 | ['qm', 'start', $vmid_linked_clone], | |
307 | ['qm', 'stop', $vmid_linked_clone], | |
308 | ], | |
309 | }, | |
310 | { | |
311 | name => 'start linked clone with krbd', | |
312 | preparations => [ | |
313 | ['pvesm', 'set', $storage_name, '--krbd', 1] | |
314 | ], | |
315 | steps => [ | |
316 | ['qm', 'start', $vmid_linked_clone], | |
317 | ['qm', 'stop', $vmid_linked_clone], | |
318 | ], | |
319 | }, | |
320 | ]; | |
321 | ||
322 | sub run_prep_cleanup { | |
323 | my ($cmds) = @_; | |
324 | ||
325 | for (@$cmds) { | |
326 | print join(' ', @$_). "\n"; | |
327 | run_cmd($_); | |
328 | } | |
329 | } | |
330 | ||
331 | sub run_steps { | |
332 | my ($steps) = @_; | |
333 | ||
334 | for (@$steps) { | |
335 | ok(run_test_cmd($_), join(' ', @$_)); | |
336 | } | |
337 | } | |
338 | ||
339 | sub run_tests { | |
340 | print "Running tests:\n"; | |
341 | ||
342 | my $num_tests = 0; | |
343 | for (@$tests) { | |
344 | $num_tests += scalar(@{$_->{steps}}) if defined $_->{steps}; | |
345 | } | |
346 | ||
347 | print("Tests: $num_tests\n"); | |
348 | plan tests => $num_tests; | |
349 | ||
350 | for my $test (@$tests) { | |
351 | print "Section: $test->{name}\n"; | |
352 | run_prep_cleanup($test->{preparations}) if defined $test->{preparations}; | |
353 | run_steps($test->{steps}) if defined $test->{steps}; | |
354 | run_prep_cleanup($test->{cleanup}) if defined $test->{cleanup}; | |
355 | } | |
356 | ||
357 | done_testing(); | |
358 | ||
359 | if (Test::More->builder->is_passing()) { | |
360 | cleanup(); | |
361 | } | |
362 | } | |
363 | ||
364 | if ($cleanup) { | |
365 | cleanup(); | |
366 | } else { | |
367 | prepare(); | |
368 | run_tests(); | |
369 | } | |
370 |