test/list_volumes_test.pm

   1 package PVE::Storage::TestListVolumes;
   2
   3 use strict;
   4 use warnings;
   5
   6 use lib qw(..);
   7
   8 use PVE::Storage;
   9 use PVE::Cluster;
  10 use PVE::Tools qw(run_command);
  11
  12 use Test::More;
  13 use Test::MockModule;
  14
  15 use Cwd;
  16 use File::Basename;
  17 use File::Path qw(make_path remove_tree);
  18 use File::stat qw();
  19 use File::Temp;
  20 use Storable qw(dclone);
  21
  22 use constant DEFAULT_SIZE => 131072; # 128 kiB
  23 use constant DEFAULT_USED => 262144; # 256 kiB
  24 use constant DEFAULT_CTIME => 1234567890;
  25
  26 # get_vmlist() return values
  27 my $mocked_vmlist = {
  28     'version' => 1,
  29     'ids' => {
  30         '16110' => {
  31             'node'    => 'x42',
  32             'type'    => 'qemu',
  33             'version' => 4,
  34         },
  35         '16112' => {
  36             'node'    => 'x42',
  37             'type'    => 'lxc',
  38             'version' => 7,
  39         },
  40         '16114' => {
  41             'node'    => 'x42',
  42             'type'    => 'qemu',
  43             'version' => 2,
  44         },
  45         '16113' => {
  46             'node'    => 'x42',
  47             'type'    => 'qemu',
  48             'version' => 5,
  49         },
  50         '16115' => {
  51             'node'    => 'x42',
  52             'type'    => 'qemu',
  53             'version' => 1,
  54         },
  55         '9004' => {
  56             'node'    => 'x42',
  57             'type'    => 'qemu',
  58             'version' => 6,
  59         }
  60     }
  61 };
  62
  63 my $storage_dir = File::Temp->newdir();
  64 my $scfg = {
  65     'type'     => 'dir',
  66     'maxfiles' => 0,
  67     'path'     => $storage_dir,
  68     'shared'   => 0,
  69     'content'  => {
  70         'iso'      => 1,
  71         'rootdir'  => 1,
  72         'vztmpl'   => 1,
  73         'images'   => 1,
  74         'snippets' => 1,
  75         'backup'   => 1,
  76     },
  77 };
  78
  79 # The test cases are comprised of an arry of hashes with the following keys:
  80 # description => displayed on error by Test::More
  81 # vmid        => used for image matches by list_volume
  82 # files       => array of files for qemu-img to create
  83 # expected    => returned result hash
  84 #                (content, ctime, format, parent, size, used, vimd, volid)
  85 my @tests = (
  86     {
  87         description => 'VMID: 16110, VM, qcow2, backup, snippets',
  88         vmid => '16110',
  89         files => [
  90             "$storage_dir/images/16110/vm-16110-disk-0.qcow2",
  91             "$storage_dir/images/16110/vm-16110-disk-1.raw",
  92             "$storage_dir/images/16110/vm-16110-disk-2.vmdk",
  93             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
  94             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
  95             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
  96             "$storage_dir/snippets/userconfig.yaml",
  97             "$storage_dir/snippets/hookscript.pl",
  98         ],
  99         expected => [
 100             {
 101                 'content' => 'images',
 102                 'ctime'   => DEFAULT_CTIME,
 103                 'format'  => 'qcow2',
 104                 'parent'  => undef,
 105                 'size'    => DEFAULT_SIZE,
 106                 'used'    => DEFAULT_USED,
 107                 'vmid'    => '16110',
 108                 'volid'   => 'local:16110/vm-16110-disk-0.qcow2',
 109             },
 110             {
 111                 'content' => 'images',
 112                 'ctime'   => DEFAULT_CTIME,
 113                 'format'  => 'raw',
 114                 'parent'  => undef,
 115                 'size'    => DEFAULT_SIZE,
 116                 'used'    => DEFAULT_USED,
 117                 'vmid'    => '16110',
 118                 'volid'   => 'local:16110/vm-16110-disk-1.raw',
 119             },
 120             {
 121                 'content' => 'images',
 122                 'ctime'   => DEFAULT_CTIME,
 123                 'format'  => 'vmdk',
 124                 'parent'  => undef,
 125                 'size'    => DEFAULT_SIZE,
 126                 'used'    => DEFAULT_USED,
 127                 'vmid'    => '16110',
 128                 'volid'   => 'local:16110/vm-16110-disk-2.vmdk',
 129             },
 130             {
 131                 'content' => 'backup',
 132                 'ctime'   => DEFAULT_CTIME,
 133                 'format'  => 'vma.gz',
 134                 'size'    => DEFAULT_SIZE,
 135                 'vmid'    => '16110',
 136                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz',
 137             },
 138             {
 139                 'content' => 'backup',
 140                 'ctime'   => DEFAULT_CTIME,
 141                 'format'  => 'vma.lzo',
 142                 'size'    => DEFAULT_SIZE,
 143                 'vmid'    => '16110',
 144                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo',
 145             },
 146             {
 147                 'content' => 'backup',
 148                 'ctime'   => DEFAULT_CTIME,
 149                 'format'  => 'vma',
 150                 'size'    => DEFAULT_SIZE,
 151                 'vmid'    => '16110',
 152                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma',
 153             },
 154             {
 155                 'content' => 'snippets',
 156                 'ctime'   => DEFAULT_CTIME,
 157                 'format'  => 'snippet',
 158                 'size'    => DEFAULT_SIZE,
 159                 'volid'   => 'local:snippets/hookscript.pl',
 160             },
 161             {
 162                 'content' => 'snippets',
 163                 'ctime'   => DEFAULT_CTIME,
 164                 'format'  => 'snippet',
 165                 'size'    => DEFAULT_SIZE,
 166                 'volid'   => 'local:snippets/userconfig.yaml',
 167             },
 168         ],
 169     },
 170     {
 171         description => 'VMID: 16112, lxc, raw, backup',
 172         vmid => '16112',
 173         files => [
 174             "$storage_dir/images/16112/vm-16112-disk-0.raw",
 175             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
 176             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
 177             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
 178         ],
 179         expected => [
 180             {
 181                 'content' => 'rootdir',
 182                 'ctime'   => DEFAULT_CTIME,
 183                 'format'  => 'raw',
 184                 'parent'  => undef,
 185                 'size'    => DEFAULT_SIZE,
 186                 'used'    => DEFAULT_USED,
 187                 'vmid'    => '16112',
 188                 'volid'   => 'local:16112/vm-16112-disk-0.raw',
 189             },
 190             {
 191                 'content' => 'backup',
 192                 'ctime'   => DEFAULT_CTIME,
 193                 'format'  => 'tar.lzo',
 194                 'size'    => DEFAULT_SIZE,
 195                 'vmid'    => '16112',
 196                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo',
 197             },
 198             {
 199                 'content' => 'backup',
 200                 'ctime'   => DEFAULT_CTIME,
 201                 'format'  => 'tar.gz',
 202                 'size'    => DEFAULT_SIZE,
 203                 'vmid'    => '16112',
 204                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz',
 205             },
 206             {
 207                 'content' => 'backup',
 208                 'ctime'   => DEFAULT_CTIME,
 209                 'format'  => 'tgz',
 210                 'size'    => DEFAULT_SIZE,
 211                 'vmid'    => '16112',
 212                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz',
 213             },
 214         ],
 215     },
 216     {
 217         description => 'VMID: 16114, VM, qcow2, linked clone',
 218         vmid => '16114',
 219         files => [
 220             "$storage_dir/images/16114/vm-16114-disk-0.qcow2",
 221             "$storage_dir/images/16114/vm-16114-disk-1.qcow2",
 222         ],
 223         parent => [
 224             "../9004/base-9004-disk-0.qcow2",
 225             "../9004/base-9004-disk-1.qcow2",
 226         ],
 227         expected => [
 228             {
 229                 'content' => 'images',
 230                 'ctime'   => DEFAULT_CTIME,
 231                 'format'  => 'qcow2',
 232                 'parent'  => '../9004/base-9004-disk-0.qcow2',
 233                 'size'    => DEFAULT_SIZE,
 234                 'used'    => DEFAULT_USED,
 235                 'vmid'    => '16114',
 236                 'volid'   => 'local:9004/base-9004-disk-0.qcow2/16114/vm-16114-disk-0.qcow2',
 237             },
 238             {
 239                 'content' => 'images',
 240                 'ctime'   => DEFAULT_CTIME,
 241                 'format'  => 'qcow2',
 242                 'parent'  => '../9004/base-9004-disk-1.qcow2',
 243                 'size'    => DEFAULT_SIZE,
 244                 'used'    => DEFAULT_USED,
 245                 'vmid'    => '16114',
 246                 'volid'   => 'local:9004/base-9004-disk-1.qcow2/16114/vm-16114-disk-1.qcow2',
 247             },
 248         ],
 249     },
 250     {
 251         description => 'VMID: 9004, VM, template, qcow2',
 252         vmid => '9004',
 253         files => [
 254             "$storage_dir/images/9004/base-9004-disk-0.qcow2",
 255             "$storage_dir/images/9004/base-9004-disk-1.qcow2",
 256         ],
 257         expected => [
 258             {
 259                 'content' => 'images',
 260                 'ctime'   => DEFAULT_CTIME,
 261                 'format'  => 'qcow2',
 262                 'parent'  => undef,
 263                 'size'    => DEFAULT_SIZE,
 264                 'used'    => DEFAULT_USED,
 265                 'vmid'    => '9004',
 266                 'volid'   => 'local:9004/base-9004-disk-0.qcow2',
 267             },
 268             {
 269                 'content' => 'images',
 270                 'ctime'   => DEFAULT_CTIME,
 271                 'format'  => 'qcow2',
 272                 'parent'  => undef,
 273                 'size'    => DEFAULT_SIZE,
 274                 'used'    => DEFAULT_USED,
 275                 'vmid'    => '9004',
 276                 'volid'   => 'local:9004/base-9004-disk-1.qcow2',
 277             },
 278         ],
 279     },
 280     {
 281         description => 'VMID: none, templates, snippets, backup',
 282         vmid => undef,
 283         files => [
 284             "$storage_dir/dump/vzdump-lxc-19253-2020_02_03-19_57_43.tar.gz",
 285             "$storage_dir/dump/vzdump-lxc-19254-2019_01_21-19_29_19.tar",
 286             "$storage_dir/template/iso/archlinux-2020.02.01-x86_64.iso",
 287             "$storage_dir/template/iso/debian-8.11.1-amd64-DVD-1.iso",
 288             "$storage_dir/template/iso/debian-9.12.0-amd64-netinst.iso",
 289             "$storage_dir/template/iso/proxmox-ve_6.1-1.iso",
 290             "$storage_dir/template/cache/archlinux-base_20190924-1_amd64.tar.gz",
 291             "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz",
 292             "$storage_dir/template/cache/alpine-3.10-default_20190626_amd64.tar.xz",
 293             "$storage_dir/snippets/userconfig.yaml",
 294             "$storage_dir/snippets/hookscript.pl",
 295             "$storage_dir/private/1234/", # fileparse needs / at the end
 296             "$storage_dir/private/1234/subvol-1234-disk-0.subvol/", # fileparse needs / at the end
 297         ],
 298         expected => [
 299             {
 300                 'content' => 'vztmpl',
 301                 'ctime'   => DEFAULT_CTIME,
 302                 'format'  => 'txz',
 303                 'size'    => DEFAULT_SIZE,
 304                 'volid'   => 'local:vztmpl/alpine-3.10-default_20190626_amd64.tar.xz',
 305             },
 306             {
 307                 'content' => 'vztmpl',
 308                 'ctime'   => DEFAULT_CTIME,
 309                 'format'  => 'tgz',
 310                 'size'    => DEFAULT_SIZE,
 311                 'volid'   => 'local:vztmpl/archlinux-base_20190924-1_amd64.tar.gz',
 312             },
 313             {
 314                 'content' => 'vztmpl',
 315                 'ctime'   => DEFAULT_CTIME,
 316                 'format'  => 'tgz',
 317                 'size'    => DEFAULT_SIZE,
 318                 'volid'   => 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
 319             },
 320             {
 321                 'content' => 'iso',
 322                 'ctime'   => DEFAULT_CTIME,
 323                 'format'  => 'iso',
 324                 'size'    => DEFAULT_SIZE,
 325                 'volid'   => 'local:iso/archlinux-2020.02.01-x86_64.iso',
 326             },
 327             {
 328                 'content' => 'iso',
 329                 'ctime'   => DEFAULT_CTIME,
 330                 'format'  => 'iso',
 331                 'size'    => DEFAULT_SIZE,
 332                 'volid'   => 'local:iso/debian-8.11.1-amd64-DVD-1.iso',
 333             },
 334             {
 335                 'content' => 'iso',
 336                 'ctime'   => DEFAULT_CTIME,
 337                 'format'  => 'iso',
 338                 'size'    => DEFAULT_SIZE,
 339                 'volid'   => 'local:iso/debian-9.12.0-amd64-netinst.iso',
 340             },
 341             {
 342                 'content' => 'iso',
 343                 'ctime'   => DEFAULT_CTIME,
 344                 'format'  => 'iso',
 345                 'size'    => DEFAULT_SIZE,
 346                 'volid'   => 'local:iso/proxmox-ve_6.1-1.iso',
 347             },
 348             {
 349                 'content' => 'backup',
 350                 'ctime'   => DEFAULT_CTIME,
 351                 'format'  => 'tar.gz',
 352                 'size'    => DEFAULT_SIZE,
 353                 'vmid'    => '19253',
 354                 'volid'   => 'local:backup/vzdump-lxc-19253-2020_02_03-19_57_43.tar.gz',
 355             },
 356             {
 357                 'content' => 'backup',
 358                 'ctime'   => DEFAULT_CTIME,
 359                 'format'  => 'tar',
 360                 'size'    => DEFAULT_SIZE,
 361                 'vmid'    => '19254',
 362                 'volid'   => 'local:backup/vzdump-lxc-19254-2019_01_21-19_29_19.tar',
 363             },
 364             {
 365                 'content' => 'snippets',
 366                 'ctime'   => DEFAULT_CTIME,
 367                 'format'  => 'snippet',
 368                 'size'    => DEFAULT_SIZE,
 369                 'volid'   => 'local:snippets/hookscript.pl',
 370             },
 371             {
 372                 'content' => 'snippets',
 373                 'ctime'   => DEFAULT_CTIME,
 374                 'format'  => 'snippet',
 375                 'size'    => DEFAULT_SIZE,
 376                 'volid'   => 'local:snippets/userconfig.yaml',
 377             },
 378         ],
 379     },
 380     {
 381         description => 'VMID: none, parent, non-matching',
 382         # string instead of vmid in folder
 383         #"$storage_dir/images/ssss/base-4321-disk-0.qcow2/1234/vm-1234-disk-0.qcow2",
 384         vmid => undef,
 385         files => [
 386             "$storage_dir/images/1234/vm-1234-disk-0.qcow2",
 387         ],
 388         parent => [
 389             "../ssss/base-4321-disk-0.qcow2",
 390         ],
 391         expected => [
 392             {
 393                 'content' => 'images',
 394                 'ctime'   => DEFAULT_CTIME,
 395                 'format'  => 'qcow2',
 396                 'parent'  => '../ssss/base-4321-disk-0.qcow2',
 397                 'size'    => DEFAULT_SIZE,
 398                 'used'    => DEFAULT_USED,
 399                 'vmid'    => '1234',
 400                 'volid'   => 'local:1234/vm-1234-disk-0.qcow2',
 401             }
 402         ],
 403     },
 404     {
 405         description => 'VMID: none, non-matching',
 406         # failed matches
 407         vmid => undef,
 408         files => [
 409             "$storage_dir/images/ssss/base-4321-disk-0.raw",
 410             "$storage_dir/images/ssss/vm-1234-disk-0.qcow2",
 411             "$storage_dir/template/iso/yet-again-a-installation-disk.dvd",
 412             "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz",
 413             "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2",
 414             "$storage_dir/private/subvol-19254-disk-0/19254",
 415             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
 416             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
 417             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
 418             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
 419             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz",
 420         ],
 421         expected => [], # returns empty list
 422     },
 423 );
 424
 425
 426 # provide static vmlist for tests
 427 my $mock_cluster = Test::MockModule->new('PVE::Cluster', no_auto => 1);
 428 $mock_cluster->redefine(get_vmlist => sub { return $mocked_vmlist; });
 429
 430 # populate is File::stat's method to fill all information from CORE::stat into
 431 # an blessed array.
 432 my $mock_stat = Test::MockModule->new('File::stat', no_auto => 1);
 433 $mock_stat->redefine(populate => sub {
 434         my (@st) = @_;
 435         $st[7] = DEFAULT_SIZE;
 436         $st[10] = DEFAULT_CTIME;
 437
 438         my $result = $mock_stat->original('populate')->(@st);
 439
 440         return $result;
 441 });
 442
 443 # override info provided by qemu-img in file_size_info
 444 my $mock_fsi = Test::MockModule->new('PVE::Storage::Plugin', no_auto => 1);
 445 $mock_fsi->redefine(file_size_info => sub {
 446         my ($filename, $timeout) = @_;
 447         my ($size, $format, $used, $parent, $ctime) = $mock_fsi->original('file_size_info')->($filename, $timeout);
 448
 449         $size = DEFAULT_SIZE;
 450         $used = DEFAULT_USED;
 451
 452         return wantarray ? ($size, $format, $used, $parent, $ctime) : $size;
 453 });
 454
 455 my $plan = scalar @tests;
 456 plan tests => $plan + 1;
 457
 458 {
 459     # don't accidentally modify vmlist, see bug report
 460     # https://pve.proxmox.com/pipermail/pve-devel/2020-January/041096.html
 461     my $scfg_with_type = { path => $storage_dir, type => 'dir' };
 462     my $original_vmlist = { ids => {} };
 463     my $tested_vmlist = dclone($original_vmlist);
 464
 465     PVE::Storage::Plugin->list_volumes('sid', $scfg_with_type, undef, ['images']);
 466
 467     is_deeply ($tested_vmlist, $original_vmlist,
 468         'PVE::Cluster::vmlist remains unmodified')
 469     || diag ("Expected vmlist to remain\n", explain($original_vmlist),
 470         "but it turned to\n", explain($tested_vmlist));
 471 }
 472
 473
 474 {
 475     my $sid = 'local';
 476     my $types = [ 'rootdir', 'images', 'vztmpl', 'iso', 'backup', 'snippets' ];
 477     my @suffixes = ( 'qcow2', 'raw', 'vmdk', 'vhdx' );
 478
 479     # run through test cases
 480     foreach my $tt (@tests) {
 481         my $vmid = $tt->{vmid};
 482         my $files = $tt->{files};
 483         my $expected = $tt->{expected};
 484         my $description = $tt->{description};
 485         my $parent = $tt->{parent};
 486
 487         # prepare environment
 488         my $num = 0; #parent disks
 489         for my $file (@$files) {
 490             my ($name, $dir, $suffix) = fileparse($file, @suffixes);
 491
 492             make_path($dir, { verbose => 1, mode => 0755 });
 493
 494             if ($name) {
 495                 # using qemu-img to also be able to represent the backing device
 496                 my @cmd = ( '/usr/bin/qemu-img', 'create', "$file", DEFAULT_SIZE );
 497                 push @cmd, ( '-f', $suffix ) if $suffix;
 498                 push @cmd, ( '-u', '-b', @$parent[$num] ) if $parent;
 499                 $num++;
 500
 501                 run_command([@cmd]);
 502             }
 503         }
 504
 505         my $got;
 506         eval { $got = PVE::Storage::Plugin->list_volumes($sid, $scfg, $vmid, $types) };
 507         $got = $@ if $@;
 508
 509         is_deeply($got, $expected, $description) || diag(explain($got));
 510
 511         # clean up after each test case, otherwise
 512         # we get wrong results from leftover files
 513         remove_tree($storage_dir, { verbose => 1 });
 514     }
 515 }
 516
 517 done_testing();
 518
 519 1;