test/list_volumes_test.pm

   1 package PVE::Storage::TestListVolumes;
   2
   3 use strict;
   4 use warnings;
   5
   6 use lib qw(..);
   7
   8 use PVE::Storage;
   9 use PVE::Cluster;
  10 use PVE::Tools qw(run_command);
  11
  12 use Test::More;
  13 use Test::MockModule;
  14
  15 use Cwd;
  16 use File::Basename;
  17 use File::Path qw(make_path remove_tree);
  18 use File::stat qw();
  19 use File::Temp;
  20 use Storable qw(dclone);
  21
  22 use constant DEFAULT_SIZE => 131072; # 128 kiB
  23 use constant DEFAULT_USED => 262144; # 256 kiB
  24 use constant DEFAULT_CTIME => 1234567890;
  25
  26 # get_vmlist() return values
  27 my $mocked_vmlist = {
  28     'version' => 1,
  29     'ids' => {
  30         '16110' => {
  31             'node'    => 'x42',
  32             'type'    => 'qemu',
  33             'version' => 4,
  34         },
  35         '16112' => {
  36             'node'    => 'x42',
  37             'type'    => 'lxc',
  38             'version' => 7,
  39         },
  40         '16114' => {
  41             'node'    => 'x42',
  42             'type'    => 'qemu',
  43             'version' => 2,
  44         },
  45         '16113' => {
  46             'node'    => 'x42',
  47             'type'    => 'qemu',
  48             'version' => 5,
  49         },
  50         '16115' => {
  51             'node'    => 'x42',
  52             'type'    => 'qemu',
  53             'version' => 1,
  54         },
  55         '9004' => {
  56             'node'    => 'x42',
  57             'type'    => 'qemu',
  58             'version' => 6,
  59         }
  60     }
  61 };
  62
  63 my $storage_dir = File::Temp->newdir();
  64 my $scfg = {
  65     'type'     => 'dir',
  66     'maxfiles' => 0,
  67     'path'     => $storage_dir,
  68     'shared'   => 0,
  69     'content'  => {
  70         'iso'      => 1,
  71         'rootdir'  => 1,
  72         'vztmpl'   => 1,
  73         'images'   => 1,
  74         'snippets' => 1,
  75         'backup'   => 1,
  76     },
  77 };
  78
  79 # The test cases are comprised of an arry of hashes with the following keys:
  80 # description => displayed on error by Test::More
  81 # vmid        => used for image matches by list_volume
  82 # files       => array of files for qemu-img to create
  83 # expected    => returned result hash
  84 #                (content, ctime, format, parent, size, used, vimd, volid)
  85 my @tests = (
  86     {
  87         description => 'VMID: 16110, VM, qcow2, backup, snippets',
  88         vmid => '16110',
  89         files => [
  90             "$storage_dir/images/16110/vm-16110-disk-0.qcow2",
  91             "$storage_dir/images/16110/vm-16110-disk-1.raw",
  92             "$storage_dir/images/16110/vm-16110-disk-2.vmdk",
  93             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
  94             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
  95             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
  96             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
  97             "$storage_dir/snippets/userconfig.yaml",
  98             "$storage_dir/snippets/hookscript.pl",
  99         ],
 100         expected => [
 101             {
 102                 'content' => 'images',
 103                 'ctime'   => DEFAULT_CTIME,
 104                 'format'  => 'qcow2',
 105                 'parent'  => undef,
 106                 'size'    => DEFAULT_SIZE,
 107                 'used'    => DEFAULT_USED,
 108                 'vmid'    => '16110',
 109                 'volid'   => 'local:16110/vm-16110-disk-0.qcow2',
 110             },
 111             {
 112                 'content' => 'images',
 113                 'ctime'   => DEFAULT_CTIME,
 114                 'format'  => 'raw',
 115                 'parent'  => undef,
 116                 'size'    => DEFAULT_SIZE,
 117                 'used'    => DEFAULT_USED,
 118                 'vmid'    => '16110',
 119                 'volid'   => 'local:16110/vm-16110-disk-1.raw',
 120             },
 121             {
 122                 'content' => 'images',
 123                 'ctime'   => DEFAULT_CTIME,
 124                 'format'  => 'vmdk',
 125                 'parent'  => undef,
 126                 'size'    => DEFAULT_SIZE,
 127                 'used'    => DEFAULT_USED,
 128                 'vmid'    => '16110',
 129                 'volid'   => 'local:16110/vm-16110-disk-2.vmdk',
 130             },
 131             {
 132                 'content' => 'backup',
 133                 'ctime'   => 1585602700,
 134                 'format'  => 'vma.gz',
 135                 'size'    => DEFAULT_SIZE,
 136                 'subtype' => 'qemu',
 137                 'vmid'    => '16110',
 138                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz',
 139             },
 140             {
 141                 'content' => 'backup',
 142                 'ctime'   => 1585602765,
 143                 'format'  => 'vma.lzo',
 144                 'size'    => DEFAULT_SIZE,
 145                 'subtype' => 'qemu',
 146                 'vmid'    => '16110',
 147                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo',
 148             },
 149             {
 150                 'content' => 'backup',
 151                 'ctime'   => 1585602835,
 152                 'format'  => 'vma',
 153                 'size'    => DEFAULT_SIZE,
 154                 'subtype' => 'qemu',
 155                 'vmid'    => '16110',
 156                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma',
 157             },
 158             {
 159                 'content' => 'backup',
 160                 'ctime'   => 1585602835,
 161                 'format'  => 'vma.zst',
 162                 'size'    => DEFAULT_SIZE,
 163                 'subtype' => 'qemu',
 164                 'vmid'    => '16110',
 165                 'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst',
 166             },
 167             {
 168                 'content' => 'snippets',
 169                 'ctime'   => DEFAULT_CTIME,
 170                 'format'  => 'snippet',
 171                 'size'    => DEFAULT_SIZE,
 172                 'volid'   => 'local:snippets/hookscript.pl',
 173             },
 174             {
 175                 'content' => 'snippets',
 176                 'ctime'   => DEFAULT_CTIME,
 177                 'format'  => 'snippet',
 178                 'size'    => DEFAULT_SIZE,
 179                 'volid'   => 'local:snippets/userconfig.yaml',
 180             },
 181         ],
 182     },
 183     {
 184         description => 'VMID: 16112, lxc, raw, backup',
 185         vmid => '16112',
 186         files => [
 187             "$storage_dir/images/16112/vm-16112-disk-0.raw",
 188             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
 189             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
 190             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst",
 191             "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
 192         ],
 193         expected => [
 194             {
 195                 'content' => 'rootdir',
 196                 'ctime'   => DEFAULT_CTIME,
 197                 'format'  => 'raw',
 198                 'parent'  => undef,
 199                 'size'    => DEFAULT_SIZE,
 200                 'used'    => DEFAULT_USED,
 201                 'vmid'    => '16112',
 202                 'volid'   => 'local:16112/vm-16112-disk-0.raw',
 203             },
 204             {
 205                 'content' => 'backup',
 206                 'ctime'   => 1585604370,
 207                 'format'  => 'tar.lzo',
 208                 'size'    => DEFAULT_SIZE,
 209                 'subtype' => 'lxc',
 210                 'vmid'    => '16112',
 211                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo',
 212             },
 213             {
 214                 'content' => 'backup',
 215                 'ctime'   => 1585604970,
 216                 'format'  => 'tar.gz',
 217                 'size'    => DEFAULT_SIZE,
 218                 'subtype' => 'lxc',
 219                 'vmid'    => '16112',
 220                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz',
 221             },
 222             {
 223                 'content' => 'backup',
 224                 'ctime'   => 1585604970,
 225                 'format'  => 'tar.zst',
 226                 'size'    => DEFAULT_SIZE,
 227                 'subtype' => 'lxc',
 228                 'vmid'    => '16112',
 229                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst',
 230             },
 231             {
 232                 'content' => 'backup',
 233                 'ctime'   => 1585605570,
 234                 'format'  => 'tgz',
 235                 'size'    => DEFAULT_SIZE,
 236                 'subtype' => 'lxc',
 237                 'vmid'    => '16112',
 238                 'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz',
 239             },
 240         ],
 241     },
 242     {
 243         description => 'VMID: 16114, VM, qcow2, linked clone',
 244         vmid => '16114',
 245         files => [
 246             "$storage_dir/images/16114/vm-16114-disk-0.qcow2",
 247             "$storage_dir/images/16114/vm-16114-disk-1.qcow2",
 248         ],
 249         parent => [
 250             "../9004/base-9004-disk-0.qcow2",
 251             "../9004/base-9004-disk-1.qcow2",
 252         ],
 253         expected => [
 254             {
 255                 'content' => 'images',
 256                 'ctime'   => DEFAULT_CTIME,
 257                 'format'  => 'qcow2',
 258                 'parent'  => '../9004/base-9004-disk-0.qcow2',
 259                 'size'    => DEFAULT_SIZE,
 260                 'used'    => DEFAULT_USED,
 261                 'vmid'    => '16114',
 262                 'volid'   => 'local:9004/base-9004-disk-0.qcow2/16114/vm-16114-disk-0.qcow2',
 263             },
 264             {
 265                 'content' => 'images',
 266                 'ctime'   => DEFAULT_CTIME,
 267                 'format'  => 'qcow2',
 268                 'parent'  => '../9004/base-9004-disk-1.qcow2',
 269                 'size'    => DEFAULT_SIZE,
 270                 'used'    => DEFAULT_USED,
 271                 'vmid'    => '16114',
 272                 'volid'   => 'local:9004/base-9004-disk-1.qcow2/16114/vm-16114-disk-1.qcow2',
 273             },
 274         ],
 275     },
 276     {
 277         description => 'VMID: 9004, VM, template, qcow2',
 278         vmid => '9004',
 279         files => [
 280             "$storage_dir/images/9004/base-9004-disk-0.qcow2",
 281             "$storage_dir/images/9004/base-9004-disk-1.qcow2",
 282         ],
 283         expected => [
 284             {
 285                 'content' => 'images',
 286                 'ctime'   => DEFAULT_CTIME,
 287                 'format'  => 'qcow2',
 288                 'parent'  => undef,
 289                 'size'    => DEFAULT_SIZE,
 290                 'used'    => DEFAULT_USED,
 291                 'vmid'    => '9004',
 292                 'volid'   => 'local:9004/base-9004-disk-0.qcow2',
 293             },
 294             {
 295                 'content' => 'images',
 296                 'ctime'   => DEFAULT_CTIME,
 297                 'format'  => 'qcow2',
 298                 'parent'  => undef,
 299                 'size'    => DEFAULT_SIZE,
 300                 'used'    => DEFAULT_USED,
 301                 'vmid'    => '9004',
 302                 'volid'   => 'local:9004/base-9004-disk-1.qcow2',
 303             },
 304         ],
 305     },
 306     {
 307         description => 'VMID: none, templates, snippets, backup',
 308         vmid => undef,
 309         files => [
 310             "$storage_dir/dump/vzdump-lxc-19253-2020_02_03-19_57_43.tar.gz",
 311             "$storage_dir/dump/vzdump-lxc-19254-2019_01_21-19_29_19.tar",
 312             "$storage_dir/template/iso/archlinux-2020.02.01-x86_64.iso",
 313             "$storage_dir/template/iso/debian-8.11.1-amd64-DVD-1.iso",
 314             "$storage_dir/template/iso/debian-9.12.0-amd64-netinst.iso",
 315             "$storage_dir/template/iso/proxmox-ve_6.1-1.iso",
 316             "$storage_dir/template/cache/archlinux-base_20190924-1_amd64.tar.gz",
 317             "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz",
 318             "$storage_dir/template/cache/alpine-3.10-default_20190626_amd64.tar.xz",
 319             "$storage_dir/snippets/userconfig.yaml",
 320             "$storage_dir/snippets/hookscript.pl",
 321             "$storage_dir/private/1234/", # fileparse needs / at the end
 322             "$storage_dir/private/1234/subvol-1234-disk-0.subvol/", # fileparse needs / at the end
 323         ],
 324         expected => [
 325             {
 326                 'content' => 'vztmpl',
 327                 'ctime'   => DEFAULT_CTIME,
 328                 'format'  => 'txz',
 329                 'size'    => DEFAULT_SIZE,
 330                 'volid'   => 'local:vztmpl/alpine-3.10-default_20190626_amd64.tar.xz',
 331             },
 332             {
 333                 'content' => 'vztmpl',
 334                 'ctime'   => DEFAULT_CTIME,
 335                 'format'  => 'tgz',
 336                 'size'    => DEFAULT_SIZE,
 337                 'volid'   => 'local:vztmpl/archlinux-base_20190924-1_amd64.tar.gz',
 338             },
 339             {
 340                 'content' => 'vztmpl',
 341                 'ctime'   => DEFAULT_CTIME,
 342                 'format'  => 'tgz',
 343                 'size'    => DEFAULT_SIZE,
 344                 'volid'   => 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
 345             },
 346             {
 347                 'content' => 'iso',
 348                 'ctime'   => DEFAULT_CTIME,
 349                 'format'  => 'iso',
 350                 'size'    => DEFAULT_SIZE,
 351                 'volid'   => 'local:iso/archlinux-2020.02.01-x86_64.iso',
 352             },
 353             {
 354                 'content' => 'iso',
 355                 'ctime'   => DEFAULT_CTIME,
 356                 'format'  => 'iso',
 357                 'size'    => DEFAULT_SIZE,
 358                 'volid'   => 'local:iso/debian-8.11.1-amd64-DVD-1.iso',
 359             },
 360             {
 361                 'content' => 'iso',
 362                 'ctime'   => DEFAULT_CTIME,
 363                 'format'  => 'iso',
 364                 'size'    => DEFAULT_SIZE,
 365                 'volid'   => 'local:iso/debian-9.12.0-amd64-netinst.iso',
 366             },
 367             {
 368                 'content' => 'iso',
 369                 'ctime'   => DEFAULT_CTIME,
 370                 'format'  => 'iso',
 371                 'size'    => DEFAULT_SIZE,
 372                 'volid'   => 'local:iso/proxmox-ve_6.1-1.iso',
 373             },
 374             {
 375                 'content' => 'backup',
 376                 'ctime'   => 1580759863,
 377                 'format'  => 'tar.gz',
 378                 'size'    => DEFAULT_SIZE,
 379                 'subtype' => 'lxc',
 380                 'vmid'    => '19253',
 381                 'volid'   => 'local:backup/vzdump-lxc-19253-2020_02_03-19_57_43.tar.gz',
 382             },
 383             {
 384                 'content' => 'backup',
 385                 'ctime'   => 1548098959,
 386                 'format'  => 'tar',
 387                 'size'    => DEFAULT_SIZE,
 388                 'subtype' => 'lxc',
 389                 'vmid'    => '19254',
 390                 'volid'   => 'local:backup/vzdump-lxc-19254-2019_01_21-19_29_19.tar',
 391             },
 392             {
 393                 'content' => 'snippets',
 394                 'ctime'   => DEFAULT_CTIME,
 395                 'format'  => 'snippet',
 396                 'size'    => DEFAULT_SIZE,
 397                 'volid'   => 'local:snippets/hookscript.pl',
 398             },
 399             {
 400                 'content' => 'snippets',
 401                 'ctime'   => DEFAULT_CTIME,
 402                 'format'  => 'snippet',
 403                 'size'    => DEFAULT_SIZE,
 404                 'volid'   => 'local:snippets/userconfig.yaml',
 405             },
 406         ],
 407     },
 408     {
 409         description => 'VMID: none, parent, non-matching',
 410         # string instead of vmid in folder
 411         #"$storage_dir/images/ssss/base-4321-disk-0.qcow2/1234/vm-1234-disk-0.qcow2",
 412         vmid => undef,
 413         files => [
 414             "$storage_dir/images/1234/vm-1234-disk-0.qcow2",
 415         ],
 416         parent => [
 417             "../ssss/base-4321-disk-0.qcow2",
 418         ],
 419         expected => [
 420             {
 421                 'content' => 'images',
 422                 'ctime'   => DEFAULT_CTIME,
 423                 'format'  => 'qcow2',
 424                 'parent'  => '../ssss/base-4321-disk-0.qcow2',
 425                 'size'    => DEFAULT_SIZE,
 426                 'used'    => DEFAULT_USED,
 427                 'vmid'    => '1234',
 428                 'volid'   => 'local:1234/vm-1234-disk-0.qcow2',
 429             }
 430         ],
 431     },
 432     {
 433         description => 'VMID: none, non-matching',
 434         # failed matches
 435         vmid => undef,
 436         files => [
 437             "$storage_dir/images/ssss/base-4321-disk-0.raw",
 438             "$storage_dir/images/ssss/vm-1234-disk-0.qcow2",
 439             "$storage_dir/template/iso/yet-again-a-installation-disk.dvd",
 440             "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz",
 441             "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2",
 442             "$storage_dir/private/subvol-19254-disk-0/19254",
 443             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
 444             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
 445             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
 446             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
 447             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz",
 448         ],
 449         expected => [], # returns empty list
 450     },
 451 );
 452
 453
 454 # provide static vmlist for tests
 455 my $mock_cluster = Test::MockModule->new('PVE::Cluster', no_auto => 1);
 456 $mock_cluster->redefine(get_vmlist => sub { return $mocked_vmlist; });
 457
 458 # populate is File::stat's method to fill all information from CORE::stat into
 459 # an blessed array.
 460 my $mock_stat = Test::MockModule->new('File::stat', no_auto => 1);
 461 $mock_stat->redefine(populate => sub {
 462         my (@st) = @_;
 463         $st[7] = DEFAULT_SIZE;
 464         $st[10] = DEFAULT_CTIME;
 465
 466         my $result = $mock_stat->original('populate')->(@st);
 467
 468         return $result;
 469 });
 470
 471 # override info provided by qemu-img in file_size_info
 472 my $mock_fsi = Test::MockModule->new('PVE::Storage::Plugin', no_auto => 1);
 473 $mock_fsi->redefine(file_size_info => sub {
 474         my ($filename, $timeout) = @_;
 475         my ($size, $format, $used, $parent, $ctime) = $mock_fsi->original('file_size_info')->($filename, $timeout);
 476
 477         $size = DEFAULT_SIZE;
 478         $used = DEFAULT_USED;
 479
 480         return wantarray ? ($size, $format, $used, $parent, $ctime) : $size;
 481 });
 482
 483 my $plan = scalar @tests;
 484 plan tests => $plan + 1;
 485
 486 {
 487     # don't accidentally modify vmlist, see bug report
 488     # https://pve.proxmox.com/pipermail/pve-devel/2020-January/041096.html
 489     my $scfg_with_type = { path => $storage_dir, type => 'dir' };
 490     my $original_vmlist = { ids => {} };
 491     my $tested_vmlist = dclone($original_vmlist);
 492
 493     PVE::Storage::Plugin->list_volumes('sid', $scfg_with_type, undef, ['images']);
 494
 495     is_deeply ($tested_vmlist, $original_vmlist,
 496         'PVE::Cluster::vmlist remains unmodified')
 497     || diag ("Expected vmlist to remain\n", explain($original_vmlist),
 498         "but it turned to\n", explain($tested_vmlist));
 499 }
 500
 501
 502 {
 503     my $sid = 'local';
 504     my $types = [ 'rootdir', 'images', 'vztmpl', 'iso', 'backup', 'snippets' ];
 505     my @suffixes = ( 'qcow2', 'raw', 'vmdk', 'vhdx' );
 506
 507     # run through test cases
 508     foreach my $tt (@tests) {
 509         my $vmid = $tt->{vmid};
 510         my $files = $tt->{files};
 511         my $expected = $tt->{expected};
 512         my $description = $tt->{description};
 513         my $parent = $tt->{parent};
 514
 515         # prepare environment
 516         my $num = 0; #parent disks
 517         for my $file (@$files) {
 518             my ($name, $dir, $suffix) = fileparse($file, @suffixes);
 519
 520             make_path($dir, { verbose => 1, mode => 0755 });
 521
 522             if ($name) {
 523                 # using qemu-img to also be able to represent the backing device
 524                 my @cmd = ( '/usr/bin/qemu-img', 'create', "$file", DEFAULT_SIZE );
 525                 push @cmd, ( '-f', $suffix ) if $suffix;
 526                 push @cmd, ( '-u', '-b', @$parent[$num] ) if $parent;
 527                 push @cmd, ( '-F', $suffix ) if $parent && $suffix;
 528                 $num++;
 529
 530                 run_command([@cmd]);
 531             }
 532         }
 533
 534         my $got;
 535         eval { $got = PVE::Storage::Plugin->list_volumes($sid, $scfg, $vmid, $types) };
 536         $got = $@ if $@;
 537
 538         is_deeply($got, $expected, $description) || diag(explain($got));
 539
 540         # clean up after each test case, otherwise
 541         # we get wrong results from leftover files
 542         remove_tree($storage_dir, { verbose => 1 });
 543     }
 544 }
 545
 546 done_testing();
 547
 548 1;