PVE/API2/Storage/Status.pm

   1 package PVE::API2::Storage::Status;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use POSIX qw(ENOENT);
   9
  10 use PVE::Cluster;
  11 use PVE::Exception qw(raise_param_exc);
  12 use PVE::INotify;
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::RESTHandler;
  15 use PVE::RPCEnvironment;
  16 use PVE::RRD;
  17 use PVE::Tools qw(run_command);
  18
  19 use PVE::API2::Storage::Content;
  20 use PVE::API2::Storage::FileRestore;
  21 use PVE::API2::Storage::PruneBackups;
  22 use PVE::Storage;
  23
  24 use base qw(PVE::RESTHandler);
  25
  26 __PACKAGE__->register_method ({
  27     subclass => "PVE::API2::Storage::PruneBackups",
  28     path => '{storage}/prunebackups',
  29 });
  30
  31 __PACKAGE__->register_method ({
  32     subclass => "PVE::API2::Storage::Content",
  33     # set fragment delimiter (no subdirs) - we need that, because volume
  34     # IDs may contain a slash '/'
  35     fragmentDelimiter => '',
  36     path => '{storage}/content',
  37 });
  38
  39 __PACKAGE__->register_method ({
  40    subclass => "PVE::API2::Storage::FileRestore",
  41    path => '{storage}/file-restore',
  42 });
  43
  44 __PACKAGE__->register_method ({
  45     name => 'index',
  46     path => '',
  47     method => 'GET',
  48     description => "Get status for all datastores.",
  49     permissions => {
  50         description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
  51         user => 'all',
  52     },
  53     protected => 1,
  54     proxyto => 'node',
  55     parameters => {
  56         additionalProperties => 0,
  57         properties => {
  58             node => get_standard_option('pve-node'),
  59             storage => get_standard_option('pve-storage-id', {
  60                 description => "Only list status for  specified storage",
  61                 optional => 1,
  62                 completion => \&PVE::Storage::complete_storage_enabled,
  63             }),
  64             content => {
  65                 description => "Only list stores which support this content type.",
  66                 type => 'string', format => 'pve-storage-content-list',
  67                 optional => 1,
  68                 completion => \&PVE::Storage::complete_content_type,
  69             },
  70             enabled => {
  71                 description => "Only list stores which are enabled (not disabled in config).",
  72                 type => 'boolean',
  73                 optional => 1,
  74                 default => 0,
  75             },
  76             target => get_standard_option('pve-node', {
  77                 description => "If target is different to 'node', we only lists shared storages which " .
  78                     "content is accessible on this 'node' and the specified 'target' node.",
  79                 optional => 1,
  80                 completion => \&PVE::Cluster::get_nodelist,
  81             }),
  82             'format' => {
  83                 description => "Include information about formats",
  84                 type => 'boolean',
  85                 optional => 1,
  86                 default => 0,
  87             },
  88         },
  89     },
  90     returns => {
  91         type => 'array',
  92         items => {
  93             type => "object",
  94             properties => {
  95                 storage => get_standard_option('pve-storage-id'),
  96                 type => {
  97                     description => "Storage type.",
  98                     type => 'string',
  99                 },
 100                 content => {
 101                     description => "Allowed storage content types.",
 102                     type => 'string', format => 'pve-storage-content-list',
 103                 },
 104                 enabled => {
 105                     description => "Set when storage is enabled (not disabled).",
 106                     type => 'boolean',
 107                     optional => 1,
 108                 },
 109                 active => {
 110                     description => "Set when storage is accessible.",
 111                     type => 'boolean',
 112                     optional => 1,
 113                 },
 114                 shared => {
 115                     description => "Shared flag from storage configuration.",
 116                     type => 'boolean',
 117                     optional => 1,
 118                 },
 119                 total => {
 120                     description => "Total storage space in bytes.",
 121                     type => 'integer',
 122                     renderer => 'bytes',
 123                     optional => 1,
 124                 },
 125                 used => {
 126                     description => "Used storage space in bytes.",
 127                     type => 'integer',
 128                     renderer => 'bytes',
 129                     optional => 1,
 130                 },
 131                 avail => {
 132                     description => "Available storage space in bytes.",
 133                     type => 'integer',
 134                     renderer => 'bytes',
 135                     optional => 1,
 136                 },
 137                 used_fraction => {
 138                     description => "Used fraction (used/total).",
 139                     type => 'number',
 140                     renderer => 'fraction_as_percentage',
 141                     optional => 1,
 142                 },
 143             },
 144         },
 145         links => [ { rel => 'child', href => "{storage}" } ],
 146     },
 147     code => sub {
 148         my ($param) = @_;
 149
 150         my $rpcenv = PVE::RPCEnvironment::get();
 151         my $authuser = $rpcenv->get_user();
 152
 153         my $localnode = PVE::INotify::nodename();
 154
 155         my $target = $param->{target};
 156
 157         undef $target if $target && ($target eq $localnode || $target eq 'localhost');
 158
 159         my $cfg = PVE::Storage::config();
 160
 161         my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
 162
 163         raise_param_exc({ storage => "No such storage." })
 164             if $param->{storage} && !defined($info->{$param->{storage}});
 165
 166         my $res = {};
 167         my @sids = PVE::Storage::storage_ids($cfg);
 168         foreach my $storeid (@sids) {
 169             my $data = $info->{$storeid};
 170             next if !$data;
 171             my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
 172             next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
 173             next if $param->{storage} && $param->{storage} ne $storeid;
 174
 175             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
 176
 177             next if $param->{enabled} && $scfg->{disable};
 178
 179             if ($target) {
 180                 # check if storage content is accessible on local node and specified target node
 181                 # we use this on the Clone GUI
 182
 183                 next if !$scfg->{shared};
 184                 next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
 185                 next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
 186             }
 187
 188             if ($data->{total}) {
 189                 $data->{used_fraction} = ($data->{used} // 0) / $data->{total};
 190             }
 191
 192             $res->{$storeid} = $data;
 193         }
 194
 195         return PVE::RESTHandler::hash_to_array($res, 'storage');
 196     }});
 197
 198 __PACKAGE__->register_method ({
 199     name => 'diridx',
 200     path => '{storage}',
 201     method => 'GET',
 202     description => "",
 203     permissions => {
 204         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 205     },
 206     parameters => {
 207         additionalProperties => 0,
 208         properties => {
 209             node => get_standard_option('pve-node'),
 210             storage => get_standard_option('pve-storage-id'),
 211         },
 212     },
 213     returns => {
 214         type => 'array',
 215         items => {
 216             type => "object",
 217             properties => {
 218                 subdir => { type => 'string' },
 219             },
 220         },
 221         links => [ { rel => 'child', href => "{subdir}" } ],
 222     },
 223     code => sub {
 224         my ($param) = @_;
 225
 226         my $res = [
 227             { subdir => 'content' },
 228             { subdir => 'download-url' },
 229             { subdir => 'file-restore' },
 230             { subdir => 'prunebackups' },
 231             { subdir => 'rrd' },
 232             { subdir => 'rrddata' },
 233             { subdir => 'status' },
 234             { subdir => 'upload' },
 235         ];
 236
 237         return $res;
 238     }});
 239
 240 __PACKAGE__->register_method ({
 241     name => 'read_status',
 242     path => '{storage}/status',
 243     method => 'GET',
 244     description => "Read storage status.",
 245     permissions => {
 246         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 247     },
 248     protected => 1,
 249     proxyto => 'node',
 250     parameters => {
 251         additionalProperties => 0,
 252         properties => {
 253             node => get_standard_option('pve-node'),
 254             storage => get_standard_option('pve-storage-id'),
 255         },
 256     },
 257     returns => {
 258         type => "object",
 259         properties => {},
 260     },
 261     code => sub {
 262         my ($param) = @_;
 263
 264         my $cfg = PVE::Storage::config();
 265
 266         my $info = PVE::Storage::storage_info($cfg, $param->{content});
 267
 268         my $data = $info->{$param->{storage}};
 269
 270         raise_param_exc({ storage => "No such storage." })
 271             if !defined($data);
 272
 273         return $data;
 274     }});
 275
 276 __PACKAGE__->register_method ({
 277     name => 'rrd',
 278     path => '{storage}/rrd',
 279     method => 'GET',
 280     description => "Read storage RRD statistics (returns PNG).",
 281     permissions => {
 282         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 283     },
 284     protected => 1,
 285     proxyto => 'node',
 286     parameters => {
 287         additionalProperties => 0,
 288         properties => {
 289             node => get_standard_option('pve-node'),
 290             storage => get_standard_option('pve-storage-id'),
 291             timeframe => {
 292                 description => "Specify the time frame you are interested in.",
 293                 type => 'string',
 294                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 295             },
 296             ds => {
 297                 description => "The list of datasources you want to display.",
 298                 type => 'string', format => 'pve-configid-list',
 299             },
 300             cf => {
 301                 description => "The RRD consolidation function",
 302                 type => 'string',
 303                 enum => [ 'AVERAGE', 'MAX' ],
 304                 optional => 1,
 305             },
 306         },
 307     },
 308     returns => {
 309         type => "object",
 310         properties => {
 311             filename => { type => 'string' },
 312         },
 313     },
 314     code => sub {
 315         my ($param) = @_;
 316
 317         return PVE::RRD::create_rrd_graph(
 318             "pve2-storage/$param->{node}/$param->{storage}",
 319             $param->{timeframe}, $param->{ds}, $param->{cf});
 320     }});
 321
 322 __PACKAGE__->register_method ({
 323     name => 'rrddata',
 324     path => '{storage}/rrddata',
 325     method => 'GET',
 326     description => "Read storage RRD statistics.",
 327     permissions => {
 328         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 329     },
 330     protected => 1,
 331     proxyto => 'node',
 332     parameters => {
 333         additionalProperties => 0,
 334         properties => {
 335             node => get_standard_option('pve-node'),
 336             storage => get_standard_option('pve-storage-id'),
 337             timeframe => {
 338                 description => "Specify the time frame you are interested in.",
 339                 type => 'string',
 340                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 341             },
 342             cf => {
 343                 description => "The RRD consolidation function",
 344                 type => 'string',
 345                 enum => [ 'AVERAGE', 'MAX' ],
 346                 optional => 1,
 347             },
 348         },
 349     },
 350     returns => {
 351         type => "array",
 352         items => {
 353             type => "object",
 354             properties => {},
 355         },
 356     },
 357     code => sub {
 358         my ($param) = @_;
 359
 360         return PVE::RRD::create_rrd_data(
 361             "pve2-storage/$param->{node}/$param->{storage}",
 362             $param->{timeframe}, $param->{cf});
 363     }});
 364
 365 # makes no sense for big images and backup files (because it
 366 # create a copy of the file).
 367 __PACKAGE__->register_method ({
 368     name => 'upload',
 369     path => '{storage}/upload',
 370     method => 'POST',
 371     description => "Upload templates and ISO images.",
 372     permissions => {
 373         check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
 374     },
 375     protected => 1,
 376     parameters => {
 377         additionalProperties => 0,
 378         properties => {
 379             node => get_standard_option('pve-node'),
 380             storage => get_standard_option('pve-storage-id'),
 381             content => {
 382                 description => "Content type.",
 383                 type => 'string', format => 'pve-storage-content',
 384                 enum => ['iso', 'vztmpl'],
 385             },
 386             filename => {
 387                 description => "The name of the file to create. Caution: This will be normalized!",
 388                 maxLength => 255,
 389                 type => 'string',
 390             },
 391             checksum => {
 392                 description => "The expected checksum of the file.",
 393                 type => 'string',
 394                 requires => 'checksum-algorithm',
 395                 optional => 1,
 396             },
 397             'checksum-algorithm' => {
 398                 description => "The algorithm to calculate the checksum of the file.",
 399                 type => 'string',
 400                 enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
 401                 requires => 'checksum',
 402                 optional => 1,
 403             },
 404             tmpfilename => {
 405                 description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
 406                 type => 'string',
 407                 optional => 1,
 408             },
 409         },
 410     },
 411     returns => { type => "string" },
 412     code => sub {
 413         my ($param) = @_;
 414
 415         my $rpcenv = PVE::RPCEnvironment::get();
 416
 417         my $user = $rpcenv->get_user();
 418
 419         my $cfg = PVE::Storage::config();
 420
 421         my $node = $param->{node};
 422         my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
 423
 424         die "can't upload to storage type '$scfg->{type}'\n"
 425             if !defined($scfg->{path});
 426
 427         my $content = $param->{content};
 428
 429         my $tmpfilename = $param->{tmpfilename};
 430         die "missing temporary file name\n" if !$tmpfilename;
 431
 432         my $size = -s $tmpfilename;
 433         die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
 434
 435         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 436
 437         my $path;
 438
 439         if ($content eq 'iso') {
 440             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 441                 raise_param_exc({ filename => "wrong file extension" });
 442             }
 443             $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
 444         } elsif ($content eq 'vztmpl') {
 445             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 446                 raise_param_exc({ filename => "wrong file extension" });
 447             }
 448             $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
 449         } else {
 450             raise_param_exc({ content => "upload content type '$content' not allowed" });
 451         }
 452
 453         die "storage '$param->{storage}' does not support '$content' content\n"
 454             if !$scfg->{content}->{$content};
 455
 456         my $dest = "$path/$filename";
 457         my $dirname = dirname($dest);
 458
 459         # best effort to match apl_download behaviour
 460         chmod 0644, $tmpfilename;
 461
 462         my $err_cleanup = sub { unlink $dest, $tmpfilename; die "cleanup failed: $!" if $! && $! != ENOENT };
 463
 464         my $cmd;
 465         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
 466             my $remip = PVE::Cluster::remote_node_ip($node);
 467
 468             my @ssh_options = ('-o', 'BatchMode=yes');
 469
 470             my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
 471
 472             eval { # activate remote storage
 473                 run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]);
 474             };
 475             die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
 476
 477             run_command(
 478                 [@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
 479                 errmsg => "mkdir failed",
 480             );
 481
 482             $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
 483
 484             $err_cleanup = sub { run_command([@remcmd, 'rm', '-f', '--', $dest, $tmpfilename]) };
 485         } else {
 486             PVE::Storage::activate_storage($cfg, $param->{storage});
 487             File::Path::make_path($dirname);
 488             $cmd = ['cp', '--', $tmpfilename, $dest];
 489         }
 490
 491         # NOTE: we simply overwrite the destination file if it already exists
 492         my $worker = sub {
 493             my $upid = shift;
 494
 495             print "starting file import from: $tmpfilename\n";
 496
 497             eval {
 498                 my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
 499                 if ($checksum_algorithm) {
 500                     print "calculating checksum...";
 501
 502                     my $checksum_got = PVE::Tools::get_file_hash($checksum_algorithm, $tmpfilename);
 503
 504                     if (lc($checksum_got) eq lc($checksum)) {
 505                         print "OK, checksum verified\n";
 506                     } else {
 507                         print "\n";  # the front end expects the error to reside at the last line without any noise
 508                         die "checksum mismatch: got '$checksum_got' != expect '$checksum'\n";
 509                     }
 510                 }
 511             };
 512             if (my $err = $@) {
 513                 # unlinks only the temporary file from the http server
 514                 unlink $tmpfilename;
 515                 warn "unable to clean up temporory file '$tmpfilename' - $!\n"
 516                     if $! && $! != ENOENT;
 517                 die $err;
 518             }
 519
 520             print "target node: $node\n";
 521             print "target file: $dest\n";
 522             print "file size is: $size\n";
 523             print "command: " . join(' ', @$cmd) . "\n";
 524
 525             eval { run_command($cmd, errmsg => 'import failed'); };
 526
 527             unlink $tmpfilename; # the temporary file got only uploaded locally, no need to rm remote
 528             warn "unable to clean up temporary file '$tmpfilename' - $!\n" if $! && $! != ENOENT;
 529
 530             if (my $err = $@) {
 531                 eval { $err_cleanup->() };
 532                 warn "$@" if $@;
 533                 die $err;
 534             }
 535             print "finished file import successfully\n";
 536         };
 537
 538         return $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
 539    }});
 540
 541 __PACKAGE__->register_method({
 542     name => 'download_url',
 543     path => '{storage}/download-url',
 544     method => 'POST',
 545     description => "Download templates and ISO images by using an URL.",
 546     proxyto => 'node',
 547     permissions => {
 548         check => [ 'and',
 549             ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
 550             ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
 551         ],
 552     },
 553     protected => 1,
 554     parameters => {
 555         additionalProperties => 0,
 556         properties => {
 557             node => get_standard_option('pve-node'),
 558             storage => get_standard_option('pve-storage-id'),
 559             url => {
 560                 description => "The URL to download the file from.",
 561                 type => 'string',
 562                 pattern => 'https?://.*',
 563             },
 564             content => {
 565                 description => "Content type.", # TODO: could be optional & detected in most cases
 566                 type => 'string', format => 'pve-storage-content',
 567                 enum => ['iso', 'vztmpl'],
 568             },
 569             filename => {
 570                 description => "The name of the file to create. Caution: This will be normalized!",
 571                 maxLength => 255,
 572                 type => 'string',
 573             },
 574             checksum => {
 575                 description => "The expected checksum of the file.",
 576                 type => 'string',
 577                 requires => 'checksum-algorithm',
 578                 optional => 1,
 579             },
 580             'checksum-algorithm' => {
 581                 description => "The algorithm to calculate the checksum of the file.",
 582                 type => 'string',
 583                 enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
 584                 requires => 'checksum',
 585                 optional => 1,
 586             },
 587             'verify-certificates' => {
 588                 description => "If false, no SSL/TLS certificates will be verified.",
 589                 type => 'boolean',
 590                 optional => 1,
 591                 default => 1,
 592             },
 593         },
 594     },
 595     returns => {
 596         type => "string"
 597     },
 598     code => sub {
 599         my ($param) = @_;
 600
 601         my $rpcenv = PVE::RPCEnvironment::get();
 602         my $user = $rpcenv->get_user();
 603
 604         my $cfg = PVE::Storage::config();
 605
 606         my ($node, $storage) = $param->@{'node', 'storage'};
 607         my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 608
 609         die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
 610             if !defined($scfg->{path});
 611
 612         my ($content, $url) = $param->@{'content', 'url'};
 613
 614         die "storage '$storage' is not configured for content-type '$content'\n"
 615             if !$scfg->{content}->{$content};
 616
 617         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 618
 619         my $path;
 620         if ($content eq 'iso') {
 621             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 622                 raise_param_exc({ filename => "wrong file extension" });
 623             }
 624             $path = PVE::Storage::get_iso_dir($cfg, $storage);
 625         } elsif ($content eq 'vztmpl') {
 626             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 627                 raise_param_exc({ filename => "wrong file extension" });
 628             }
 629             $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
 630         } else {
 631             raise_param_exc({ content => "upload content-type '$content' is not allowed" });
 632         }
 633
 634         PVE::Storage::activate_storage($cfg, $storage);
 635         File::Path::make_path($path);
 636
 637         my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
 638         my $opts = {
 639             hash_required => 0,
 640             verify_certificates => $param->{'verify-certificates'} // 1,
 641             http_proxy => $dccfg->{http_proxy},
 642         };
 643
 644         my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
 645         if ($checksum) {
 646             $opts->{"${checksum_algorithm}sum"} = $checksum;
 647             $opts->{hash_required} = 1;
 648         }
 649
 650         my $worker = sub {
 651             PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 652         };
 653
 654         my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID
 655
 656         return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
 657     }});
 658
 659 1;