PVE/API2/Storage/Status.pm

   1 package PVE::API2::Storage::Status;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use POSIX qw(ENOENT);
   9
  10 use PVE::Cluster;
  11 use PVE::Exception qw(raise_param_exc);
  12 use PVE::INotify;
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::RESTHandler;
  15 use PVE::RPCEnvironment;
  16 use PVE::RRD;
  17 use PVE::Tools qw(run_command);
  18
  19 use PVE::API2::Storage::Content;
  20 use PVE::API2::Storage::FileRestore;
  21 use PVE::API2::Storage::PruneBackups;
  22 use PVE::Storage;
  23
  24 use base qw(PVE::RESTHandler);
  25
  26 __PACKAGE__->register_method ({
  27     subclass => "PVE::API2::Storage::PruneBackups",
  28     path => '{storage}/prunebackups',
  29 });
  30
  31 __PACKAGE__->register_method ({
  32     subclass => "PVE::API2::Storage::Content",
  33     # set fragment delimiter (no subdirs) - we need that, because volume
  34     # IDs may contain a slash '/'
  35     fragmentDelimiter => '',
  36     path => '{storage}/content',
  37 });
  38
  39 __PACKAGE__->register_method ({
  40    subclass => "PVE::API2::Storage::FileRestore",
  41    path => '{storage}/file-restore',
  42 });
  43
  44 __PACKAGE__->register_method ({
  45     name => 'index',
  46     path => '',
  47     method => 'GET',
  48     description => "Get status for all datastores.",
  49     permissions => {
  50         description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
  51         user => 'all',
  52     },
  53     protected => 1,
  54     proxyto => 'node',
  55     parameters => {
  56         additionalProperties => 0,
  57         properties => {
  58             node => get_standard_option('pve-node'),
  59             storage => get_standard_option('pve-storage-id', {
  60                 description => "Only list status for  specified storage",
  61                 optional => 1,
  62                 completion => \&PVE::Storage::complete_storage_enabled,
  63             }),
  64             content => {
  65                 description => "Only list stores which support this content type.",
  66                 type => 'string', format => 'pve-storage-content-list',
  67                 optional => 1,
  68                 completion => \&PVE::Storage::complete_content_type,
  69             },
  70             enabled => {
  71                 description => "Only list stores which are enabled (not disabled in config).",
  72                 type => 'boolean',
  73                 optional => 1,
  74                 default => 0,
  75             },
  76             target => get_standard_option('pve-node', {
  77                 description => "If target is different to 'node', we only lists shared storages which " .
  78                     "content is accessible on this 'node' and the specified 'target' node.",
  79                 optional => 1,
  80                 completion => \&PVE::Cluster::get_nodelist,
  81             }),
  82             'format' => {
  83                 description => "Include information about formats",
  84                 type => 'boolean',
  85                 optional => 1,
  86                 default => 0,
  87             },
  88         },
  89     },
  90     returns => {
  91         type => 'array',
  92         items => {
  93             type => "object",
  94             properties => {
  95                 storage => get_standard_option('pve-storage-id'),
  96                 type => {
  97                     description => "Storage type.",
  98                     type => 'string',
  99                 },
 100                 content => {
 101                     description => "Allowed storage content types.",
 102                     type => 'string', format => 'pve-storage-content-list',
 103                 },
 104                 enabled => {
 105                     description => "Set when storage is enabled (not disabled).",
 106                     type => 'boolean',
 107                     optional => 1,
 108                 },
 109                 active => {
 110                     description => "Set when storage is accessible.",
 111                     type => 'boolean',
 112                     optional => 1,
 113                 },
 114                 shared => {
 115                     description => "Shared flag from storage configuration.",
 116                     type => 'boolean',
 117                     optional => 1,
 118                 },
 119                 total => {
 120                     description => "Total storage space in bytes.",
 121                     type => 'integer',
 122                     renderer => 'bytes',
 123                     optional => 1,
 124                 },
 125                 used => {
 126                     description => "Used storage space in bytes.",
 127                     type => 'integer',
 128                     renderer => 'bytes',
 129                     optional => 1,
 130                 },
 131                 avail => {
 132                     description => "Available storage space in bytes.",
 133                     type => 'integer',
 134                     renderer => 'bytes',
 135                     optional => 1,
 136                 },
 137                 used_fraction => {
 138                     description => "Used fraction (used/total).",
 139                     type => 'number',
 140                     renderer => 'fraction_as_percentage',
 141                     optional => 1,
 142                 },
 143             },
 144         },
 145         links => [ { rel => 'child', href => "{storage}" } ],
 146     },
 147     code => sub {
 148         my ($param) = @_;
 149
 150         my $rpcenv = PVE::RPCEnvironment::get();
 151         my $authuser = $rpcenv->get_user();
 152
 153         my $localnode = PVE::INotify::nodename();
 154
 155         my $target = $param->{target};
 156
 157         undef $target if $target && ($target eq $localnode || $target eq 'localhost');
 158
 159         my $cfg = PVE::Storage::config();
 160
 161         my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
 162
 163         raise_param_exc({ storage => "No such storage." })
 164             if $param->{storage} && !defined($info->{$param->{storage}});
 165
 166         my $res = {};
 167         my @sids = PVE::Storage::storage_ids($cfg);
 168         foreach my $storeid (@sids) {
 169             my $data = $info->{$storeid};
 170             next if !$data;
 171             my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
 172             next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
 173             next if $param->{storage} && $param->{storage} ne $storeid;
 174
 175             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
 176
 177             next if $param->{enabled} && $scfg->{disable};
 178
 179             if ($target) {
 180                 # check if storage content is accessible on local node and specified target node
 181                 # we use this on the Clone GUI
 182
 183                 next if !$scfg->{shared};
 184                 next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
 185                 next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
 186             }
 187
 188             if ($data->{total}) {
 189                 $data->{used_fraction} = ($data->{used} // 0) / $data->{total};
 190             }
 191
 192             $res->{$storeid} = $data;
 193         }
 194
 195         return PVE::RESTHandler::hash_to_array($res, 'storage');
 196     }});
 197
 198 __PACKAGE__->register_method ({
 199     name => 'diridx',
 200     path => '{storage}',
 201     method => 'GET',
 202     description => "",
 203     permissions => {
 204         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 205     },
 206     parameters => {
 207         additionalProperties => 0,
 208         properties => {
 209             node => get_standard_option('pve-node'),
 210             storage => get_standard_option('pve-storage-id'),
 211         },
 212     },
 213     returns => {
 214         type => 'array',
 215         items => {
 216             type => "object",
 217             properties => {
 218                 subdir => { type => 'string' },
 219             },
 220         },
 221         links => [ { rel => 'child', href => "{subdir}" } ],
 222     },
 223     code => sub {
 224         my ($param) = @_;
 225
 226         my $res = [
 227             { subdir => 'content' },
 228             { subdir => 'download-url' },
 229             { subdir => 'file-restore' },
 230             { subdir => 'prunebackups' },
 231             { subdir => 'rrd' },
 232             { subdir => 'rrddata' },
 233             { subdir => 'status' },
 234             { subdir => 'upload' },
 235         ];
 236
 237         return $res;
 238     }});
 239
 240 __PACKAGE__->register_method ({
 241     name => 'read_status',
 242     path => '{storage}/status',
 243     method => 'GET',
 244     description => "Read storage status.",
 245     permissions => {
 246         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 247     },
 248     protected => 1,
 249     proxyto => 'node',
 250     parameters => {
 251         additionalProperties => 0,
 252         properties => {
 253             node => get_standard_option('pve-node'),
 254             storage => get_standard_option('pve-storage-id'),
 255         },
 256     },
 257     returns => {
 258         type => "object",
 259         properties => {},
 260     },
 261     code => sub {
 262         my ($param) = @_;
 263
 264         my $cfg = PVE::Storage::config();
 265
 266         my $info = PVE::Storage::storage_info($cfg, $param->{content});
 267
 268         my $data = $info->{$param->{storage}};
 269
 270         raise_param_exc({ storage => "No such storage." })
 271             if !defined($data);
 272
 273         return $data;
 274     }});
 275
 276 __PACKAGE__->register_method ({
 277     name => 'rrd',
 278     path => '{storage}/rrd',
 279     method => 'GET',
 280     description => "Read storage RRD statistics (returns PNG).",
 281     permissions => {
 282         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 283     },
 284     protected => 1,
 285     proxyto => 'node',
 286     parameters => {
 287         additionalProperties => 0,
 288         properties => {
 289             node => get_standard_option('pve-node'),
 290             storage => get_standard_option('pve-storage-id'),
 291             timeframe => {
 292                 description => "Specify the time frame you are interested in.",
 293                 type => 'string',
 294                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 295             },
 296             ds => {
 297                 description => "The list of datasources you want to display.",
 298                 type => 'string', format => 'pve-configid-list',
 299             },
 300             cf => {
 301                 description => "The RRD consolidation function",
 302                 type => 'string',
 303                 enum => [ 'AVERAGE', 'MAX' ],
 304                 optional => 1,
 305             },
 306         },
 307     },
 308     returns => {
 309         type => "object",
 310         properties => {
 311             filename => { type => 'string' },
 312         },
 313     },
 314     code => sub {
 315         my ($param) = @_;
 316
 317         return PVE::RRD::create_rrd_graph(
 318             "pve2-storage/$param->{node}/$param->{storage}",
 319             $param->{timeframe}, $param->{ds}, $param->{cf});
 320     }});
 321
 322 __PACKAGE__->register_method ({
 323     name => 'rrddata',
 324     path => '{storage}/rrddata',
 325     method => 'GET',
 326     description => "Read storage RRD statistics.",
 327     permissions => {
 328         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 329     },
 330     protected => 1,
 331     proxyto => 'node',
 332     parameters => {
 333         additionalProperties => 0,
 334         properties => {
 335             node => get_standard_option('pve-node'),
 336             storage => get_standard_option('pve-storage-id'),
 337             timeframe => {
 338                 description => "Specify the time frame you are interested in.",
 339                 type => 'string',
 340                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 341             },
 342             cf => {
 343                 description => "The RRD consolidation function",
 344                 type => 'string',
 345                 enum => [ 'AVERAGE', 'MAX' ],
 346                 optional => 1,
 347             },
 348         },
 349     },
 350     returns => {
 351         type => "array",
 352         items => {
 353             type => "object",
 354             properties => {},
 355         },
 356     },
 357     code => sub {
 358         my ($param) = @_;
 359
 360         return PVE::RRD::create_rrd_data(
 361             "pve2-storage/$param->{node}/$param->{storage}",
 362             $param->{timeframe}, $param->{cf});
 363     }});
 364
 365 # makes no sense for big images and backup files (because it
 366 # create a copy of the file).
 367 __PACKAGE__->register_method ({
 368     name => 'upload',
 369     path => '{storage}/upload',
 370     method => 'POST',
 371     description => "Upload templates and ISO images.",
 372     permissions => {
 373         check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
 374     },
 375     protected => 1,
 376     parameters => {
 377         additionalProperties => 0,
 378         properties => {
 379             node => get_standard_option('pve-node'),
 380             storage => get_standard_option('pve-storage-id'),
 381             content => {
 382                 description => "Content type.",
 383                 type => 'string', format => 'pve-storage-content',
 384                 enum => ['iso', 'vztmpl'],
 385             },
 386             filename => {
 387                 description => "The name of the file to create. Caution: This will be normalized!",
 388                 maxLength => 255,
 389                 type => 'string',
 390             },
 391             tmpfilename => {
 392                 description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
 393                 type => 'string',
 394                 optional => 1,
 395             },
 396         },
 397     },
 398     returns => { type => "string" },
 399     code => sub {
 400         my ($param) = @_;
 401
 402         my $rpcenv = PVE::RPCEnvironment::get();
 403
 404         my $user = $rpcenv->get_user();
 405
 406         my $cfg = PVE::Storage::config();
 407
 408         my $node = $param->{node};
 409         my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
 410
 411         die "can't upload to storage type '$scfg->{type}'\n"
 412             if !defined($scfg->{path});
 413
 414         my $content = $param->{content};
 415
 416         my $tmpfilename = $param->{tmpfilename};
 417         die "missing temporary file name\n" if !$tmpfilename;
 418
 419         my $size = -s $tmpfilename;
 420         die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
 421
 422         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 423
 424         my $path;
 425
 426         if ($content eq 'iso') {
 427             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 428                 raise_param_exc({ filename => "wrong file extension" });
 429             }
 430             $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
 431         } elsif ($content eq 'vztmpl') {
 432             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 433                 raise_param_exc({ filename => "wrong file extension" });
 434             }
 435             $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
 436         } else {
 437             raise_param_exc({ content => "upload content type '$content' not allowed" });
 438         }
 439
 440         die "storage '$param->{storage}' does not support '$content' content\n"
 441             if !$scfg->{content}->{$content};
 442
 443         my $dest = "$path/$filename";
 444         my $dirname = dirname($dest);
 445
 446         # best effort to match apl_download behaviour
 447         chmod 0644, $tmpfilename;
 448
 449         my $err_cleanup = sub { unlink $dest, $tmpfilename; die "cleanup failed: $!" if $! && $! != ENOENT };
 450
 451         my $cmd;
 452         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
 453             my $remip = PVE::Cluster::remote_node_ip($node);
 454
 455             my @ssh_options = ('-o', 'BatchMode=yes');
 456
 457             my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
 458
 459             eval { # activate remote storage
 460                 run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]);
 461             };
 462             die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
 463
 464             run_command(
 465                 [@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
 466                 errmsg => "mkdir failed",
 467             );
 468
 469             $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
 470
 471             $err_cleanup = sub { run_command([@remcmd, 'rm', '-f', '--', $dest, $tmpfilename]) };
 472         } else {
 473             PVE::Storage::activate_storage($cfg, $param->{storage});
 474             File::Path::make_path($dirname);
 475             $cmd = ['cp', '--', $tmpfilename, $dest];
 476         }
 477
 478         # NOTE: we simply overwrite the destination file if it already exists
 479         my $worker = sub {
 480             my $upid = shift;
 481
 482             print "starting file import from: $tmpfilename\n";
 483             print "target node: $node\n";
 484             print "target file: $dest\n";
 485             print "file size is: $size\n";
 486             print "command: " . join(' ', @$cmd) . "\n";
 487
 488             eval { run_command($cmd, errmsg => 'import failed'); };
 489             if (my $err = $@) {
 490                 eval { $err_cleanup->() };
 491                 warn "$@" if $@;
 492                 die $err;
 493             }
 494             print "finished file import successfully\n";
 495         };
 496
 497         my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
 498
 499         # apache removes the temporary file on return, so we need
 500         # to wait here to make sure the worker process starts and
 501         # opens the file before it gets removed.
 502         sleep(1);
 503
 504         return $upid;
 505    }});
 506
 507 __PACKAGE__->register_method({
 508     name => 'download_url',
 509     path => '{storage}/download-url',
 510     method => 'POST',
 511     description => "Download templates and ISO images by using an URL.",
 512     proxyto => 'node',
 513     permissions => {
 514         check => [ 'and',
 515             ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
 516             ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
 517         ],
 518     },
 519     protected => 1,
 520     parameters => {
 521         additionalProperties => 0,
 522         properties => {
 523             node => get_standard_option('pve-node'),
 524             storage => get_standard_option('pve-storage-id'),
 525             url => {
 526                 description => "The URL to download the file from.",
 527                 type => 'string',
 528                 pattern => 'https?://.*',
 529             },
 530             content => {
 531                 description => "Content type.", # TODO: could be optional & detected in most cases
 532                 type => 'string', format => 'pve-storage-content',
 533                 enum => ['iso', 'vztmpl'],
 534             },
 535             filename => {
 536                 description => "The name of the file to create. Caution: This will be normalized!",
 537                 maxLength => 255,
 538                 type => 'string',
 539             },
 540             checksum => {
 541                 description => "The expected checksum of the file.",
 542                 type => 'string',
 543                 requires => 'checksum-algorithm',
 544                 optional => 1,
 545             },
 546             'checksum-algorithm' => {
 547                 description => "The algorithm to calculate the checksum of the file.",
 548                 type => 'string',
 549                 enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
 550                 requires => 'checksum',
 551                 optional => 1,
 552             },
 553             'verify-certificates' => {
 554                 description => "If false, no SSL/TLS certificates will be verified.",
 555                 type => 'boolean',
 556                 optional => 1,
 557                 default => 1,
 558             },
 559         },
 560     },
 561     returns => {
 562         type => "string"
 563     },
 564     code => sub {
 565         my ($param) = @_;
 566
 567         my $rpcenv = PVE::RPCEnvironment::get();
 568         my $user = $rpcenv->get_user();
 569
 570         my $cfg = PVE::Storage::config();
 571
 572         my ($node, $storage) = $param->@{'node', 'storage'};
 573         my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 574
 575         die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
 576             if !defined($scfg->{path});
 577
 578         my ($content, $url) = $param->@{'content', 'url'};
 579
 580         die "storage '$storage' is not configured for content-type '$content'\n"
 581             if !$scfg->{content}->{$content};
 582
 583         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 584
 585         my $path;
 586         if ($content eq 'iso') {
 587             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 588                 raise_param_exc({ filename => "wrong file extension" });
 589             }
 590             $path = PVE::Storage::get_iso_dir($cfg, $storage);
 591         } elsif ($content eq 'vztmpl') {
 592             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 593                 raise_param_exc({ filename => "wrong file extension" });
 594             }
 595             $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
 596         } else {
 597             raise_param_exc({ content => "upload content-type '$content' is not allowed" });
 598         }
 599
 600         PVE::Storage::activate_storage($cfg, $storage);
 601         File::Path::make_path($path);
 602
 603         my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
 604         my $opts = {
 605             hash_required => 0,
 606             verify_certificates => $param->{'verify-certificates'} // 1,
 607             http_proxy => $dccfg->{http_proxy},
 608         };
 609
 610         my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
 611         if ($checksum) {
 612             $opts->{"${checksum_algorithm}sum"} = $checksum;
 613             $opts->{hash_required} = 1;
 614         }
 615
 616         my $worker = sub {
 617             PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 618         };
 619
 620         my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID
 621
 622         return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
 623     }});
 624
 625 1;