PVE/API2/Storage/Status.pm

   1 package PVE::API2::Storage::Status;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8
   9 use PVE::Cluster;
  10 use PVE::Exception qw(raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::RESTHandler;
  14 use PVE::RPCEnvironment;
  15 use PVE::RRD;
  16 use PVE::Tools;
  17
  18 use PVE::API2::Storage::Content;
  19 use PVE::API2::Storage::FileRestore;
  20 use PVE::API2::Storage::PruneBackups;
  21 use PVE::Storage;
  22
  23 use base qw(PVE::RESTHandler);
  24
  25 __PACKAGE__->register_method ({
  26     subclass => "PVE::API2::Storage::PruneBackups",
  27     path => '{storage}/prunebackups',
  28 });
  29
  30 __PACKAGE__->register_method ({
  31     subclass => "PVE::API2::Storage::Content",
  32     # set fragment delimiter (no subdirs) - we need that, because volume
  33     # IDs may contain a slash '/'
  34     fragmentDelimiter => '',
  35     path => '{storage}/content',
  36 });
  37
  38 __PACKAGE__->register_method ({
  39    subclass => "PVE::API2::Storage::FileRestore",
  40    path => '{storage}/file-restore',
  41 });
  42
  43 __PACKAGE__->register_method ({
  44     name => 'index',
  45     path => '',
  46     method => 'GET',
  47     description => "Get status for all datastores.",
  48     permissions => {
  49         description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
  50         user => 'all',
  51     },
  52     protected => 1,
  53     proxyto => 'node',
  54     parameters => {
  55         additionalProperties => 0,
  56         properties => {
  57             node => get_standard_option('pve-node'),
  58             storage => get_standard_option('pve-storage-id', {
  59                 description => "Only list status for  specified storage",
  60                 optional => 1,
  61                 completion => \&PVE::Storage::complete_storage_enabled,
  62             }),
  63             content => {
  64                 description => "Only list stores which support this content type.",
  65                 type => 'string', format => 'pve-storage-content-list',
  66                 optional => 1,
  67                 completion => \&PVE::Storage::complete_content_type,
  68             },
  69             enabled => {
  70                 description => "Only list stores which are enabled (not disabled in config).",
  71                 type => 'boolean',
  72                 optional => 1,
  73                 default => 0,
  74             },
  75             target => get_standard_option('pve-node', {
  76                 description => "If target is different to 'node', we only lists shared storages which " .
  77                     "content is accessible on this 'node' and the specified 'target' node.",
  78                 optional => 1,
  79                 completion => \&PVE::Cluster::get_nodelist,
  80             }),
  81             'format' => {
  82                 description => "Include information about formats",
  83                 type => 'boolean',
  84                 optional => 1,
  85                 default => 0,
  86             },
  87         },
  88     },
  89     returns => {
  90         type => 'array',
  91         items => {
  92             type => "object",
  93             properties => {
  94                 storage => get_standard_option('pve-storage-id'),
  95                 type => {
  96                     description => "Storage type.",
  97                     type => 'string',
  98                 },
  99                 content => {
 100                     description => "Allowed storage content types.",
 101                     type => 'string', format => 'pve-storage-content-list',
 102                 },
 103                 enabled => {
 104                     description => "Set when storage is enabled (not disabled).",
 105                     type => 'boolean',
 106                     optional => 1,
 107                 },
 108                 active => {
 109                     description => "Set when storage is accessible.",
 110                     type => 'boolean',
 111                     optional => 1,
 112                 },
 113                 shared => {
 114                     description => "Shared flag from storage configuration.",
 115                     type => 'boolean',
 116                     optional => 1,
 117                 },
 118                 total => {
 119                     description => "Total storage space in bytes.",
 120                     type => 'integer',
 121                     renderer => 'bytes',
 122                     optional => 1,
 123                 },
 124                 used => {
 125                     description => "Used storage space in bytes.",
 126                     type => 'integer',
 127                     renderer => 'bytes',
 128                     optional => 1,
 129                 },
 130                 avail => {
 131                     description => "Available storage space in bytes.",
 132                     type => 'integer',
 133                     renderer => 'bytes',
 134                     optional => 1,
 135                 },
 136                 used_fraction => {
 137                     description => "Used fraction (used/total).",
 138                     type => 'number',
 139                     renderer => 'fraction_as_percentage',
 140                     optional => 1,
 141                 },
 142             },
 143         },
 144         links => [ { rel => 'child', href => "{storage}" } ],
 145     },
 146     code => sub {
 147         my ($param) = @_;
 148
 149         my $rpcenv = PVE::RPCEnvironment::get();
 150         my $authuser = $rpcenv->get_user();
 151
 152         my $localnode = PVE::INotify::nodename();
 153
 154         my $target = $param->{target};
 155
 156         undef $target if $target && ($target eq $localnode || $target eq 'localhost');
 157
 158         my $cfg = PVE::Storage::config();
 159
 160         my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
 161
 162         raise_param_exc({ storage => "No such storage." })
 163             if $param->{storage} && !defined($info->{$param->{storage}});
 164
 165         my $res = {};
 166         my @sids = PVE::Storage::storage_ids($cfg);
 167         foreach my $storeid (@sids) {
 168             my $data = $info->{$storeid};
 169             next if !$data;
 170             my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
 171             next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
 172             next if $param->{storage} && $param->{storage} ne $storeid;
 173
 174             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
 175
 176             next if $param->{enabled} && $scfg->{disable};
 177
 178             if ($target) {
 179                 # check if storage content is accessible on local node and specified target node
 180                 # we use this on the Clone GUI
 181
 182                 next if !$scfg->{shared};
 183                 next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
 184                 next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
 185             }
 186
 187             if ($data->{total}) {
 188                 $data->{used_fraction} = ($data->{used} // 0) / $data->{total};
 189             }
 190
 191             $res->{$storeid} = $data;
 192         }
 193
 194         return PVE::RESTHandler::hash_to_array($res, 'storage');
 195     }});
 196
 197 __PACKAGE__->register_method ({
 198     name => 'diridx',
 199     path => '{storage}',
 200     method => 'GET',
 201     description => "",
 202     permissions => {
 203         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 204     },
 205     parameters => {
 206         additionalProperties => 0,
 207         properties => {
 208             node => get_standard_option('pve-node'),
 209             storage => get_standard_option('pve-storage-id'),
 210         },
 211     },
 212     returns => {
 213         type => 'array',
 214         items => {
 215             type => "object",
 216             properties => {
 217                 subdir => { type => 'string' },
 218             },
 219         },
 220         links => [ { rel => 'child', href => "{subdir}" } ],
 221     },
 222     code => sub {
 223         my ($param) = @_;
 224
 225         my $res = [
 226             { subdir => 'content' },
 227             { subdir => 'download-url' },
 228             { subdir => 'file-restore' },
 229             { subdir => 'prunebackups' },
 230             { subdir => 'rrd' },
 231             { subdir => 'rrddata' },
 232             { subdir => 'status' },
 233             { subdir => 'upload' },
 234         ];
 235
 236         return $res;
 237     }});
 238
 239 __PACKAGE__->register_method ({
 240     name => 'read_status',
 241     path => '{storage}/status',
 242     method => 'GET',
 243     description => "Read storage status.",
 244     permissions => {
 245         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 246     },
 247     protected => 1,
 248     proxyto => 'node',
 249     parameters => {
 250         additionalProperties => 0,
 251         properties => {
 252             node => get_standard_option('pve-node'),
 253             storage => get_standard_option('pve-storage-id'),
 254         },
 255     },
 256     returns => {
 257         type => "object",
 258         properties => {},
 259     },
 260     code => sub {
 261         my ($param) = @_;
 262
 263         my $cfg = PVE::Storage::config();
 264
 265         my $info = PVE::Storage::storage_info($cfg, $param->{content});
 266
 267         my $data = $info->{$param->{storage}};
 268
 269         raise_param_exc({ storage => "No such storage." })
 270             if !defined($data);
 271
 272         return $data;
 273     }});
 274
 275 __PACKAGE__->register_method ({
 276     name => 'rrd',
 277     path => '{storage}/rrd',
 278     method => 'GET',
 279     description => "Read storage RRD statistics (returns PNG).",
 280     permissions => {
 281         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 282     },
 283     protected => 1,
 284     proxyto => 'node',
 285     parameters => {
 286         additionalProperties => 0,
 287         properties => {
 288             node => get_standard_option('pve-node'),
 289             storage => get_standard_option('pve-storage-id'),
 290             timeframe => {
 291                 description => "Specify the time frame you are interested in.",
 292                 type => 'string',
 293                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 294             },
 295             ds => {
 296                 description => "The list of datasources you want to display.",
 297                 type => 'string', format => 'pve-configid-list',
 298             },
 299             cf => {
 300                 description => "The RRD consolidation function",
 301                 type => 'string',
 302                 enum => [ 'AVERAGE', 'MAX' ],
 303                 optional => 1,
 304             },
 305         },
 306     },
 307     returns => {
 308         type => "object",
 309         properties => {
 310             filename => { type => 'string' },
 311         },
 312     },
 313     code => sub {
 314         my ($param) = @_;
 315
 316         return PVE::RRD::create_rrd_graph(
 317             "pve2-storage/$param->{node}/$param->{storage}",
 318             $param->{timeframe}, $param->{ds}, $param->{cf});
 319     }});
 320
 321 __PACKAGE__->register_method ({
 322     name => 'rrddata',
 323     path => '{storage}/rrddata',
 324     method => 'GET',
 325     description => "Read storage RRD statistics.",
 326     permissions => {
 327         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 328     },
 329     protected => 1,
 330     proxyto => 'node',
 331     parameters => {
 332         additionalProperties => 0,
 333         properties => {
 334             node => get_standard_option('pve-node'),
 335             storage => get_standard_option('pve-storage-id'),
 336             timeframe => {
 337                 description => "Specify the time frame you are interested in.",
 338                 type => 'string',
 339                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 340             },
 341             cf => {
 342                 description => "The RRD consolidation function",
 343                 type => 'string',
 344                 enum => [ 'AVERAGE', 'MAX' ],
 345                 optional => 1,
 346             },
 347         },
 348     },
 349     returns => {
 350         type => "array",
 351         items => {
 352             type => "object",
 353             properties => {},
 354         },
 355     },
 356     code => sub {
 357         my ($param) = @_;
 358
 359         return PVE::RRD::create_rrd_data(
 360             "pve2-storage/$param->{node}/$param->{storage}",
 361             $param->{timeframe}, $param->{cf});
 362     }});
 363
 364 # makes no sense for big images and backup files (because it
 365 # create a copy of the file).
 366 __PACKAGE__->register_method ({
 367     name => 'upload',
 368     path => '{storage}/upload',
 369     method => 'POST',
 370     description => "Upload templates and ISO images.",
 371     permissions => {
 372         check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
 373     },
 374     protected => 1,
 375     parameters => {
 376         additionalProperties => 0,
 377         properties => {
 378             node => get_standard_option('pve-node'),
 379             storage => get_standard_option('pve-storage-id'),
 380             content => {
 381                 description => "Content type.",
 382                 type => 'string', format => 'pve-storage-content',
 383                 enum => ['iso', 'vztmpl'],
 384             },
 385             filename => {
 386                 description => "The name of the file to create. Caution: This will be normalized!",
 387                 maxLength => 255,
 388                 type => 'string',
 389             },
 390             tmpfilename => {
 391                 description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
 392                 type => 'string',
 393                 optional => 1,
 394             },
 395         },
 396     },
 397     returns => { type => "string" },
 398     code => sub {
 399         my ($param) = @_;
 400
 401         my $rpcenv = PVE::RPCEnvironment::get();
 402
 403         my $user = $rpcenv->get_user();
 404
 405         my $cfg = PVE::Storage::config();
 406
 407         my $node = $param->{node};
 408         my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
 409
 410         die "can't upload to storage type '$scfg->{type}'\n"
 411             if !defined($scfg->{path});
 412
 413         my $content = $param->{content};
 414
 415         my $tmpfilename = $param->{tmpfilename};
 416         die "missing temporary file name\n" if !$tmpfilename;
 417
 418         my $size = -s $tmpfilename;
 419         die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
 420
 421         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 422
 423         my $path;
 424
 425         if ($content eq 'iso') {
 426             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 427                 raise_param_exc({ filename => "wrong file extension" });
 428             }
 429             $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
 430         } elsif ($content eq 'vztmpl') {
 431             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 432                 raise_param_exc({ filename => "wrong file extension" });
 433             }
 434             $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
 435         } else {
 436             raise_param_exc({ content => "upload content type '$content' not allowed" });
 437         }
 438
 439         die "storage '$param->{storage}' does not support '$content' content\n"
 440             if !$scfg->{content}->{$content};
 441
 442         my $dest = "$path/$filename";
 443         my $dirname = dirname($dest);
 444
 445         # best effort to match apl_download behaviour
 446         chmod 0644, $tmpfilename;
 447
 448         # we simply overwrite the destination file if it already exists
 449
 450         my $cmd;
 451         my $err_cmd;
 452         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
 453             my $remip = PVE::Cluster::remote_node_ip($node);
 454
 455             my @ssh_options = ('-o', 'BatchMode=yes');
 456
 457             my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
 458
 459             eval {
 460                 # activate remote storage
 461                 PVE::Tools::run_command([@remcmd, '/usr/sbin/pvesm', 'status',
 462                                          '--storage', $param->{storage}]);
 463             };
 464             die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
 465
 466             PVE::Tools::run_command([@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
 467                                     errmsg => "mkdir failed");
 468
 469             $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
 470             $err_cmd = [@remcmd, 'unlink', '--', $dest];
 471         } else {
 472             PVE::Storage::activate_storage($cfg, $param->{storage});
 473             File::Path::make_path($dirname);
 474             $cmd = ['cp', '--', $tmpfilename, $dest];
 475             $err_cmd = ['unlink', '--', $dest];
 476         }
 477
 478         my $worker = sub {
 479             my $upid = shift;
 480
 481             print "starting file import from: $tmpfilename\n";
 482             print "target node: $node\n";
 483             print "target file: $dest\n";
 484             print "file size is: $size\n";
 485             print "command: " . join(' ', @$cmd) . "\n";
 486
 487             eval { PVE::Tools::run_command($cmd, errmsg => 'import failed'); };
 488             if (my $err = $@) {
 489                 eval { PVE::Tools::run_command($err_cmd); };
 490                 die $err;
 491             }
 492             print "finished file import successfully\n";
 493         };
 494
 495         my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
 496
 497         # apache removes the temporary file on return, so we need
 498         # to wait here to make sure the worker process starts and
 499         # opens the file before it gets removed.
 500         sleep(1);
 501
 502         return $upid;
 503    }});
 504
 505 __PACKAGE__->register_method({
 506     name => 'download_url',
 507     path => '{storage}/download-url',
 508     method => 'POST',
 509     description => "Download templates and ISO images by using an URL.",
 510     proxyto => 'node',
 511     permissions => {
 512         check => [ 'and',
 513             ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
 514             ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
 515         ],
 516     },
 517     protected => 1,
 518     parameters => {
 519         additionalProperties => 0,
 520         properties => {
 521             node => get_standard_option('pve-node'),
 522             storage => get_standard_option('pve-storage-id'),
 523             url => {
 524                 description => "The URL to download the file from.",
 525                 type => 'string',
 526                 pattern => 'https?://.*',
 527             },
 528             content => {
 529                 description => "Content type.", # TODO: could be optional & detected in most cases
 530                 type => 'string', format => 'pve-storage-content',
 531                 enum => ['iso', 'vztmpl'],
 532             },
 533             filename => {
 534                 description => "The name of the file to create. Caution: This will be normalized!",
 535                 maxLength => 255,
 536                 type => 'string',
 537             },
 538             checksum => {
 539                 description => "The expected checksum of the file.",
 540                 type => 'string',
 541                 requires => 'checksum-algorithm',
 542                 optional => 1,
 543             },
 544             'checksum-algorithm' => {
 545                 description => "The algorithm to calculate the checksum of the file.",
 546                 type => 'string',
 547                 enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
 548                 requires => 'checksum',
 549                 optional => 1,
 550             },
 551             'verify-certificates' => {
 552                 description => "If false, no SSL/TLS certificates will be verified.",
 553                 type => 'boolean',
 554                 optional => 1,
 555                 default => 1,
 556             },
 557         },
 558     },
 559     returns => {
 560         type => "string"
 561     },
 562     code => sub {
 563         my ($param) = @_;
 564
 565         my $rpcenv = PVE::RPCEnvironment::get();
 566         my $user = $rpcenv->get_user();
 567
 568         my $cfg = PVE::Storage::config();
 569
 570         my ($node, $storage) = $param->@{'node', 'storage'};
 571         my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 572
 573         die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
 574             if !defined($scfg->{path});
 575
 576         my ($content, $url) = $param->@{'content', 'url'};
 577
 578         die "storage '$storage' is not configured for content-type '$content'\n"
 579             if !$scfg->{content}->{$content};
 580
 581         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 582
 583         my $path;
 584         if ($content eq 'iso') {
 585             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 586                 raise_param_exc({ filename => "wrong file extension" });
 587             }
 588             $path = PVE::Storage::get_iso_dir($cfg, $storage);
 589         } elsif ($content eq 'vztmpl') {
 590             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 591                 raise_param_exc({ filename => "wrong file extension" });
 592             }
 593             $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
 594         } else {
 595             raise_param_exc({ content => "upload content-type '$content' is not allowed" });
 596         }
 597
 598         PVE::Storage::activate_storage($cfg, $storage);
 599         File::Path::make_path($path);
 600
 601         my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
 602         my $opts = {
 603             hash_required => 0,
 604             verify_certificates => $param->{'verify-certificates'} // 1,
 605             http_proxy => $dccfg->{http_proxy},
 606         };
 607
 608         my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
 609         if ($checksum) {
 610             $opts->{"${checksum_algorithm}sum"} = $checksum;
 611             $opts->{hash_required} = 1;
 612         }
 613
 614         my $worker = sub {
 615             PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 616         };
 617
 618         my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID
 619
 620         return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
 621     }});
 622
 623 1;