PVE/API2/Storage/Status.pm

   1 package PVE::API2::Storage::Status;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use File::Basename;
   8 use PVE::Tools;
   9 use PVE::INotify;
  10 use PVE::Cluster;
  11 use PVE::RRD;
  12 use PVE::Storage;
  13 use PVE::API2::Storage::Content;
  14 use PVE::API2::Storage::PruneBackups;
  15 use PVE::API2::Storage::FileRestore;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::Exception qw(raise_param_exc);
  20
  21 use base qw(PVE::RESTHandler);
  22
  23 __PACKAGE__->register_method ({
  24     subclass => "PVE::API2::Storage::PruneBackups",
  25     path => '{storage}/prunebackups',
  26 });
  27
  28 __PACKAGE__->register_method ({
  29     subclass => "PVE::API2::Storage::Content",
  30     # set fragment delimiter (no subdirs) - we need that, because volume
  31     # IDs may contain a slash '/'
  32     fragmentDelimiter => '',
  33     path => '{storage}/content',
  34 });
  35
  36 __PACKAGE__->register_method ({
  37    subclass => "PVE::API2::Storage::FileRestore",
  38    path => '{storage}/file-restore',
  39 });
  40
  41 __PACKAGE__->register_method ({
  42     name => 'index',
  43     path => '',
  44     method => 'GET',
  45     description => "Get status for all datastores.",
  46     permissions => {
  47         description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
  48         user => 'all',
  49     },
  50     protected => 1,
  51     proxyto => 'node',
  52     parameters => {
  53         additionalProperties => 0,
  54         properties => {
  55             node => get_standard_option('pve-node'),
  56             storage => get_standard_option('pve-storage-id', {
  57                 description => "Only list status for  specified storage",
  58                 optional => 1,
  59                 completion => \&PVE::Storage::complete_storage_enabled,
  60             }),
  61             content => {
  62                 description => "Only list stores which support this content type.",
  63                 type => 'string', format => 'pve-storage-content-list',
  64                 optional => 1,
  65                 completion => \&PVE::Storage::complete_content_type,
  66             },
  67             enabled => {
  68                 description => "Only list stores which are enabled (not disabled in config).",
  69                 type => 'boolean',
  70                 optional => 1,
  71                 default => 0,
  72             },
  73             target => get_standard_option('pve-node', {
  74                 description => "If target is different to 'node', we only lists shared storages which " .
  75                     "content is accessible on this 'node' and the specified 'target' node.",
  76                 optional => 1,
  77                 completion => \&PVE::Cluster::get_nodelist,
  78             }),
  79             'format' => {
  80                 description => "Include information about formats",
  81                 type => 'boolean',
  82                 optional => 1,
  83                 default => 0,
  84             },
  85         },
  86     },
  87     returns => {
  88         type => 'array',
  89         items => {
  90             type => "object",
  91             properties => {
  92                 storage => get_standard_option('pve-storage-id'),
  93                 type => {
  94                     description => "Storage type.",
  95                     type => 'string',
  96                 },
  97                 content => {
  98                     description => "Allowed storage content types.",
  99                     type => 'string', format => 'pve-storage-content-list',
 100                 },
 101                 enabled => {
 102                     description => "Set when storage is enabled (not disabled).",
 103                     type => 'boolean',
 104                     optional => 1,
 105                 },
 106                 active => {
 107                     description => "Set when storage is accessible.",
 108                     type => 'boolean',
 109                     optional => 1,
 110                 },
 111                 shared => {
 112                     description => "Shared flag from storage configuration.",
 113                     type => 'boolean',
 114                     optional => 1,
 115                 },
 116                 total => {
 117                     description => "Total storage space in bytes.",
 118                     type => 'integer',
 119                     renderer => 'bytes',
 120                     optional => 1,
 121                 },
 122                 used => {
 123                     description => "Used storage space in bytes.",
 124                     type => 'integer',
 125                     renderer => 'bytes',
 126                     optional => 1,
 127                 },
 128                 avail => {
 129                     description => "Available storage space in bytes.",
 130                     type => 'integer',
 131                     renderer => 'bytes',
 132                     optional => 1,
 133                 },
 134                 used_fraction => {
 135                     description => "Used fraction (used/total).",
 136                     type => 'number',
 137                     renderer => 'fraction_as_percentage',
 138                     optional => 1,
 139                 },
 140             },
 141         },
 142         links => [ { rel => 'child', href => "{storage}" } ],
 143     },
 144     code => sub {
 145         my ($param) = @_;
 146
 147         my $rpcenv = PVE::RPCEnvironment::get();
 148         my $authuser = $rpcenv->get_user();
 149
 150         my $localnode = PVE::INotify::nodename();
 151
 152         my $target = $param->{target};
 153
 154         undef $target if $target && ($target eq $localnode || $target eq 'localhost');
 155
 156         my $cfg = PVE::Storage::config();
 157
 158         my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
 159
 160         raise_param_exc({ storage => "No such storage." })
 161             if $param->{storage} && !defined($info->{$param->{storage}});
 162
 163         my $res = {};
 164         my @sids = PVE::Storage::storage_ids($cfg);
 165         foreach my $storeid (@sids) {
 166             my $data = $info->{$storeid};
 167             next if !$data;
 168             my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
 169             next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
 170             next if $param->{storage} && $param->{storage} ne $storeid;
 171
 172             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
 173
 174             next if $param->{enabled} && $scfg->{disable};
 175
 176             if ($target) {
 177                 # check if storage content is accessible on local node and specified target node
 178                 # we use this on the Clone GUI
 179
 180                 next if !$scfg->{shared};
 181                 next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
 182                 next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
 183             }
 184
 185             if ($data->{total}) {
 186                 $data->{used_fraction} = ($data->{used} // 0) / $data->{total};
 187             }
 188
 189             $res->{$storeid} = $data;
 190         }
 191
 192         return PVE::RESTHandler::hash_to_array($res, 'storage');
 193     }});
 194
 195 __PACKAGE__->register_method ({
 196     name => 'diridx',
 197     path => '{storage}',
 198     method => 'GET',
 199     description => "",
 200     permissions => {
 201         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 202     },
 203     parameters => {
 204         additionalProperties => 0,
 205         properties => {
 206             node => get_standard_option('pve-node'),
 207             storage => get_standard_option('pve-storage-id'),
 208         },
 209     },
 210     returns => {
 211         type => 'array',
 212         items => {
 213             type => "object",
 214             properties => {
 215                 subdir => { type => 'string' },
 216             },
 217         },
 218         links => [ { rel => 'child', href => "{subdir}" } ],
 219     },
 220     code => sub {
 221         my ($param) = @_;
 222
 223         my $res = [
 224             { subdir => 'content' },
 225             { subdir => 'download-url' },
 226             { subdir => 'file-restore' },
 227             { subdir => 'prunebackups' },
 228             { subdir => 'rrd' },
 229             { subdir => 'rrddata' },
 230             { subdir => 'status' },
 231             { subdir => 'upload' },
 232         ];
 233
 234         return $res;
 235     }});
 236
 237 __PACKAGE__->register_method ({
 238     name => 'read_status',
 239     path => '{storage}/status',
 240     method => 'GET',
 241     description => "Read storage status.",
 242     permissions => {
 243         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 244     },
 245     protected => 1,
 246     proxyto => 'node',
 247     parameters => {
 248         additionalProperties => 0,
 249         properties => {
 250             node => get_standard_option('pve-node'),
 251             storage => get_standard_option('pve-storage-id'),
 252         },
 253     },
 254     returns => {
 255         type => "object",
 256         properties => {},
 257     },
 258     code => sub {
 259         my ($param) = @_;
 260
 261         my $cfg = PVE::Storage::config();
 262
 263         my $info = PVE::Storage::storage_info($cfg, $param->{content});
 264
 265         my $data = $info->{$param->{storage}};
 266
 267         raise_param_exc({ storage => "No such storage." })
 268             if !defined($data);
 269
 270         return $data;
 271     }});
 272
 273 __PACKAGE__->register_method ({
 274     name => 'rrd',
 275     path => '{storage}/rrd',
 276     method => 'GET',
 277     description => "Read storage RRD statistics (returns PNG).",
 278     permissions => {
 279         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 280     },
 281     protected => 1,
 282     proxyto => 'node',
 283     parameters => {
 284         additionalProperties => 0,
 285         properties => {
 286             node => get_standard_option('pve-node'),
 287             storage => get_standard_option('pve-storage-id'),
 288             timeframe => {
 289                 description => "Specify the time frame you are interested in.",
 290                 type => 'string',
 291                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 292             },
 293             ds => {
 294                 description => "The list of datasources you want to display.",
 295                 type => 'string', format => 'pve-configid-list',
 296             },
 297             cf => {
 298                 description => "The RRD consolidation function",
 299                 type => 'string',
 300                 enum => [ 'AVERAGE', 'MAX' ],
 301                 optional => 1,
 302             },
 303         },
 304     },
 305     returns => {
 306         type => "object",
 307         properties => {
 308             filename => { type => 'string' },
 309         },
 310     },
 311     code => sub {
 312         my ($param) = @_;
 313
 314         return PVE::RRD::create_rrd_graph(
 315             "pve2-storage/$param->{node}/$param->{storage}",
 316             $param->{timeframe}, $param->{ds}, $param->{cf});
 317     }});
 318
 319 __PACKAGE__->register_method ({
 320     name => 'rrddata',
 321     path => '{storage}/rrddata',
 322     method => 'GET',
 323     description => "Read storage RRD statistics.",
 324     permissions => {
 325         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 326     },
 327     protected => 1,
 328     proxyto => 'node',
 329     parameters => {
 330         additionalProperties => 0,
 331         properties => {
 332             node => get_standard_option('pve-node'),
 333             storage => get_standard_option('pve-storage-id'),
 334             timeframe => {
 335                 description => "Specify the time frame you are interested in.",
 336                 type => 'string',
 337                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 338             },
 339             cf => {
 340                 description => "The RRD consolidation function",
 341                 type => 'string',
 342                 enum => [ 'AVERAGE', 'MAX' ],
 343                 optional => 1,
 344             },
 345         },
 346     },
 347     returns => {
 348         type => "array",
 349         items => {
 350             type => "object",
 351             properties => {},
 352         },
 353     },
 354     code => sub {
 355         my ($param) = @_;
 356
 357         return PVE::RRD::create_rrd_data(
 358             "pve2-storage/$param->{node}/$param->{storage}",
 359             $param->{timeframe}, $param->{cf});
 360     }});
 361
 362 # makes no sense for big images and backup files (because it
 363 # create a copy of the file).
 364 __PACKAGE__->register_method ({
 365     name => 'upload',
 366     path => '{storage}/upload',
 367     method => 'POST',
 368     description => "Upload templates and ISO images.",
 369     permissions => {
 370         check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
 371     },
 372     protected => 1,
 373     parameters => {
 374         additionalProperties => 0,
 375         properties => {
 376             node => get_standard_option('pve-node'),
 377             storage => get_standard_option('pve-storage-id'),
 378             content => {
 379                 description => "Content type.",
 380                 type => 'string', format => 'pve-storage-content',
 381                 enum => ['iso', 'vztmpl'],
 382             },
 383             filename => {
 384                 description => "The name of the file to create.",
 385                 type => 'string',
 386             },
 387             tmpfilename => {
 388                 description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
 389                 type => 'string',
 390                 optional => 1,
 391             },
 392         },
 393     },
 394     returns => { type => "string" },
 395     code => sub {
 396         my ($param) = @_;
 397
 398         my $rpcenv = PVE::RPCEnvironment::get();
 399
 400         my $user = $rpcenv->get_user();
 401
 402         my $cfg = PVE::Storage::config();
 403
 404         my $node = $param->{node};
 405         my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
 406
 407         die "can't upload to storage type '$scfg->{type}'\n"
 408             if !defined($scfg->{path});
 409
 410         my $content = $param->{content};
 411
 412         my $tmpfilename = $param->{tmpfilename};
 413         die "missing temporary file name\n" if !$tmpfilename;
 414
 415         my $size = -s $tmpfilename;
 416         die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
 417
 418         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 419
 420         my $path;
 421
 422         if ($content eq 'iso') {
 423             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 424                 raise_param_exc({ filename => "wrong file extension" });
 425             }
 426             $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
 427         } elsif ($content eq 'vztmpl') {
 428             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 429                 raise_param_exc({ filename => "wrong file extension" });
 430             }
 431             $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
 432         } else {
 433             raise_param_exc({ content => "upload content type '$content' not allowed" });
 434         }
 435
 436         die "storage '$param->{storage}' does not support '$content' content\n"
 437             if !$scfg->{content}->{$content};
 438
 439         my $dest = "$path/$filename";
 440         my $dirname = dirname($dest);
 441
 442         # best effort to match apl_download behaviour
 443         chmod 0644, $tmpfilename;
 444
 445         # we simply overwrite the destination file if it already exists
 446
 447         my $cmd;
 448         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
 449             my $remip = PVE::Cluster::remote_node_ip($node);
 450
 451             my @ssh_options = ('-o', 'BatchMode=yes');
 452
 453             my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
 454
 455             eval {
 456                 # activate remote storage
 457                 PVE::Tools::run_command([@remcmd, '/usr/sbin/pvesm', 'status',
 458                                          '--storage', $param->{storage}]);
 459             };
 460             die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
 461
 462             PVE::Tools::run_command([@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
 463                                     errmsg => "mkdir failed");
 464
 465             $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
 466         } else {
 467             PVE::Storage::activate_storage($cfg, $param->{storage});
 468             File::Path::make_path($dirname);
 469             $cmd = ['cp', '--', $tmpfilename, $dest];
 470         }
 471
 472         my $worker = sub {
 473             my $upid = shift;
 474
 475             print "starting file import from: $tmpfilename\n";
 476             print "target node: $node\n";
 477             print "target file: $dest\n";
 478             print "file size is: $size\n";
 479             print "command: " . join(' ', @$cmd) . "\n";
 480
 481             eval { PVE::Tools::run_command($cmd, errmsg => 'import failed'); };
 482             if (my $err = $@) {
 483                 unlink $dest;
 484                 die $err;
 485             }
 486             print "finished file import successfully\n";
 487         };
 488
 489         my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
 490
 491         # apache removes the temporary file on return, so we need
 492         # to wait here to make sure the worker process starts and
 493         # opens the file before it gets removed.
 494         sleep(1);
 495
 496         return $upid;
 497    }});
 498
 499 __PACKAGE__->register_method({
 500     name => 'download_url',
 501     path => '{storage}/download-url',
 502     method => 'POST',
 503     description => "Download templates and ISO images by using an URL.",
 504     proxyto => 'node',
 505     permissions => {
 506         check => [ 'and',
 507             ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
 508             ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
 509         ],
 510     },
 511     protected => 1,
 512     parameters => {
 513         additionalProperties => 0,
 514         properties => {
 515             node => get_standard_option('pve-node'),
 516             storage => get_standard_option('pve-storage-id'),
 517             url => {
 518                 description => "The URL to download the file from.",
 519                 type => 'string',
 520                 pattern => 'https?://.*',
 521             },
 522             content => {
 523                 description => "Content type.", # TODO: could be optional & detected in most cases
 524                 type => 'string', format => 'pve-storage-content',
 525                 enum => ['iso', 'vztmpl'],
 526             },
 527             filename => {
 528                 description => "The name of the file to create. Caution: This will be normalized!",
 529                 maxLength => 255,
 530                 type => 'string',
 531             },
 532             checksum => {
 533                 description => "The expected checksum of the file.",
 534                 type => 'string',
 535                 requires => 'checksum-algorithm',
 536                 optional => 1,
 537             },
 538             'checksum-algorithm' => {
 539                 description => "The algorithm to calculate the checksum of the file.",
 540                 type => 'string',
 541                 enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
 542                 requires => 'checksum',
 543                 optional => 1,
 544             },
 545             'verify-certificates' => {
 546                 description => "If false, no SSL/TLS certificates will be verified.",
 547                 type => 'boolean',
 548                 optional => 1,
 549                 default => 1,
 550             },
 551         },
 552     },
 553     returns => {
 554         type => "string"
 555     },
 556     code => sub {
 557         my ($param) = @_;
 558
 559         my $rpcenv = PVE::RPCEnvironment::get();
 560         my $user = $rpcenv->get_user();
 561
 562         my $cfg = PVE::Storage::config();
 563
 564         my ($node, $storage) = $param->@{'node', 'storage'};
 565         my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 566
 567         die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
 568             if !defined($scfg->{path});
 569
 570         my ($content, $url) = $param->@{'content', 'url'};
 571
 572         die "storage '$storage' is not configured for content-type '$content'\n"
 573             if !$scfg->{content}->{$content};
 574
 575         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 576
 577         my $path;
 578         if ($content eq 'iso') {
 579             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 580                 raise_param_exc({ filename => "wrong file extension" });
 581             }
 582             $path = PVE::Storage::get_iso_dir($cfg, $storage);
 583         } elsif ($content eq 'vztmpl') {
 584             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 585                 raise_param_exc({ filename => "wrong file extension" });
 586             }
 587             $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
 588         } else {
 589             raise_param_exc({ content => "upload content-type '$content' is not allowed" });
 590         }
 591
 592         PVE::Storage::activate_storage($cfg, $storage);
 593         File::Path::make_path($path);
 594
 595         my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
 596         my $opts = {
 597             hash_required => 0,
 598             verify_certificates => $param->{'verify-certificates'} // 1,
 599             http_proxy => $dccfg->{http_proxy},
 600         };
 601
 602         my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
 603         if ($checksum) {
 604             $opts->{"${checksum_algorithm}sum"} = $checksum;
 605             $opts->{hash_required} = 1;
 606         }
 607
 608         my $worker = sub {
 609             PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 610         };
 611
 612         my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID
 613
 614         return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
 615     }});
 616
 617 1;