PVE/API2/Storage/Status.pm

   1 package PVE::API2::Storage::Status;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use File::Basename;
   8 use PVE::Tools;
   9 use PVE::INotify;
  10 use PVE::Cluster;
  11 use PVE::RRD;
  12 use PVE::Storage;
  13 use PVE::API2::Storage::Content;
  14 use PVE::API2::Storage::PruneBackups;
  15 use PVE::API2::Storage::FileRestore;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::Exception qw(raise_param_exc);
  20
  21 use base qw(PVE::RESTHandler);
  22
  23 __PACKAGE__->register_method ({
  24     subclass => "PVE::API2::Storage::PruneBackups",
  25     path => '{storage}/prunebackups',
  26 });
  27
  28 __PACKAGE__->register_method ({
  29     subclass => "PVE::API2::Storage::Content",
  30     # set fragment delimiter (no subdirs) - we need that, because volume
  31     # IDs may contain a slash '/'
  32     fragmentDelimiter => '',
  33     path => '{storage}/content',
  34 });
  35
  36 __PACKAGE__->register_method ({
  37    subclass => "PVE::API2::Storage::FileRestore",
  38    path => '{storage}/file-restore',
  39 });
  40
  41 __PACKAGE__->register_method ({
  42     name => 'index',
  43     path => '',
  44     method => 'GET',
  45     description => "Get status for all datastores.",
  46     permissions => {
  47         description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
  48         user => 'all',
  49     },
  50     protected => 1,
  51     proxyto => 'node',
  52     parameters => {
  53         additionalProperties => 0,
  54         properties => {
  55             node => get_standard_option('pve-node'),
  56             storage => get_standard_option('pve-storage-id', {
  57                 description => "Only list status for  specified storage",
  58                 optional => 1,
  59                 completion => \&PVE::Storage::complete_storage_enabled,
  60             }),
  61             content => {
  62                 description => "Only list stores which support this content type.",
  63                 type => 'string', format => 'pve-storage-content-list',
  64                 optional => 1,
  65                 completion => \&PVE::Storage::complete_content_type,
  66             },
  67             enabled => {
  68                 description => "Only list stores which are enabled (not disabled in config).",
  69                 type => 'boolean',
  70                 optional => 1,
  71                 default => 0,
  72             },
  73             target => get_standard_option('pve-node', {
  74                 description => "If target is different to 'node', we only lists shared storages which " .
  75                     "content is accessible on this 'node' and the specified 'target' node.",
  76                 optional => 1,
  77                 completion => \&PVE::Cluster::get_nodelist,
  78             }),
  79             'format' => {
  80                 description => "Include information about formats",
  81                 type => 'boolean',
  82                 optional => 1,
  83                 default => 0,
  84             },
  85         },
  86     },
  87     returns => {
  88         type => 'array',
  89         items => {
  90             type => "object",
  91             properties => {
  92                 storage => get_standard_option('pve-storage-id'),
  93                 type => {
  94                     description => "Storage type.",
  95                     type => 'string',
  96                 },
  97                 content => {
  98                     description => "Allowed storage content types.",
  99                     type => 'string', format => 'pve-storage-content-list',
 100                 },
 101                 enabled => {
 102                     description => "Set when storage is enabled (not disabled).",
 103                     type => 'boolean',
 104                     optional => 1,
 105                 },
 106                 active => {
 107                     description => "Set when storage is accessible.",
 108                     type => 'boolean',
 109                     optional => 1,
 110                 },
 111                 shared => {
 112                     description => "Shared flag from storage configuration.",
 113                     type => 'boolean',
 114                     optional => 1,
 115                 },
 116                 total => {
 117                     description => "Total storage space in bytes.",
 118                     type => 'integer',
 119                     renderer => 'bytes',
 120                     optional => 1,
 121                 },
 122                 used => {
 123                     description => "Used storage space in bytes.",
 124                     type => 'integer',
 125                     renderer => 'bytes',
 126                     optional => 1,
 127                 },
 128                 avail => {
 129                     description => "Available storage space in bytes.",
 130                     type => 'integer',
 131                     renderer => 'bytes',
 132                     optional => 1,
 133                 },
 134                 used_fraction => {
 135                     description => "Used fraction (used/total).",
 136                     type => 'number',
 137                     renderer => 'fraction_as_percentage',
 138                     optional => 1,
 139                 },
 140             },
 141         },
 142         links => [ { rel => 'child', href => "{storage}" } ],
 143     },
 144     code => sub {
 145         my ($param) = @_;
 146
 147         my $rpcenv = PVE::RPCEnvironment::get();
 148         my $authuser = $rpcenv->get_user();
 149
 150         my $localnode = PVE::INotify::nodename();
 151
 152         my $target = $param->{target};
 153
 154         undef $target if $target && ($target eq $localnode || $target eq 'localhost');
 155
 156         my $cfg = PVE::Storage::config();
 157
 158         my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
 159
 160         raise_param_exc({ storage => "No such storage." })
 161             if $param->{storage} && !defined($info->{$param->{storage}});
 162
 163         my $res = {};
 164         my @sids = PVE::Storage::storage_ids($cfg);
 165         foreach my $storeid (@sids) {
 166             my $data = $info->{$storeid};
 167             next if !$data;
 168             my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
 169             next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
 170             next if $param->{storage} && $param->{storage} ne $storeid;
 171
 172             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
 173
 174             next if $param->{enabled} && $scfg->{disable};
 175
 176             if ($target) {
 177                 # check if storage content is accessible on local node and specified target node
 178                 # we use this on the Clone GUI
 179
 180                 next if !$scfg->{shared};
 181                 next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
 182                 next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
 183             }
 184
 185             if ($data->{total}) {
 186                 $data->{used_fraction} = ($data->{used} // 0) / $data->{total};
 187             }
 188
 189             $res->{$storeid} = $data;
 190         }
 191
 192         return PVE::RESTHandler::hash_to_array($res, 'storage');
 193     }});
 194
 195 __PACKAGE__->register_method ({
 196     name => 'diridx',
 197     path => '{storage}',
 198     method => 'GET',
 199     description => "",
 200     permissions => {
 201         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 202     },
 203     parameters => {
 204         additionalProperties => 0,
 205         properties => {
 206             node => get_standard_option('pve-node'),
 207             storage => get_standard_option('pve-storage-id'),
 208         },
 209     },
 210     returns => {
 211         type => 'array',
 212         items => {
 213             type => "object",
 214             properties => {
 215                 subdir => { type => 'string' },
 216             },
 217         },
 218         links => [ { rel => 'child', href => "{subdir}" } ],
 219     },
 220     code => sub {
 221         my ($param) = @_;
 222
 223         my $res = [
 224             { subdir => 'content' },
 225             { subdir => 'download-url' },
 226             { subdir => 'file-restore' },
 227             { subdir => 'prunebackups' },
 228             { subdir => 'rrd' },
 229             { subdir => 'rrddata' },
 230             { subdir => 'status' },
 231             { subdir => 'upload' },
 232         ];
 233
 234         return $res;
 235     }});
 236
 237 __PACKAGE__->register_method ({
 238     name => 'read_status',
 239     path => '{storage}/status',
 240     method => 'GET',
 241     description => "Read storage status.",
 242     permissions => {
 243         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 244     },
 245     protected => 1,
 246     proxyto => 'node',
 247     parameters => {
 248         additionalProperties => 0,
 249         properties => {
 250             node => get_standard_option('pve-node'),
 251             storage => get_standard_option('pve-storage-id'),
 252         },
 253     },
 254     returns => {
 255         type => "object",
 256         properties => {},
 257     },
 258     code => sub {
 259         my ($param) = @_;
 260
 261         my $cfg = PVE::Storage::config();
 262
 263         my $info = PVE::Storage::storage_info($cfg, $param->{content});
 264
 265         my $data = $info->{$param->{storage}};
 266
 267         raise_param_exc({ storage => "No such storage." })
 268             if !defined($data);
 269
 270         return $data;
 271     }});
 272
 273 __PACKAGE__->register_method ({
 274     name => 'rrd',
 275     path => '{storage}/rrd',
 276     method => 'GET',
 277     description => "Read storage RRD statistics (returns PNG).",
 278     permissions => {
 279         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 280     },
 281     protected => 1,
 282     proxyto => 'node',
 283     parameters => {
 284         additionalProperties => 0,
 285         properties => {
 286             node => get_standard_option('pve-node'),
 287             storage => get_standard_option('pve-storage-id'),
 288             timeframe => {
 289                 description => "Specify the time frame you are interested in.",
 290                 type => 'string',
 291                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 292             },
 293             ds => {
 294                 description => "The list of datasources you want to display.",
 295                 type => 'string', format => 'pve-configid-list',
 296             },
 297             cf => {
 298                 description => "The RRD consolidation function",
 299                 type => 'string',
 300                 enum => [ 'AVERAGE', 'MAX' ],
 301                 optional => 1,
 302             },
 303         },
 304     },
 305     returns => {
 306         type => "object",
 307         properties => {
 308             filename => { type => 'string' },
 309         },
 310     },
 311     code => sub {
 312         my ($param) = @_;
 313
 314         return PVE::RRD::create_rrd_graph(
 315             "pve2-storage/$param->{node}/$param->{storage}",
 316             $param->{timeframe}, $param->{ds}, $param->{cf});
 317     }});
 318
 319 __PACKAGE__->register_method ({
 320     name => 'rrddata',
 321     path => '{storage}/rrddata',
 322     method => 'GET',
 323     description => "Read storage RRD statistics.",
 324     permissions => {
 325         check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
 326     },
 327     protected => 1,
 328     proxyto => 'node',
 329     parameters => {
 330         additionalProperties => 0,
 331         properties => {
 332             node => get_standard_option('pve-node'),
 333             storage => get_standard_option('pve-storage-id'),
 334             timeframe => {
 335                 description => "Specify the time frame you are interested in.",
 336                 type => 'string',
 337                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 338             },
 339             cf => {
 340                 description => "The RRD consolidation function",
 341                 type => 'string',
 342                 enum => [ 'AVERAGE', 'MAX' ],
 343                 optional => 1,
 344             },
 345         },
 346     },
 347     returns => {
 348         type => "array",
 349         items => {
 350             type => "object",
 351             properties => {},
 352         },
 353     },
 354     code => sub {
 355         my ($param) = @_;
 356
 357         return PVE::RRD::create_rrd_data(
 358             "pve2-storage/$param->{node}/$param->{storage}",
 359             $param->{timeframe}, $param->{cf});
 360     }});
 361
 362 # makes no sense for big images and backup files (because it
 363 # create a copy of the file).
 364 __PACKAGE__->register_method ({
 365     name => 'upload',
 366     path => '{storage}/upload',
 367     method => 'POST',
 368     description => "Upload templates and ISO images.",
 369     permissions => {
 370         check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
 371     },
 372     protected => 1,
 373     parameters => {
 374         additionalProperties => 0,
 375         properties => {
 376             node => get_standard_option('pve-node'),
 377             storage => get_standard_option('pve-storage-id'),
 378             content => {
 379                 description => "Content type.",
 380                 type => 'string', format => 'pve-storage-content',
 381             },
 382             filename => {
 383                 description => "The name of the file to create.",
 384                 type => 'string',
 385             },
 386             tmpfilename => {
 387                 description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
 388                 type => 'string',
 389                 optional => 1,
 390             },
 391         },
 392     },
 393     returns => { type => "string" },
 394     code => sub {
 395         my ($param) = @_;
 396
 397         my $rpcenv = PVE::RPCEnvironment::get();
 398
 399         my $user = $rpcenv->get_user();
 400
 401         my $cfg = PVE::Storage::config();
 402
 403         my $node = $param->{node};
 404         my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
 405
 406         die "can't upload to storage type '$scfg->{type}'\n"
 407             if !defined($scfg->{path});
 408
 409         my $content = $param->{content};
 410
 411         my $tmpfilename = $param->{tmpfilename};
 412         die "missing temporary file name\n" if !$tmpfilename;
 413
 414         my $size = -s $tmpfilename;
 415         die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
 416
 417         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 418
 419         my $path;
 420
 421         if ($content eq 'iso') {
 422             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 423                 raise_param_exc({ filename => "wrong file extension" });
 424             }
 425             $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
 426         } elsif ($content eq 'vztmpl') {
 427             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 428                 raise_param_exc({ filename => "wrong file extension" });
 429             }
 430             $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
 431         } else {
 432             raise_param_exc({ content => "upload content type '$content' not allowed" });
 433         }
 434
 435         die "storage '$param->{storage}' does not support '$content' content\n"
 436             if !$scfg->{content}->{$content};
 437
 438         my $dest = "$path/$filename";
 439         my $dirname = dirname($dest);
 440
 441         # best effort to match apl_download behaviour
 442         chmod 0644, $tmpfilename;
 443
 444         # we simply overwrite the destination file if it already exists
 445
 446         my $cmd;
 447         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
 448             my $remip = PVE::Cluster::remote_node_ip($node);
 449
 450             my @ssh_options = ('-o', 'BatchMode=yes');
 451
 452             my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
 453
 454             eval {
 455                 # activate remote storage
 456                 PVE::Tools::run_command([@remcmd, '/usr/sbin/pvesm', 'status',
 457                                          '--storage', $param->{storage}]);
 458             };
 459             die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
 460
 461             PVE::Tools::run_command([@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
 462                                     errmsg => "mkdir failed");
 463
 464             $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
 465         } else {
 466             PVE::Storage::activate_storage($cfg, $param->{storage});
 467             File::Path::make_path($dirname);
 468             $cmd = ['cp', '--', $tmpfilename, $dest];
 469         }
 470
 471         my $worker = sub {
 472             my $upid = shift;
 473
 474             print "starting file import from: $tmpfilename\n";
 475             print "target node: $node\n";
 476             print "target file: $dest\n";
 477             print "file size is: $size\n";
 478             print "command: " . join(' ', @$cmd) . "\n";
 479
 480             eval { PVE::Tools::run_command($cmd, errmsg => 'import failed'); };
 481             if (my $err = $@) {
 482                 unlink $dest;
 483                 die $err;
 484             }
 485             print "finished file import successfully\n";
 486         };
 487
 488         my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
 489
 490         # apache removes the temporary file on return, so we need
 491         # to wait here to make sure the worker process starts and
 492         # opens the file before it gets removed.
 493         sleep(1);
 494
 495         return $upid;
 496    }});
 497
 498 __PACKAGE__->register_method({
 499     name => 'download_url',
 500     path => '{storage}/download-url',
 501     method => 'POST',
 502     description => "Download templates and ISO images by using an URL.",
 503     proxyto => 'node',
 504     permissions => {
 505         check => [ 'and',
 506             ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
 507             ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
 508         ],
 509     },
 510     protected => 1,
 511     parameters => {
 512         additionalProperties => 0,
 513         properties => {
 514             node => get_standard_option('pve-node'),
 515             storage => get_standard_option('pve-storage-id'),
 516             url => {
 517                 description => "The URL to download the file from.",
 518                 type => 'string',
 519                 pattern => 'https?://.*',
 520             },
 521             content => {
 522                 description => "Content type.", # TODO: could be optional & detected in most cases
 523                 type => 'string', format => 'pve-storage-content',
 524                 enum => ['iso', 'vztmpl'],
 525             },
 526             filename => {
 527                 description => "The name of the file to create. Caution: This will be normalized!",
 528                 type => 'string',
 529             },
 530             checksum => {
 531                 description => "The expected checksum of the file.",
 532                 type => 'string',
 533                 requires => 'checksum-algorithm',
 534                 optional => 1,
 535             },
 536             'checksum-algorithm' => {
 537                 description => "The algorithm to calculate the checksum of the file.",
 538                 type => 'string',
 539                 enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
 540                 requires => 'checksum',
 541                 optional => 1,
 542             },
 543             'verify-certificates' => {
 544                 description => "If false, no SSL/TLS certificates will be verified.",
 545                 type => 'boolean',
 546                 optional => 1,
 547                 default => 1,
 548             },
 549         },
 550     },
 551     returns => {
 552         type => "string"
 553     },
 554     code => sub {
 555         my ($param) = @_;
 556
 557         my $rpcenv = PVE::RPCEnvironment::get();
 558         my $user = $rpcenv->get_user();
 559
 560         my $cfg = PVE::Storage::config();
 561
 562         my ($node, $storage) = $param->@{'node', 'storage'};
 563         my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 564
 565         die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
 566             if !defined($scfg->{path});
 567
 568         my ($content, $url) = $param->@{'content', 'url'};
 569
 570         die "storage '$storage' is not configured for content-type '$content'\n"
 571             if !$scfg->{content}->{$content};
 572
 573         my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 574
 575         my $path;
 576         if ($content eq 'iso') {
 577             if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
 578                 raise_param_exc({ filename => "wrong file extension" });
 579             }
 580             $path = PVE::Storage::get_iso_dir($cfg, $storage);
 581         } elsif ($content eq 'vztmpl') {
 582             if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
 583                 raise_param_exc({ filename => "wrong file extension" });
 584             }
 585             $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
 586         } else {
 587             raise_param_exc({ content => "upload content-type '$content' is not allowed" });
 588         }
 589
 590         PVE::Storage::activate_storage($cfg, $storage);
 591         File::Path::make_path($path);
 592
 593         my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
 594         my $opts = {
 595             hash_required => 0,
 596             verify_certificates => $param->{'verify-certificates'} // 1,
 597             http_proxy => $dccfg->{http_proxy},
 598         };
 599
 600         my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
 601         if ($checksum) {
 602             $opts->{"${checksum_algorithm}sum"} = $checksum;
 603             $opts->{hash_required} = 1;
 604         }
 605
 606         my $worker = sub {
 607             PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 608         };
 609
 610         return $rpcenv->fork_worker('download', $filename, $user, $worker);
 611     }});
 612
 613 1;