]>
git.proxmox.com Git - pve-manager.git/blob - PVE/API2/Ceph.pm
1 package PVE
::API2
::Ceph
;
11 use PVE
::Ceph
::Services
;
12 use PVE
::Cluster
qw(cfs_read_file cfs_write_file);
13 use PVE
::JSONSchema
qw(get_standard_option);
17 use PVE
::RPCEnvironment
;
19 use PVE
::Tools
qw(run_command file_get_contents file_set_contents);
21 use PVE
::API2
::Ceph
::OSD
;
22 use PVE
::API2
::Ceph
::FS
;
23 use PVE
::API2
::Ceph
::MDS
;
24 use PVE
::API2
::Ceph
::MGR
;
25 use PVE
::API2
::Ceph
::MON
;
26 use PVE
::API2
::Storage
::Config
;
28 use base
qw(PVE::RESTHandler);
30 my $pve_osd_default_journal_size = 1024*5;
32 __PACKAGE__-
>register_method ({
33 subclass
=> "PVE::API2::Ceph::OSD",
37 __PACKAGE__-
>register_method ({
38 subclass
=> "PVE::API2::Ceph::MDS",
42 __PACKAGE__-
>register_method ({
43 subclass
=> "PVE::API2::Ceph::MGR",
47 __PACKAGE__-
>register_method ({
48 subclass
=> "PVE::API2::Ceph::MON",
52 __PACKAGE__-
>register_method ({
53 subclass
=> "PVE::API2::Ceph::FS",
57 __PACKAGE__-
>register_method ({
61 description
=> "Directory index.",
62 permissions
=> { user
=> 'all' },
64 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
67 additionalProperties
=> 0,
69 node
=> get_standard_option
('pve-node'),
78 links
=> [ { rel
=> 'child', href
=> "{name}" } ],
92 { name
=> 'restart' },
105 __PACKAGE__-
>register_method ({
109 description
=> "List local disks.",
113 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
116 additionalProperties
=> 0,
118 node
=> get_standard_option
('pve-node'),
120 description
=> "Only list specific types of disks.",
122 enum
=> ['unused', 'journal_disks'],
132 dev
=> { type
=> 'string' },
133 used
=> { type
=> 'string', optional
=> 1 },
134 gpt
=> { type
=> 'boolean' },
135 size
=> { type
=> 'integer' },
136 osdid
=> { type
=> 'integer' },
137 vendor
=> { type
=> 'string', optional
=> 1 },
138 model
=> { type
=> 'string', optional
=> 1 },
139 serial
=> { type
=> 'string', optional
=> 1 },
142 # links => [ { rel => 'child', href => "{}" } ],
147 PVE
::Ceph
::Tools
::check_ceph_inited
();
149 my $disks = PVE
::Diskmanage
::get_disks
(undef, 1);
152 foreach my $dev (keys %$disks) {
153 my $d = $disks->{$dev};
154 if ($param->{type
}) {
155 if ($param->{type
} eq 'journal_disks') {
156 next if $d->{osdid
} >= 0;
158 } elsif ($param->{type
} eq 'unused') {
161 die "internal error"; # should not happen
165 $d->{dev
} = "/dev/$dev";
172 __PACKAGE__-
>register_method ({
178 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
180 description
=> "Get Ceph configuration.",
182 additionalProperties
=> 0,
184 node
=> get_standard_option
('pve-node'),
187 returns
=> { type
=> 'string' },
191 PVE
::Ceph
::Tools
::check_ceph_inited
();
193 my $path = PVE
::Ceph
::Tools
::get_config
('pve_ceph_cfgpath');
194 return file_get_contents
($path);
198 my $add_storage = sub {
199 my ($pool, $storeid) = @_;
201 my $storage_params = {
206 content
=> 'rootdir,images',
209 PVE
::API2
::Storage
::Config-
>create($storage_params);
212 my $get_storages = sub {
215 my $cfg = PVE
::Storage
::config
();
217 my $storages = $cfg->{ids
};
219 foreach my $storeid (keys %$storages) {
220 my $curr = $storages->{$storeid};
221 $res->{$storeid} = $storages->{$storeid}
222 if $curr->{type
} eq 'rbd' && $pool eq $curr->{pool
};
228 __PACKAGE__-
>register_method ({
232 description
=> "Create initial ceph default configuration and setup symlinks.",
236 check
=> ['perm', '/', [ 'Sys.Modify' ]],
239 additionalProperties
=> 0,
241 node
=> get_standard_option
('pve-node'),
243 description
=> "Use specific network for all ceph related traffic",
244 type
=> 'string', format
=> 'CIDR',
248 'cluster-network' => {
249 description
=> "Declare a separate cluster network, OSDs will route" .
250 "heartbeat, object replication and recovery traffic over it",
251 type
=> 'string', format
=> 'CIDR',
252 requires
=> 'network',
257 description
=> 'Targeted number of replicas per object',
265 description
=> 'Minimum number of available replicas per object to allow I/O',
273 description
=> "Placement group bits, used to specify the " .
274 "default number of placement groups.\n\nNOTE: 'osd pool " .
275 "default pg num' does not work for default pools.",
283 description
=> "Disable cephx authentication.\n\n" .
284 "WARNING: cephx is a security feature protecting against " .
285 "man-in-the-middle attacks. Only consider disabling cephx ".
286 "if your network is private!",
293 returns
=> { type
=> 'null' },
297 my $version = PVE
::Ceph
::Tools
::get_local_version
(1);
299 if (!$version || $version < 14) {
300 die "Ceph Nautilus required - please run 'pveceph install'\n";
302 PVE
::Ceph
::Tools
::check_ceph_installed
('ceph_bin');
305 # simply load old config if it already exists
306 my $cfg = cfs_read_file
('ceph.conf');
308 if (!$cfg->{global
}) {
313 UUID
::generate
($uuid);
314 UUID
::unparse
($uuid, $fsid);
316 my $auth = $param->{disable_cephx
} ?
'none' : 'cephx';
320 'auth cluster required' => $auth,
321 'auth service required' => $auth,
322 'auth client required' => $auth,
323 'osd pool default size' => $param->{size
} // 3,
324 'osd pool default min size' => $param->{min_size
} // 2,
325 'mon allow pool delete' => 'true',
328 # this does not work for default pools
329 #'osd pool default pg num' => $pg_num,
330 #'osd pool default pgp num' => $pg_num,
333 $cfg->{client
}->{keyring
} = '/etc/pve/priv/$cluster.$name.keyring';
335 if ($param->{pg_bits
}) {
336 $cfg->{global
}->{'osd pg bits'} = $param->{pg_bits
};
337 $cfg->{global
}->{'osd pgp bits'} = $param->{pg_bits
};
340 if ($param->{network
}) {
341 $cfg->{global
}->{'public network'} = $param->{network
};
342 $cfg->{global
}->{'cluster network'} = $param->{network
};
345 if ($param->{'cluster-network'}) {
346 $cfg->{global
}->{'cluster network'} = $param->{'cluster-network'};
349 cfs_write_file
('ceph.conf', $cfg);
351 PVE
::Ceph
::Tools
::setup_pve_symlinks
();
356 __PACKAGE__-
>register_method ({
360 description
=> "Stop ceph services.",
364 check
=> ['perm', '/', [ 'Sys.Modify' ]],
367 additionalProperties
=> 0,
369 node
=> get_standard_option
('pve-node'),
371 description
=> 'Ceph service name.',
374 default => 'ceph.target',
375 pattern
=> '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
379 returns
=> { type
=> 'string' },
383 my $rpcenv = PVE
::RPCEnvironment
::get
();
385 my $authuser = $rpcenv->get_user();
387 PVE
::Ceph
::Tools
::check_ceph_inited
();
389 my $cfg = cfs_read_file
('ceph.conf');
390 scalar(keys %$cfg) || die "no configuration\n";
396 if ($param->{service
}) {
397 push @$cmd, $param->{service
};
400 PVE
::Ceph
::Services
::ceph_service_cmd
(@$cmd);
403 return $rpcenv->fork_worker('srvstop', $param->{service
} || 'ceph',
407 __PACKAGE__-
>register_method ({
411 description
=> "Start ceph services.",
415 check
=> ['perm', '/', [ 'Sys.Modify' ]],
418 additionalProperties
=> 0,
420 node
=> get_standard_option
('pve-node'),
422 description
=> 'Ceph service name.',
425 default => 'ceph.target',
426 pattern
=> '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
430 returns
=> { type
=> 'string' },
434 my $rpcenv = PVE
::RPCEnvironment
::get
();
436 my $authuser = $rpcenv->get_user();
438 PVE
::Ceph
::Tools
::check_ceph_inited
();
440 my $cfg = cfs_read_file
('ceph.conf');
441 scalar(keys %$cfg) || die "no configuration\n";
447 if ($param->{service
}) {
448 push @$cmd, $param->{service
};
451 PVE
::Ceph
::Services
::ceph_service_cmd
(@$cmd);
454 return $rpcenv->fork_worker('srvstart', $param->{service
} || 'ceph',
458 __PACKAGE__-
>register_method ({
462 description
=> "Restart ceph services.",
466 check
=> ['perm', '/', [ 'Sys.Modify' ]],
469 additionalProperties
=> 0,
471 node
=> get_standard_option
('pve-node'),
473 description
=> 'Ceph service name.',
476 default => 'ceph.target',
477 pattern
=> '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
481 returns
=> { type
=> 'string' },
485 my $rpcenv = PVE
::RPCEnvironment
::get
();
487 my $authuser = $rpcenv->get_user();
489 PVE
::Ceph
::Tools
::check_ceph_inited
();
491 my $cfg = cfs_read_file
('ceph.conf');
492 scalar(keys %$cfg) || die "no configuration\n";
497 my $cmd = ['restart'];
498 if ($param->{service
}) {
499 push @$cmd, $param->{service
};
502 PVE
::Ceph
::Services
::ceph_service_cmd
(@$cmd);
505 return $rpcenv->fork_worker('srvrestart', $param->{service
} || 'ceph',
509 __PACKAGE__-
>register_method ({
513 description
=> "Get ceph status.",
517 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
520 additionalProperties
=> 0,
522 node
=> get_standard_option
('pve-node'),
525 returns
=> { type
=> 'object' },
529 PVE
::Ceph
::Tools
::check_ceph_inited
();
531 my $rados = PVE
::RADOS-
>new();
532 my $status = $rados->mon_command({ prefix
=> 'status' });
533 $status->{health
} = $rados->mon_command({ prefix
=> 'health', detail
=> 'detail' });
537 __PACKAGE__-
>register_method ({
541 description
=> "List all pools.",
545 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
548 additionalProperties
=> 0,
550 node
=> get_standard_option
('pve-node'),
558 pool
=> { type
=> 'integer' },
559 pool_name
=> { type
=> 'string' },
560 size
=> { type
=> 'integer' },
563 links
=> [ { rel
=> 'child', href
=> "{pool_name}" } ],
568 PVE
::Ceph
::Tools
::check_ceph_inited
();
570 my $rados = PVE
::RADOS-
>new();
573 my $res = $rados->mon_command({ prefix
=> 'df' });
575 foreach my $d (@{$res->{pools
}}) {
576 next if !$d->{stats
};
577 next if !defined($d->{id
});
578 $stats->{$d->{id
}} = $d->{stats
};
581 $res = $rados->mon_command({ prefix
=> 'osd dump' });
582 my $rulestmp = $rados->mon_command({ prefix
=> 'osd crush rule dump'});
585 for my $rule (@$rulestmp) {
586 $rules->{$rule->{rule_id
}} = $rule->{rule_name
};
590 foreach my $e (@{$res->{pools
}}) {
592 foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
593 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
596 if (defined($d->{crush_rule
}) && defined($rules->{$d->{crush_rule
}})) {
597 $d->{crush_rule_name
} = $rules->{$d->{crush_rule
}};
600 if (my $s = $stats->{$d->{pool
}}) {
601 $d->{bytes_used
} = $s->{bytes_used
};
602 $d->{percent_used
} = $s->{percent_used
};
611 __PACKAGE__-
>register_method ({
612 name
=> 'createpool',
615 description
=> "Create POOL",
619 check
=> ['perm', '/', [ 'Sys.Modify' ]],
622 additionalProperties
=> 0,
624 node
=> get_standard_option
('pve-node'),
626 description
=> "The name of the pool. It must be unique.",
630 description
=> 'Number of replicas per object',
638 description
=> 'Minimum number of replicas per object',
646 description
=> "Number of placement groups.",
654 description
=> "The rule to use for mapping object placement in the cluster.",
659 description
=> "The application of the pool, 'rbd' by default.",
661 enum
=> ['rbd', 'cephfs', 'rgw'],
665 description
=> "Configure VM and CT storage using the new pool.",
671 returns
=> { type
=> 'string' },
675 PVE
::Cluster
::check_cfs_quorum
();
676 PVE
::Ceph
::Tools
::check_ceph_inited
();
678 my $pve_ckeyring_path = PVE
::Ceph
::Tools
::get_config
('pve_ckeyring_path');
680 die "not fully configured - missing '$pve_ckeyring_path'\n"
681 if ! -f
$pve_ckeyring_path;
683 my $pool = $param->{name
};
684 my $rpcenv = PVE
::RPCEnvironment
::get
();
685 my $user = $rpcenv->get_user();
687 if ($param->{add_storages
}) {
688 $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
689 die "pool name contains characters which are illegal for storage naming\n"
690 if !PVE
::JSONSchema
::parse_storage_id
($pool);
693 my $pg_num = $param->{pg_num
} || 128;
694 my $size = $param->{size
} || 3;
695 my $min_size = $param->{min_size
} || 2;
696 my $application = $param->{application
} // 'rbd';
700 PVE
::Ceph
::Tools
::create_pool
($pool, $param);
702 if ($param->{add_storages
}) {
704 eval { $add_storage->($pool, "${pool}"); };
706 warn "failed to add storage: $@";
709 die "adding storage for pool '$pool' failed, check log and add manually!\n"
714 return $rpcenv->fork_worker('cephcreatepool', $pool, $user, $worker);
717 __PACKAGE__-
>register_method ({
721 description
=> "get all set ceph flags",
725 check
=> ['perm', '/', [ 'Sys.Audit' ]],
728 additionalProperties
=> 0,
730 node
=> get_standard_option
('pve-node'),
733 returns
=> { type
=> 'string' },
737 PVE
::Ceph
::Tools
::check_ceph_inited
();
739 my $pve_ckeyring_path = PVE
::Ceph
::Tools
::get_config
('pve_ckeyring_path');
741 die "not fully configured - missing '$pve_ckeyring_path'\n"
742 if ! -f
$pve_ckeyring_path;
744 my $rados = PVE
::RADOS-
>new();
746 my $stat = $rados->mon_command({ prefix
=> 'osd dump' });
748 return $stat->{flags
} // '';
751 __PACKAGE__-
>register_method ({
753 path
=> 'flags/{flag}',
755 description
=> "Set a ceph flag",
759 check
=> ['perm', '/', [ 'Sys.Modify' ]],
762 additionalProperties
=> 0,
764 node
=> get_standard_option
('pve-node'),
766 description
=> 'The ceph flag to set/unset',
768 enum
=> [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
772 returns
=> { type
=> 'null' },
776 PVE
::Ceph
::Tools
::check_ceph_inited
();
778 my $pve_ckeyring_path = PVE
::Ceph
::Tools
::get_config
('pve_ckeyring_path');
780 die "not fully configured - missing '$pve_ckeyring_path'\n"
781 if ! -f
$pve_ckeyring_path;
783 my $set = $param->{set
} // !$param->{unset
};
784 my $rados = PVE
::RADOS-
>new();
786 $rados->mon_command({
788 key
=> $param->{flag
},
794 __PACKAGE__-
>register_method ({
795 name
=> 'unset_flag',
796 path
=> 'flags/{flag}',
798 description
=> "Unset a ceph flag",
802 check
=> ['perm', '/', [ 'Sys.Modify' ]],
805 additionalProperties
=> 0,
807 node
=> get_standard_option
('pve-node'),
809 description
=> 'The ceph flag to set/unset',
811 enum
=> [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
815 returns
=> { type
=> 'null' },
819 PVE
::Ceph
::Tools
::check_ceph_inited
();
821 my $pve_ckeyring_path = PVE
::Ceph
::Tools
::get_config
('pve_ckeyring_path');
823 die "not fully configured - missing '$pve_ckeyring_path'\n"
824 if ! -f
$pve_ckeyring_path;
826 my $set = $param->{set
} // !$param->{unset
};
827 my $rados = PVE
::RADOS-
>new();
829 $rados->mon_command({
830 prefix
=> "osd unset",
831 key
=> $param->{flag
},
837 __PACKAGE__-
>register_method ({
838 name
=> 'destroypool',
839 path
=> 'pools/{name}',
841 description
=> "Destroy pool",
845 check
=> ['perm', '/', [ 'Sys.Modify' ]],
848 additionalProperties
=> 0,
850 node
=> get_standard_option
('pve-node'),
852 description
=> "The name of the pool. It must be unique.",
856 description
=> "If true, destroys pool even if in use",
862 description
=> "Remove all pveceph-managed storages configured for this pool",
869 returns
=> { type
=> 'string' },
873 PVE
::Ceph
::Tools
::check_ceph_inited
();
875 my $rpcenv = PVE
::RPCEnvironment
::get
();
876 my $user = $rpcenv->get_user();
877 $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
878 if $param->{remove_storages
};
880 my $pool = $param->{name
};
883 my $storages = $get_storages->($pool);
885 # if not forced, destroy ceph pool only when no
886 # vm disks are on it anymore
887 if (!$param->{force
}) {
888 my $storagecfg = PVE
::Storage
::config
();
889 foreach my $storeid (keys %$storages) {
890 my $storage = $storages->{$storeid};
892 # check if any vm disks are on the pool
893 print "checking storage '$storeid' for RBD images..\n";
894 my $res = PVE
::Storage
::vdisk_list
($storagecfg, $storeid);
895 die "ceph pool '$pool' still in use by storage '$storeid'\n"
896 if @{$res->{$storeid}} != 0;
900 PVE
::Ceph
::Tools
::destroy_pool
($pool);
902 if ($param->{remove_storages
}) {
904 foreach my $storeid (keys %$storages) {
905 # skip external clusters, not managed by pveceph
906 next if $storages->{$storeid}->{monhost
};
907 eval { PVE
::API2
::Storage
::Config-
>delete({storage
=> $storeid}) };
909 warn "failed to remove storage '$storeid': $@\n";
913 die "failed to remove (some) storages - check log and remove manually!\n"
917 return $rpcenv->fork_worker('cephdestroypool', $pool, $user, $worker);
921 __PACKAGE__-
>register_method ({
925 description
=> "Get OSD crush map",
929 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
932 additionalProperties
=> 0,
934 node
=> get_standard_option
('pve-node'),
937 returns
=> { type
=> 'string' },
941 PVE
::Ceph
::Tools
::check_ceph_inited
();
943 # this produces JSON (difficult to read for the user)
944 # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
948 my $mapfile = "/var/tmp/ceph-crush.map.$$";
949 my $mapdata = "/var/tmp/ceph-crush.txt.$$";
951 my $rados = PVE
::RADOS-
>new();
954 my $bindata = $rados->mon_command({ prefix
=> 'osd getcrushmap', format
=> 'plain' });
955 file_set_contents
($mapfile, $bindata);
956 run_command
(['crushtool', '-d', $mapfile, '-o', $mapdata]);
957 $txt = file_get_contents
($mapdata);
969 __PACKAGE__-
>register_method({
973 description
=> "Read ceph log",
976 check
=> ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
980 additionalProperties
=> 0,
982 node
=> get_standard_option
('pve-node'),
1001 description
=> "Line number",
1005 description
=> "Line text",
1014 PVE
::Ceph
::Tools
::check_ceph_inited
();
1016 my $rpcenv = PVE
::RPCEnvironment
::get
();
1017 my $user = $rpcenv->get_user();
1018 my $node = $param->{node
};
1020 my $logfile = "/var/log/ceph/ceph.log";
1021 my ($count, $lines) = PVE
::Tools
::dump_logfile
($logfile, $param->{start
}, $param->{limit
});
1023 $rpcenv->set_result_attrib('total', $count);
1028 __PACKAGE__-
>register_method ({
1032 description
=> "List ceph rules.",
1036 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
1039 additionalProperties
=> 0,
1041 node
=> get_standard_option
('pve-node'),
1050 links
=> [ { rel
=> 'child', href
=> "{name}" } ],
1055 PVE
::Ceph
::Tools
::check_ceph_inited
();
1057 my $rados = PVE
::RADOS-
>new();
1059 my $rules = $rados->mon_command({ prefix
=> 'osd crush rule ls' });
1063 foreach my $rule (@$rules) {
1064 push @$res, { name
=> $rule };