]>
Commit | Line | Data |
---|---|---|
98fe93ae DC |
1 | package PVE::API2::Ceph::MON; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
6 | use Net::IP; | |
7 | use File::Path; | |
8 | ||
9 | use PVE::Ceph::Tools; | |
10 | use PVE::Ceph::Services; | |
11 | use PVE::Cluster qw(cfs_read_file cfs_write_file); | |
12 | use PVE::JSONSchema qw(get_standard_option); | |
13 | use PVE::Network; | |
14 | use PVE::RADOS; | |
15 | use PVE::RESTHandler; | |
16 | use PVE::RPCEnvironment; | |
17 | use PVE::Tools qw(run_command file_set_contents); | |
18 | ||
19 | use base qw(PVE::RESTHandler); | |
20 | ||
21 | my $find_mon_ip = sub { | |
22 | my ($pubnet, $node, $overwrite_ip) = @_; | |
23 | ||
24 | if (!$pubnet) { | |
25 | return $overwrite_ip // PVE::Cluster::remote_node_ip($node); | |
26 | } | |
27 | ||
28 | my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet); | |
a7a84cb4 | 29 | die "No active IP found for the requested ceph public network '$pubnet' on node '$node'\n" |
98fe93ae DC |
30 | if scalar(@$allowed_ips) < 1; |
31 | ||
32 | if (!$overwrite_ip) { | |
33 | if (scalar(@$allowed_ips) == 1) { | |
34 | return $allowed_ips->[0]; | |
35 | } | |
36 | die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n". | |
37 | join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n"; | |
38 | } else { | |
39 | if (grep { $_ eq $overwrite_ip } @$allowed_ips) { | |
40 | return $overwrite_ip; | |
41 | } | |
42 | die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n" | |
43 | if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet); | |
44 | ||
45 | die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"; | |
46 | } | |
47 | }; | |
48 | ||
49 | __PACKAGE__->register_method ({ | |
50 | name => 'listmon', | |
51 | path => '', | |
52 | method => 'GET', | |
53 | description => "Get Ceph monitor list.", | |
54 | proxyto => 'node', | |
55 | protected => 1, | |
56 | permissions => { | |
57 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
58 | }, | |
59 | parameters => { | |
60 | additionalProperties => 0, | |
61 | properties => { | |
62 | node => get_standard_option('pve-node'), | |
63 | }, | |
64 | }, | |
65 | returns => { | |
66 | type => 'array', | |
67 | items => { | |
68 | type => "object", | |
69 | properties => { | |
70 | name => { type => 'string' }, | |
d5373b7d DC |
71 | addr => { type => 'string', optional => 1 }, |
72 | host => { type => 'string', optional => 1 }, | |
98fe93ae DC |
73 | }, |
74 | }, | |
75 | links => [ { rel => 'child', href => "{name}" } ], | |
76 | }, | |
77 | code => sub { | |
78 | my ($param) = @_; | |
79 | ||
80 | PVE::Ceph::Tools::check_ceph_inited(); | |
81 | ||
82 | my $res = []; | |
83 | ||
84 | my $cfg = cfs_read_file('ceph.conf'); | |
85 | ||
86 | my $monhash = {}; | |
98fe93ae DC |
87 | |
88 | eval { | |
89 | my $rados = PVE::RADOS->new(); | |
d5373b7d | 90 | $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados); |
98fe93ae | 91 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); |
d5373b7d | 92 | |
98fe93ae DC |
93 | my $mons = $monstat->{monmap}->{mons}; |
94 | foreach my $d (@$mons) { | |
95 | next if !defined($d->{name}); | |
96 | $monhash->{$d->{name}}->{rank} = $d->{rank}; | |
97 | $monhash->{$d->{name}}->{addr} = $d->{addr}; | |
d5373b7d | 98 | $monhash->{$d->{name}}->{state} = 'running'; |
98fe93ae DC |
99 | if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) { |
100 | $monhash->{$d->{name}}->{quorum} = 1; | |
101 | } | |
102 | } | |
d5373b7d | 103 | |
98fe93ae DC |
104 | }; |
105 | warn $@ if $@; | |
106 | ||
107 | return PVE::RESTHandler::hash_to_array($monhash, 'name'); | |
108 | }}); | |
109 | ||
110 | __PACKAGE__->register_method ({ | |
111 | name => 'createmon', | |
112 | path => '', | |
113 | method => 'POST', | |
114 | description => "Create Ceph Monitor and Manager", | |
115 | proxyto => 'node', | |
116 | protected => 1, | |
117 | permissions => { | |
118 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
119 | }, | |
120 | parameters => { | |
121 | additionalProperties => 0, | |
122 | properties => { | |
123 | node => get_standard_option('pve-node'), | |
124 | id => { | |
125 | type => 'string', | |
126 | optional => 1, | |
127 | pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?', | |
128 | description => "The ID for the monitor, when omitted the same as the nodename", | |
129 | }, | |
130 | 'exclude-manager' => { | |
131 | type => 'boolean', | |
132 | optional => 1, | |
133 | default => 0, | |
134 | description => "When set, only a monitor will be created.", | |
135 | }, | |
136 | 'mon-address' => { | |
137 | description => 'Overwrites autodetected monitor IP address. ' . | |
138 | 'Must be in the public network of ceph.', | |
139 | type => 'string', format => 'ip', | |
140 | optional => 1, | |
141 | }, | |
142 | }, | |
143 | }, | |
144 | returns => { type => 'string' }, | |
145 | code => sub { | |
146 | my ($param) = @_; | |
147 | ||
148 | PVE::Ceph::Tools::check_ceph_installed('ceph_mon'); | |
149 | ||
150 | PVE::Ceph::Tools::check_ceph_installed('ceph_mgr') | |
151 | if (!$param->{'exclude-manager'}); | |
152 | ||
153 | PVE::Ceph::Tools::check_ceph_inited(); | |
154 | ||
155 | PVE::Ceph::Tools::setup_pve_symlinks(); | |
156 | ||
157 | my $rpcenv = PVE::RPCEnvironment::get(); | |
158 | ||
159 | my $authuser = $rpcenv->get_user(); | |
160 | ||
161 | my $cfg = cfs_read_file('ceph.conf'); | |
162 | ||
163 | my $moncount = 0; | |
164 | ||
165 | my $monaddrhash = {}; | |
166 | ||
98fe93ae DC |
167 | foreach my $section (keys %$cfg) { |
168 | next if $section eq 'global'; | |
169 | my $d = $cfg->{$section}; | |
170 | if ($section =~ m/^mon\./) { | |
171 | $moncount++; | |
172 | if ($d->{'mon addr'}) { | |
173 | $monaddrhash->{$d->{'mon addr'}} = $section; | |
174 | } | |
175 | } | |
176 | } | |
177 | ||
178 | my $monid = $param->{id} // $param->{node}; | |
179 | ||
180 | my $monsection = "mon.$monid"; | |
181 | my $pubnet = $cfg->{global}->{'public network'}; | |
182 | my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'}); | |
183 | ||
184 | my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789"; | |
185 | my $monname = $param->{node}; | |
186 | ||
187 | die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; | |
188 | die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" | |
189 | if $monaddrhash->{$monaddr}; | |
190 | ||
191 | my $worker = sub { | |
192 | my $upid = shift; | |
193 | ||
194 | my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path'); | |
195 | ||
196 | if (! -f $pve_ckeyring_path) { | |
197 | run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . | |
198 | "--gen-key -n client.admin"); | |
199 | } | |
200 | ||
201 | my $pve_mon_key_path = PVE::Ceph::Tools::get_config('pve_mon_key_path'); | |
202 | if (! -f $pve_mon_key_path) { | |
203 | run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); | |
97ed02b8 | 204 | run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin " . |
98fe93ae DC |
205 | "--cap mds 'allow' " . |
206 | "--cap osd 'allow *' " . | |
207 | "--cap mgr 'allow *' " . | |
208 | "--cap mon 'allow *'"); | |
bba5c712 TL |
209 | run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring"); |
210 | run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring"); | |
98fe93ae DC |
211 | run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); |
212 | run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); | |
213 | } | |
214 | ||
215 | my $ccname = PVE::Ceph::Tools::get_config('ccname'); | |
216 | ||
217 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; | |
218 | -d $mondir && die "monitor filesystem '$mondir' already exist\n"; | |
219 | ||
220 | my $monmap = "/tmp/monmap"; | |
221 | ||
222 | eval { | |
223 | mkdir $mondir; | |
224 | ||
bba5c712 | 225 | run_command("chown ceph:ceph $mondir"); |
98fe93ae DC |
226 | |
227 | if ($moncount > 0) { | |
228 | my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); | |
229 | my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' }); | |
230 | file_set_contents($monmap, $mapdata); | |
231 | } else { | |
232 | run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); | |
233 | } | |
234 | ||
235 | run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); | |
bba5c712 | 236 | run_command("chown ceph:ceph -R $mondir"); |
98fe93ae DC |
237 | }; |
238 | my $err = $@; | |
239 | unlink $monmap; | |
240 | if ($err) { | |
241 | File::Path::remove_tree($mondir); | |
242 | die $err; | |
243 | } | |
244 | ||
245 | $cfg->{$monsection} = { | |
246 | 'host' => $monname, | |
247 | 'mon addr' => $monaddr, | |
248 | }; | |
249 | ||
250 | cfs_write_file('ceph.conf', $cfg); | |
251 | ||
252 | my $create_keys_pid = fork(); | |
253 | if (!defined($create_keys_pid)) { | |
254 | die "Could not spawn ceph-create-keys to create bootstrap keys\n"; | |
255 | } elsif ($create_keys_pid == 0) { | |
256 | exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]); | |
257 | } else { | |
258 | PVE::Ceph::Services::ceph_service_cmd('start', $monsection); | |
259 | ||
bba5c712 TL |
260 | # to ensure we have the correct startup order. |
261 | eval { run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]) }; | |
262 | warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@; | |
98fe93ae DC |
263 | waitpid($create_keys_pid, 0); |
264 | } | |
265 | ||
266 | # create manager | |
267 | if (!$param->{'exclude-manager'}) { | |
268 | my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); | |
269 | PVE::Ceph::Services::create_mgr($monid, $rados); | |
270 | } | |
4e76dbd7 | 271 | PVE::Ceph::Services::broadcast_ceph_services(); |
98fe93ae DC |
272 | }; |
273 | ||
274 | return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker); | |
275 | }}); | |
276 | ||
277 | __PACKAGE__->register_method ({ | |
278 | name => 'destroymon', | |
279 | path => '{monid}', | |
280 | method => 'DELETE', | |
281 | description => "Destroy Ceph Monitor and Manager.", | |
282 | proxyto => 'node', | |
283 | protected => 1, | |
284 | permissions => { | |
285 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
286 | }, | |
287 | parameters => { | |
288 | additionalProperties => 0, | |
289 | properties => { | |
290 | node => get_standard_option('pve-node'), | |
291 | monid => { | |
292 | description => 'Monitor ID', | |
293 | type => 'string', | |
294 | pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?', | |
295 | }, | |
296 | 'exclude-manager' => { | |
297 | type => 'boolean', | |
298 | default => 0, | |
299 | optional => 1, | |
300 | description => "When set, removes only the monitor, not the manager" | |
301 | } | |
302 | }, | |
303 | }, | |
304 | returns => { type => 'string' }, | |
305 | code => sub { | |
306 | my ($param) = @_; | |
307 | ||
308 | my $rpcenv = PVE::RPCEnvironment::get(); | |
309 | ||
310 | my $authuser = $rpcenv->get_user(); | |
311 | ||
312 | PVE::Ceph::Tools::check_ceph_inited(); | |
313 | ||
314 | my $cfg = cfs_read_file('ceph.conf'); | |
315 | ||
316 | my $monid = $param->{monid}; | |
317 | my $monsection = "mon.$monid"; | |
318 | ||
319 | my $rados = PVE::RADOS->new(); | |
320 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); | |
321 | my $monlist = $monstat->{monmap}->{mons}; | |
322 | ||
323 | die "no such monitor id '$monid'\n" | |
324 | if !defined($cfg->{$monsection}); | |
325 | ||
326 | my $ccname = PVE::Ceph::Tools::get_config('ccname'); | |
327 | ||
328 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; | |
329 | -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; | |
330 | ||
331 | die "can't remove last monitor\n" if scalar(@$monlist) <= 1; | |
332 | ||
333 | my $worker = sub { | |
334 | my $upid = shift; | |
335 | ||
336 | # reopen with longer timeout | |
337 | $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); | |
338 | ||
339 | $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' }); | |
340 | ||
341 | eval { PVE::Ceph::Services::ceph_service_cmd('stop', $monsection); }; | |
342 | warn $@ if $@; | |
343 | ||
344 | delete $cfg->{$monsection}; | |
345 | cfs_write_file('ceph.conf', $cfg); | |
346 | File::Path::remove_tree($mondir); | |
347 | ||
348 | # remove manager | |
349 | if (!$param->{'exclude-manager'}) { | |
350 | eval { PVE::Ceph::Services::destroy_mgr($monid) }; | |
351 | warn $@ if $@; | |
352 | } | |
4e76dbd7 | 353 | PVE::Ceph::Services::broadcast_ceph_services(); |
98fe93ae DC |
354 | }; |
355 | ||
356 | return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker); | |
357 | }}); | |
358 | ||
359 | 1; |