]>
git.proxmox.com Git - pve-manager.git/blob - PVE/API2/Ceph.pm
1 package PVE
:: API2
:: CephOSD
;
8 use PVE
:: Tools
qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach) ;
9 use PVE
:: Exception
qw(raise raise_param_exc) ;
11 use PVE
:: Cluster
qw(cfs_lock_file cfs_read_file cfs_write_file) ;
12 use PVE
:: AccessControl
;
15 use PVE
:: RPCEnvironment
;
16 use PVE
:: JSONSchema
qw(get_standard_option) ;
20 use base
qw(PVE::RESTHandler) ;
22 use Data
:: Dumper
; # fixme: remove
24 my $get_osd_status = sub {
25 my ( $rados, $osdid ) = @_ ;
27 my $stat = $rados -> mon_command ({ prefix
=> 'osd dump' });
29 my $osdlist = $stat ->{ osds
} || [];
32 foreach my $d ( @$osdlist ) {
33 $osdstat ->{ $d ->{ osd
}} = $d if defined ( $d ->{ osd
});
35 if ( defined ( $osdid )) {
36 die "no such OSD ' $osdid ' \n " if ! $osdstat ->{ $osdid };
37 return $osdstat ->{ $osdid };
43 my $get_osd_usage = sub {
46 my $osdlist = $rados -> mon_command ({ prefix
=> 'pg dump' ,
47 dumpcontents
=> [ 'osds' ]}) || [];
50 foreach my $d ( @$osdlist ) {
51 $osdstat ->{ $d ->{ osd
}} = $d if defined ( $d ->{ osd
});
57 __PACKAGE__-
> register_method ({
61 description
=> "Get Ceph osd list/tree." ,
65 additionalProperties
=> 0 ,
67 node
=> get_standard_option
( 'pve-node' ),
70 # fixme: return a list instead of extjs tree format ?
77 PVE
:: CephTools
:: check_ceph_inited
();
79 my $rados = PVE
:: RADOS-
> new ();
80 my $res = $rados -> mon_command ({ prefix
=> 'osd tree' });
82 die "no tree nodes found \n " if !( $res && $res ->{ nodes
});
84 my $osdhash = & $get_osd_status ( $rados );
86 my $usagehash = & $get_osd_usage ( $rados );
90 foreach my $e (@{ $res ->{ nodes
}}) {
91 $nodes ->{ $e ->{ id
}} = $e ;
99 foreach my $opt ( qw(status crush_weight reweight) ) {
100 $new ->{ $opt } = $e ->{ $opt } if defined ( $e ->{ $opt });
103 if ( my $stat = $osdhash ->{ $e ->{ id
}}) {
104 $new ->{ in } = $stat ->{ in } if defined ( $stat ->{ in });
107 if ( my $stat = $usagehash ->{ $e ->{ id
}}) {
108 $new ->{ total_space
} = ( $stat ->{ kb
} || 1 ) * 1024 ;
109 $new ->{ bytes_used
} = ( $stat ->{ kb_used
} || 0 ) * 1024 ;
110 $new ->{ percent_used
} = ( $new ->{ bytes_used
}* 100 )/ $new ->{ total_space
};
111 if ( my $d = $stat ->{ fs_perf_stat
}) {
112 $new ->{ commit_latency_ms
} = $d ->{ commit_latency_ms
};
113 $new ->{ apply_latency_ms
} = $d ->{ apply_latency_ms
};
117 $newnodes ->{ $e ->{ id
}} = $new ;
120 foreach my $e (@{ $res ->{ nodes
}}) {
121 my $new = $newnodes ->{ $e ->{ id
}};
122 if ( $e ->{ children
} && scalar (@{ $e ->{ children
}})) {
123 $new ->{ children
} = [];
125 foreach my $cid (@{ $e ->{ children
}}) {
126 $nodes ->{ $cid }->{ parent
} = $e ->{ id
};
127 if ( $nodes ->{ $cid }->{ type
} eq 'osd' &&
128 $e ->{ type
} eq 'host' ) {
129 $newnodes ->{ $cid }->{ host
} = $e ->{ name
};
131 push @{ $new ->{ children
}}, $newnodes ->{ $cid };
134 $new ->{ leaf
} = ( $e ->{ id
} >= 0 ) ?
1 : 0 ;
139 foreach my $e (@{ $res ->{ nodes
}}) {
140 if (! $nodes ->{ $e ->{ id
}}->{ parent
}) {
141 $rootnode = $newnodes ->{ $e ->{ id
}};
146 die "no root node \n " if ! $rootnode ;
148 my $data = { root
=> $rootnode };
153 __PACKAGE__-
> register_method ({
157 description
=> "Create OSD" ,
161 additionalProperties
=> 0 ,
163 node
=> get_standard_option
( 'pve-node' ),
165 description
=> "Block device name." ,
169 description
=> "Block device name for journal." ,
174 description
=> "File system type." ,
176 enum
=> [ 'xfs' , 'ext4' , 'btrfs' ],
182 returns
=> { type
=> 'string' },
186 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
188 my $authuser = $rpcenv -> get_user ();
190 PVE
:: CephTools
:: check_ceph_inited
();
192 PVE
:: CephTools
:: setup_pve_symlinks
();
196 if ( $param ->{ journal_dev
} && ( $param ->{ journal_dev
} ne $param ->{ dev
})) {
197 $journal_dev = PVE
:: CephTools
:: verify_blockdev_path
( $param ->{ journal_dev
});
200 $param ->{ dev
} = PVE
:: CephTools
:: verify_blockdev_path
( $param ->{ dev
});
202 my $disklist = PVE
:: CephTools
:: list_disks
();
204 my $devname = $param ->{ dev
};
205 $devname =~ s
| /dev/ ||;
207 my $diskinfo = $disklist ->{ $devname };
208 die "unable to get device info for ' $devname ' \n "
211 die "device ' $param ->{dev}' is in use \n "
212 if $diskinfo ->{ used
};
214 my $rados = PVE
:: RADOS-
> new ();
215 my $monstat = $rados -> mon_command ({ prefix
=> 'mon_status' });
216 die "unable to get fsid \n " if ! $monstat ->{ monmap
} || ! $monstat ->{ monmap
}->{ fsid
};
218 my $fsid = $monstat ->{ monmap
}->{ fsid
};
219 $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/ ;
221 my $ceph_bootstrap_osd_keyring = PVE
:: CephTools
:: get_config
( 'ceph_bootstrap_osd_keyring' );
223 if (! - f
$ceph_bootstrap_osd_keyring ) {
224 my $bindata = $rados -> mon_command ({ prefix
=> 'auth get' , entity
=> 'client.bootstrap-osd' , format
=> 'plain' });
225 PVE
:: Tools
:: file_set_contents
( $ceph_bootstrap_osd_keyring, $bindata );
231 my $fstype = $param ->{ fstype
} || 'xfs' ;
233 print "create OSD on $param ->{dev} ( $fstype ) \n " ;
235 my $ccname = PVE
:: CephTools
:: get_config
( 'ccname' );
237 my $cmd = [ 'ceph-disk' , 'prepare' , '--zap-disk' , '--fs-type' , $fstype,
238 '--cluster' , $ccname, '--cluster-uuid' , $fsid ];
241 print "using device ' $journal_dev ' for journal \n " ;
242 push @$cmd, '--journal-dev' , $param ->{ dev
}, $journal_dev ;
244 push @$cmd, $param ->{ dev
};
250 return $rpcenv -> fork_worker ( 'cephcreateosd' , $devname, $authuser, $worker );
253 __PACKAGE__-
> register_method ({
254 name
=> 'destroyosd' ,
257 description
=> "Destroy OSD" ,
261 additionalProperties
=> 0 ,
263 node
=> get_standard_option
( 'pve-node' ),
265 description
=> 'OSD ID' ,
269 description
=> "If set, we remove partition table entries." ,
276 returns
=> { type
=> 'string' },
280 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
282 my $authuser = $rpcenv -> get_user ();
284 PVE
:: CephTools
:: check_ceph_inited
();
286 my $osdid = $param ->{ osdid
};
288 my $rados = PVE
:: RADOS-
> new ();
289 my $osdstat = & $get_osd_status ( $rados, $osdid );
291 die "osd is in use (in == 1) \n " if $osdstat ->{ in };
292 #&$run_ceph_cmd(['osd', 'out', $osdid]);
294 die "osd is still runnung (up == 1) \n " if $osdstat ->{ up
};
296 my $osdsection = "osd. $osdid " ;
301 # reopen with longer timeout
302 $rados = PVE
:: RADOS-
> new ( timeout
=> PVE
:: CephTools
:: get_config
( 'long_rados_timeout' ));
304 print "destroy OSD $osdsection\n " ;
306 eval { PVE
:: CephTools
:: ceph_service_cmd
( 'stop' , $osdsection ); };
309 print "Remove $osdsection from the CRUSH map \n " ;
310 $rados -> mon_command ({ prefix
=> "osd crush remove" , name
=> $osdsection, format
=> 'plain' });
312 print "Remove the $osdsection authentication key. \n " ;
313 $rados -> mon_command ({ prefix
=> "auth del" , entity
=> $osdsection, format
=> 'plain' });
315 print "Remove OSD $osdsection\n " ;
316 $rados -> mon_command ({ prefix
=> "osd rm" , ids
=> [ $osdsection ], format
=> 'plain' });
318 # try to unmount from standard mount point
319 my $mountpoint = "/var/lib/ceph/osd/ceph- $osdid " ;
321 my $remove_partition = sub {
322 my ( $disklist, $part ) = @_ ;
324 return if ! $part || (! - b
$part );
326 foreach my $real_dev ( keys %$disklist ) {
327 my $diskinfo = $disklist ->{ $real_dev };
328 next if ! $diskinfo ->{ gpt
};
329 if ( $part =~ m
|^ /dev/ ${ real_dev
}( \d
+)$|) {
331 print "remove partition $part (disk '/dev/${real_dev}', partnum $partnum ) \n " ;
332 eval { run_command
([ '/sbin/sgdisk' , '-d' , $partnum, "/dev/${real_dev}" ]); };
342 if ( $param ->{ cleanup
}) {
343 my $jpath = " $mountpoint/journal " ;
344 $journal_part = abs_path
( $jpath );
346 if ( my $fd = IO
:: File-
> new ( "/proc/mounts" , "r" )) {
347 while ( defined ( my $line = < $fd >)) {
348 my ( $dev, $path, $fstype ) = split ( /\s+/ , $line );
349 next if !( $dev && $path && $fstype );
350 next if $dev !~ m
|^ /dev/ |;
351 if ( $path eq $mountpoint ) {
352 $data_part = abs_path
( $dev );
360 print "Unmount OSD $osdsection from $mountpoint\n " ;
361 eval { run_command
([ 'umount' , $mountpoint ]); };
364 } elsif ( $param ->{ cleanup
}) {
365 my $disklist = PVE
:: CephTools
:: list_disks
();
366 & $remove_partition ( $disklist, $journal_part );
367 & $remove_partition ( $disklist, $data_part );
371 return $rpcenv -> fork_worker ( 'cephdestroyosd' , $osdsection, $authuser, $worker );
374 __PACKAGE__-
> register_method ({
376 path
=> '{osdid}/in' ,
378 description
=> "ceph osd in" ,
382 additionalProperties
=> 0 ,
384 node
=> get_standard_option
( 'pve-node' ),
386 description
=> 'OSD ID' ,
391 returns
=> { type
=> "null" },
395 PVE
:: CephTools
:: check_ceph_inited
();
397 my $osdid = $param ->{ osdid
};
399 my $rados = PVE
:: RADOS-
> new ();
401 my $osdstat = & $get_osd_status ( $rados, $osdid ); # osd exists?
403 my $osdsection = "osd. $osdid " ;
405 $rados -> mon_command ({ prefix
=> "osd in" , ids
=> [ $osdsection ], format
=> 'plain' });
410 __PACKAGE__-
> register_method ({
412 path
=> '{osdid}/out' ,
414 description
=> "ceph osd out" ,
418 additionalProperties
=> 0 ,
420 node
=> get_standard_option
( 'pve-node' ),
422 description
=> 'OSD ID' ,
427 returns
=> { type
=> "null" },
431 PVE
:: CephTools
:: check_ceph_inited
();
433 my $osdid = $param ->{ osdid
};
435 my $rados = PVE
:: RADOS-
> new ();
437 my $osdstat = & $get_osd_status ( $rados, $osdid ); # osd exists?
439 my $osdsection = "osd. $osdid " ;
441 $rados -> mon_command ({ prefix
=> "osd out" , ids
=> [ $osdsection ], format
=> 'plain' });
446 package PVE
:: API2
:: Ceph
;
452 use POSIX qw
( LONG_MAX
);
453 use Cwd
qw(abs_path) ;
459 use PVE
:: Tools
qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach) ;
460 use PVE
:: Exception
qw(raise raise_param_exc) ;
462 use PVE
:: Cluster
qw(cfs_lock_file cfs_read_file cfs_write_file) ;
463 use PVE
:: AccessControl
;
465 use PVE
:: RESTHandler
;
466 use PVE
:: RPCEnvironment
;
467 use PVE
:: JSONSchema
qw(get_standard_option) ;
472 use base
qw(PVE::RESTHandler) ;
474 use Data
:: Dumper
; # fixme: remove
476 my $pve_osd_default_journal_size = 1024 * 5 ;
478 __PACKAGE__-
> register_method ({
479 subclass
=> "PVE::API2::CephOSD" ,
483 __PACKAGE__-
> register_method ({
487 description
=> "Directory index." ,
488 permissions
=> { user
=> 'all' },
490 additionalProperties
=> 0 ,
492 node
=> get_standard_option
( 'pve-node' ),
501 links
=> [ { rel
=> 'child' , href
=> "{name}" } ],
513 { name
=> 'status' },
515 { name
=> 'config' },
523 __PACKAGE__-
> register_method ({
527 description
=> "List local disks." ,
531 additionalProperties
=> 0 ,
533 node
=> get_standard_option
( 'pve-node' ),
535 description
=> "Only list specific types of disks." ,
537 enum
=> [ 'unused' , 'journal_disks' ],
547 dev
=> { type
=> 'string' },
548 used
=> { type
=> 'string' , optional
=> 1 },
549 gpt
=> { type
=> 'boolean' },
550 size
=> { type
=> 'integer' },
551 osdid
=> { type
=> 'integer' },
552 vendor
=> { type
=> 'string' , optional
=> 1 },
553 model
=> { type
=> 'string' , optional
=> 1 },
554 serial
=> { type
=> 'string' , optional
=> 1 },
557 # links => [ { rel => 'child', href => "{}" } ],
562 PVE
:: CephTools
:: check_ceph_inited
();
564 my $disks = PVE
:: CephTools
:: list_disks
();
567 foreach my $dev ( keys %$disks ) {
568 my $d = $disks ->{ $dev };
569 if ( $param ->{ type
}) {
570 if ( $param ->{ type
} eq 'journal_disks' ) {
571 next if $d ->{ osdid
} >= 0 ;
573 } elsif ( $param ->{ type
} eq 'unused' ) {
576 die "internal error" ; # should not happen
580 $d ->{ dev
} = "/dev/ $dev " ;
587 __PACKAGE__-
> register_method ({
591 description
=> "Get Ceph configuration." ,
593 additionalProperties
=> 0 ,
595 node
=> get_standard_option
( 'pve-node' ),
598 returns
=> { type
=> 'string' },
602 PVE
:: CephTools
:: check_ceph_inited
();
604 my $path = PVE
:: CephTools
:: get_config
( 'pve_ceph_cfgpath' );
605 return PVE
:: Tools
:: file_get_contents
( $path );
609 __PACKAGE__-
> register_method ({
613 description
=> "Get Ceph monitor list." ,
617 additionalProperties
=> 0 ,
619 node
=> get_standard_option
( 'pve-node' ),
627 name
=> { type
=> 'string' },
628 addr
=> { type
=> 'string' },
631 links
=> [ { rel
=> 'child' , href
=> "{name}" } ],
636 PVE
:: CephTools
:: check_ceph_inited
();
640 my $cfg = PVE
:: CephTools
:: parse_ceph_config
();
643 foreach my $section ( keys %$cfg ) {
644 my $d = $cfg ->{ $section };
645 if ( $section =~ m/^mon\.(\S+)$/ ) {
647 if ( $d ->{ 'mon addr' } && $d ->{ 'host' }) {
648 $monhash ->{ $monid } = {
649 addr
=> $d ->{ 'mon addr' },
650 host
=> $d ->{ 'host' },
658 my $rados = PVE
:: RADOS-
> new ();
659 my $monstat = $rados -> mon_command ({ prefix
=> 'mon_status' });
660 my $mons = $monstat ->{ monmap
}->{ mons
};
661 foreach my $d ( @$mons ) {
662 next if ! defined ( $d ->{ name
});
663 $monhash ->{ $d ->{ name
}}->{ rank
} = $d ->{ rank
};
664 $monhash ->{ $d ->{ name
}}->{ addr
} = $d ->{ addr
};
665 if ( grep { $_ eq $d ->{ rank
} } @{ $monstat ->{ quorum
}}) {
666 $monhash ->{ $d ->{ name
}}->{ quorum
} = 1 ;
672 return PVE
:: RESTHandler
:: hash_to_array
( $monhash, 'name' );
675 __PACKAGE__-
> register_method ({
679 description
=> "Create initial ceph default configuration and setup symlinks." ,
683 additionalProperties
=> 0 ,
685 node
=> get_standard_option
( 'pve-node' ),
687 description
=> "Use specific network for all ceph related traffic" ,
688 type
=> 'string' , format
=> 'CIDR' ,
693 description
=> 'Number of replicas per object' ,
701 description
=> "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)" ,
710 returns
=> { type
=> 'null' },
714 PVE
:: CephTools
:: check_ceph_installed
();
716 # simply load old config if it already exists
717 my $cfg = PVE
:: CephTools
:: parse_ceph_config
();
719 if (! $cfg ->{ global
}) {
724 UUID
:: generate
( $uuid );
725 UUID
:: unparse
( $uuid, $fsid );
729 'auth supported' => 'cephx' ,
730 'auth cluster required' => 'cephx' ,
731 'auth service required' => 'cephx' ,
732 'auth client required' => 'cephx' ,
733 'filestore xattr use omap' => 'true' ,
734 'osd journal size' => $pve_osd_default_journal_size,
735 'osd pool default min size' => 1 ,
738 # this does not work for default pools
739 #'osd pool default pg num' => $pg_num,
740 #'osd pool default pgp num' => $pg_num,
743 $cfg ->{ global
}->{ keyring
} = '/etc/pve/priv/ $cluster . $name .keyring' ;
744 $cfg ->{ osd
}->{ keyring
} = '/var/lib/ceph/osd/ceph- $id/keyring ' ;
746 $cfg ->{ global
}->{ 'osd pool default size' } = $param ->{ size
} if $param ->{ size
};
748 if ( $param ->{ pg_bits
}) {
749 $cfg ->{ global
}->{ 'osd pg bits' } = $param ->{ pg_bits
};
750 $cfg ->{ global
}->{ 'osd pgp bits' } = $param ->{ pg_bits
};
753 if ( $param ->{ network
}) {
754 $cfg ->{ global
}->{ 'public network' } = $param ->{ network
};
755 $cfg ->{ global
}->{ 'cluster network' } = $param ->{ network
};
758 PVE
:: CephTools
:: write_ceph_config
( $cfg );
760 PVE
:: CephTools
:: setup_pve_symlinks
();
765 my $find_node_ip = sub {
768 my $net = Net
:: IP-
> new ( $cidr ) || die Net
:: IP
:: Error
() . " \n " ;
769 my $id = $net -> version == 6 ?
'address6' : 'address' ;
771 my $config = PVE
:: INotify
:: read_file
( 'interfaces' );
772 my $ifaces = $config ->{ ifaces
};
774 foreach my $iface ( keys %$ifaces ) {
775 my $d = $ifaces ->{ $iface };
777 my $a = Net
:: IP-
> new ( $d ->{ $id });
779 return $d ->{ $id } if $net -> overlaps ( $a );
782 die "unable to find local address within network ' $cidr ' \n " ;
785 __PACKAGE__-
> register_method ({
789 description
=> "Create Ceph Monitor" ,
793 additionalProperties
=> 0 ,
795 node
=> get_standard_option
( 'pve-node' ),
798 returns
=> { type
=> 'string' },
802 PVE
:: CephTools
:: check_ceph_inited
();
804 PVE
:: CephTools
:: setup_pve_symlinks
();
806 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
808 my $authuser = $rpcenv -> get_user ();
810 my $cfg = PVE
:: CephTools
:: parse_ceph_config
();
814 my $monaddrhash = {};
816 foreach my $section ( keys %$cfg ) {
817 next if $section eq 'global' ;
818 my $d = $cfg ->{ $section };
819 if ( $section =~ m/^mon\./ ) {
821 if ( $d ->{ 'mon addr' }) {
822 $monaddrhash ->{ $d ->{ 'mon addr' }} = $section ;
828 for ( my $i = 0 ; $i < 7 ; $i++ ) {
829 if (! $cfg ->{ "mon. $i " }) {
834 die "unable to find usable monitor id \n " if ! defined ( $monid );
836 my $monsection = "mon. $monid " ;
838 if ( my $pubnet = $cfg ->{ global
}->{ 'public network' }) {
839 $ip = & $find_node_ip ( $pubnet );
841 $ip = PVE
:: Cluster
:: remote_node_ip
( $param ->{ node
});
844 my $monaddr = " $ip :6789" ;
845 my $monname = $param ->{ node
};
847 die "monitor ' $monsection ' already exists \n " if $cfg ->{ $monsection };
848 die "monitor address ' $monaddr ' already in use by ' $monaddrhash ->{ $monaddr }' \n "
849 if $monaddrhash ->{ $monaddr };
854 my $pve_ckeyring_path = PVE
:: CephTools
:: get_config
( 'pve_ckeyring_path' );
856 if (! - f
$pve_ckeyring_path ) {
857 run_command
( "ceph-authtool $pve_ckeyring_path --create-keyring " .
858 "--gen-key -n client.admin" );
861 my $pve_mon_key_path = PVE
:: CephTools
:: get_config
( 'pve_mon_key_path' );
862 if (! - f
$pve_mon_key_path ) {
863 run_command
( "cp $pve_ckeyring_path $pve_mon_key_path .tmp" );
864 run_command
( "ceph-authtool $pve_mon_key_path .tmp -n client.admin --set-uid=0 " .
865 "--cap mds 'allow' " .
866 "--cap osd 'allow *' " .
867 "--cap mon 'allow *'" );
868 run_command
( "ceph-authtool $pve_mon_key_path .tmp --gen-key -n mon. --cap mon 'allow *'" );
869 run_command
( "mv $pve_mon_key_path .tmp $pve_mon_key_path " );
872 my $ccname = PVE
:: CephTools
:: get_config
( 'ccname' );
874 my $mondir = "/var/lib/ceph/mon/ $ccname - $monid " ;
875 - d
$mondir && die "monitor filesystem ' $mondir ' already exist \n " ;
877 my $monmap = "/tmp/monmap" ;
883 my $rados = PVE
:: RADOS-
> new ( timeout
=> PVE
:: CephTools
:: get_config
( 'long_rados_timeout' ));
884 my $mapdata = $rados -> mon_command ({ prefix
=> 'mon getmap' , format
=> 'plain' });
885 PVE
:: Tools
:: file_set_contents
( $monmap, $mapdata );
887 run_command
( "monmaptool --create --clobber --add $monid $monaddr --print $monmap " );
890 run_command
( "ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path " );
895 File
:: Path
:: remove_tree
( $mondir );
899 $cfg ->{ $monsection } = {
901 'mon addr' => $monaddr,
904 PVE
:: CephTools
:: write_ceph_config
( $cfg );
906 PVE
:: CephTools
:: ceph_service_cmd
( 'start' , $monsection );
909 return $rpcenv -> fork_worker ( 'cephcreatemon' , $monsection, $authuser, $worker );
912 __PACKAGE__-
> register_method ({
913 name
=> 'destroymon' ,
914 path
=> 'mon/{monid}' ,
916 description
=> "Destroy Ceph monitor." ,
920 additionalProperties
=> 0 ,
922 node
=> get_standard_option
( 'pve-node' ),
924 description
=> 'Monitor ID' ,
929 returns
=> { type
=> 'string' },
933 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
935 my $authuser = $rpcenv -> get_user ();
937 PVE
:: CephTools
:: check_ceph_inited
();
939 my $cfg = PVE
:: CephTools
:: parse_ceph_config
();
941 my $monid = $param ->{ monid
};
942 my $monsection = "mon. $monid " ;
944 my $rados = PVE
:: RADOS-
> new ();
945 my $monstat = $rados -> mon_command ({ prefix
=> 'mon_status' });
946 my $monlist = $monstat ->{ monmap
}->{ mons
};
948 die "no such monitor id ' $monid ' \n "
949 if ! defined ( $cfg ->{ $monsection });
951 my $ccname = PVE
:: CephTools
:: get_config
( 'ccname' );
953 my $mondir = "/var/lib/ceph/mon/ $ccname - $monid " ;
954 - d
$mondir || die "monitor filesystem ' $mondir ' does not exist on this node \n " ;
956 die "can't remove last monitor \n " if scalar ( @$monlist ) <= 1 ;
961 # reopen with longer timeout
962 $rados = PVE
:: RADOS-
> new ( timeout
=> PVE
:: CephTools
:: get_config
( 'long_rados_timeout' ));
964 $rados -> mon_command ({ prefix
=> "mon remove" , name
=> $monid, format
=> 'plain' });
966 eval { PVE
:: CephTools
:: ceph_service_cmd
( 'stop' , $monsection ); };
969 delete $cfg ->{ $monsection };
970 PVE
:: CephTools
:: write_ceph_config
( $cfg );
971 File
:: Path
:: remove_tree
( $mondir );
974 return $rpcenv -> fork_worker ( 'cephdestroymon' , $monsection, $authuser, $worker );
977 __PACKAGE__-
> register_method ({
981 description
=> "Stop ceph services." ,
985 additionalProperties
=> 0 ,
987 node
=> get_standard_option
( 'pve-node' ),
989 description
=> 'Ceph service name.' ,
992 pattern
=> '(mon|mds|osd)\.[A-Za-z0-9]{1,32}' ,
996 returns
=> { type
=> 'string' },
1000 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
1002 my $authuser = $rpcenv -> get_user ();
1004 PVE
:: CephTools
:: check_ceph_inited
();
1006 my $cfg = PVE
:: CephTools
:: parse_ceph_config
();
1007 scalar ( keys %$cfg ) || die "no configuration \n " ;
1013 if ( $param ->{ service
}) {
1014 push @$cmd, $param ->{ service
};
1017 PVE
:: CephTools
:: ceph_service_cmd
( @$cmd );
1020 return $rpcenv -> fork_worker ( 'srvstop' , $param ->{ service
} || 'ceph' ,
1021 $authuser, $worker );
1024 __PACKAGE__-
> register_method ({
1028 description
=> "Start ceph services." ,
1032 additionalProperties
=> 0 ,
1034 node
=> get_standard_option
( 'pve-node' ),
1036 description
=> 'Ceph service name.' ,
1039 pattern
=> '(mon|mds|osd)\.[A-Za-z0-9]{1,32}' ,
1043 returns
=> { type
=> 'string' },
1047 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
1049 my $authuser = $rpcenv -> get_user ();
1051 PVE
:: CephTools
:: check_ceph_inited
();
1053 my $cfg = PVE
:: CephTools
:: parse_ceph_config
();
1054 scalar ( keys %$cfg ) || die "no configuration \n " ;
1059 my $cmd = [ 'start' ];
1060 if ( $param ->{ service
}) {
1061 push @$cmd, $param ->{ service
};
1064 PVE
:: CephTools
:: ceph_service_cmd
( @$cmd );
1067 return $rpcenv -> fork_worker ( 'srvstart' , $param ->{ service
} || 'ceph' ,
1068 $authuser, $worker );
1071 __PACKAGE__-
> register_method ({
1075 description
=> "Get ceph status." ,
1079 additionalProperties
=> 0 ,
1081 node
=> get_standard_option
( 'pve-node' ),
1084 returns
=> { type
=> 'object' },
1088 PVE
:: CephTools
:: check_ceph_enabled
();
1090 my $rados = PVE
:: RADOS-
> new ();
1091 return $rados -> mon_command ({ prefix
=> 'status' });
1094 __PACKAGE__-
> register_method ({
1098 description
=> "List all pools." ,
1102 additionalProperties
=> 0 ,
1104 node
=> get_standard_option
( 'pve-node' ),
1112 pool
=> { type
=> 'integer' },
1113 pool_name
=> { type
=> 'string' },
1114 size
=> { type
=> 'integer' },
1117 links
=> [ { rel
=> 'child' , href
=> "{pool_name}" } ],
1122 PVE
:: CephTools
:: check_ceph_inited
();
1124 my $rados = PVE
:: RADOS-
> new ();
1127 my $res = $rados -> mon_command ({ prefix
=> 'df' });
1128 my $total = $res ->{ stats
}->{ total_avail_bytes
} || 0 ;
1130 foreach my $d (@{ $res ->{ pools
}}) {
1131 next if ! $d ->{ stats
};
1132 next if ! defined ( $d ->{ id
});
1133 $stats ->{ $d ->{ id
}} = $d ->{ stats
};
1136 $res = $rados -> mon_command ({ prefix
=> 'osd dump' });
1139 foreach my $e (@{ $res ->{ pools
}}) {
1141 foreach my $attr ( qw(pool pool_name size min_size pg_num crush_ruleset) ) {
1142 $d ->{ $attr } = $e ->{ $attr } if defined ( $e ->{ $attr });
1144 if ( my $s = $stats ->{ $d ->{ pool
}}) {
1145 $d ->{ bytes_used
} = $s ->{ bytes_used
};
1146 $d ->{ percent_used
} = ( $s ->{ bytes_used
} / $total )* 100
1147 if $s ->{ max_avail
} && $total ;
1156 __PACKAGE__-
> register_method ({
1157 name
=> 'createpool' ,
1160 description
=> "Create POOL" ,
1164 additionalProperties
=> 0 ,
1166 node
=> get_standard_option
( 'pve-node' ),
1168 description
=> "The name of the pool. It must be unique." ,
1172 description
=> 'Number of replicas per object' ,
1180 description
=> 'Minimum number of replicas per object' ,
1188 description
=> "Number of placement groups." ,
1196 description
=> "The ruleset to use for mapping object placement in the cluster." ,
1205 returns
=> { type
=> 'null' },
1209 PVE
:: CephTools
:: check_ceph_inited
();
1211 my $pve_ckeyring_path = PVE
:: CephTools
:: get_config
( 'pve_ckeyring_path' );
1213 die "not fully configured - missing ' $pve_ckeyring_path ' \n "
1214 if ! - f
$pve_ckeyring_path ;
1216 my $pg_num = $param ->{ pg_num
} || 64 ;
1217 my $size = $param ->{ size
} || 2 ;
1218 my $min_size = $param ->{ min_size
} || 1 ;
1219 my $ruleset = $param ->{ crush_ruleset
} || 0 ;
1220 my $rados = PVE
:: RADOS-
> new ();
1222 $rados -> mon_command ({
1223 prefix
=> "osd pool create" ,
1224 pool
=> $param ->{ name
},
1225 pg_num
=> int ( $pg_num ),
1226 # this does not work for unknown reason
1227 # properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"],
1231 $rados -> mon_command ({
1232 prefix
=> "osd pool set" ,
1233 pool
=> $param ->{ name
},
1239 $rados -> mon_command ({
1240 prefix
=> "osd pool set" ,
1241 pool
=> $param ->{ name
},
1247 if ( defined ( $param ->{ crush_ruleset
})) {
1248 $rados -> mon_command ({
1249 prefix
=> "osd pool set" ,
1250 pool
=> $param ->{ name
},
1251 var
=> 'crush_ruleset' ,
1252 val
=> $param ->{ crush_ruleset
},
1260 __PACKAGE__-
> register_method ({
1261 name
=> 'destroypool' ,
1262 path
=> 'pools/{name}' ,
1264 description
=> "Destroy pool" ,
1268 additionalProperties
=> 0 ,
1270 node
=> get_standard_option
( 'pve-node' ),
1272 description
=> "The name of the pool. It must be unique." ,
1277 returns
=> { type
=> 'null' },
1281 PVE
:: CephTools
:: check_ceph_inited
();
1283 my $rados = PVE
:: RADOS-
> new ();
1284 # fixme: '--yes-i-really-really-mean-it'
1285 $rados -> mon_command ({
1286 prefix
=> "osd pool delete" ,
1287 pool
=> $param ->{ name
},
1288 pool2
=> $param ->{ name
},
1289 sure
=> '--yes-i-really-really-mean-it' ,
1297 __PACKAGE__-
> register_method ({
1301 description
=> "Get OSD crush map" ,
1305 additionalProperties
=> 0 ,
1307 node
=> get_standard_option
( 'pve-node' ),
1310 returns
=> { type
=> 'string' },
1314 PVE
:: CephTools
:: check_ceph_inited
();
1316 # this produces JSON (difficult to read for the user)
1317 # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1321 my $mapfile = "/var/tmp/ceph-crush.map. $$ " ;
1322 my $mapdata = "/var/tmp/ceph-crush.txt. $$ " ;
1324 my $rados = PVE
:: RADOS-
> new ();
1327 my $bindata = $rados -> mon_command ({ prefix
=> 'osd getcrushmap' , format
=> 'plain' });
1328 PVE
:: Tools
:: file_set_contents
( $mapfile, $bindata );
1329 run_command
([ 'crushtool' , '-d' , $mapfile, '-o' , $mapdata ]);
1330 $txt = PVE
:: Tools
:: file_get_contents
( $mapdata );
1342 __PACKAGE__-
> register_method ({
1346 description
=> "Read ceph log" ,
1349 check
=> [ 'perm' , '/nodes/{node}' , [ 'Sys.Syslog' ]],
1353 additionalProperties
=> 0 ,
1355 node
=> get_standard_option
( 'pve-node' ),
1374 description
=> "Line number" ,
1378 description
=> "Line text" ,
1387 my $rpcenv = PVE
:: RPCEnvironment
:: get
();
1388 my $user = $rpcenv -> get_user ();
1389 my $node = $param ->{ node
};
1391 my $logfile = "/var/log/ceph/ceph.log" ;
1392 my ( $count, $lines ) = PVE
:: Tools
:: dump_logfile
( $logfile, $param ->{ start
}, $param ->{ limit
});
1394 $rpcenv -> set_result_attrib ( 'total' , $count );