ceph/src/spdk/scripts/ceph/start.sh

   1 #!/usr/bin/env bash
   2 # create mon
   3
   4 set -x
   5 set -e
   6
   7 script_dir=$(readlink -f $(dirname $0))
   8
   9 base_dir=/var/tmp/ceph
  10 mon_ip=$1
  11 mon_dir=${base_dir}/mon.a
  12 pid_dir=${base_dir}/pid
  13 ceph_conf=${base_dir}/ceph.conf
  14 mnt_dir=${base_dir}/mnt
  15 image=/var/tmp/ceph_raw.img
  16 dev=/dev/loop200
  17
  18 umount ${dev}p2 || true
  19 losetup -d $dev || true
  20
  21 # partition osd
  22 if [ -d $base_dir ]; then
  23         rm -rf $base_dir
  24 fi
  25 mkdir ${base_dir}
  26 cp ${script_dir}/ceph.conf $ceph_conf
  27
  28 if [ ! -e $image ]; then
  29         fallocate -l 4G $image
  30 fi
  31
  32 mknod ${dev} b 7 200 || true
  33 losetup ${dev} ${image} || true
  34
  35 PARTED="parted -s"
  36 SGDISK="sgdisk"
  37
  38 echo "Partitioning ${dev}"
  39 ${PARTED} ${dev} mktable gpt
  40 sleep 2
  41
  42 ${PARTED} ${dev} mkpart primary 0% 2GiB
  43 ${PARTED} ${dev} mkpart primary 2GiB 100%
  44
  45 partno=0
  46 echo "Setting name on ${dev}"
  47 ${SGDISK} -c 1:osd-device-${partno}-journal ${dev}
  48 ${SGDISK} -c 2:osd-device-${partno}-data ${dev}
  49 kpartx ${dev}
  50
  51 # later versions of ceph-12 have a lot of changes, to compatible with the new
  52 # version of ceph-deploy.
  53 ceph_version=$(ceph -v | awk '{print $3}')
  54 ceph_maj=${ceph_version%%.*}
  55 if [ $ceph_maj -gt 12 ]; then
  56         update_config=true
  57         rm -f /var/log/ceph/ceph-mon.a.log || true
  58         set_min_mon_release="--set-min-mon-release 14"
  59         ceph_osd_extra_config="--check-needs-journal --no-mon-config"
  60 else
  61         update_config=false
  62         set_min_mon_release=""
  63         ceph_osd_extra_config=""
  64 fi
  65
  66 # prep osds
  67
  68 mnt_pt=${mnt_dir}/osd-device-0-data
  69 mkdir -p ${mnt_pt}
  70 mkfs.xfs -f /dev/disk/by-partlabel/osd-device-0-data
  71 mount /dev/disk/by-partlabel/osd-device-0-data ${mnt_pt}
  72 cat << EOL >> $ceph_conf
  73 osd data = ${mnt_pt}
  74 osd journal = /dev/disk/by-partlabel/osd-device-0-journal
  75
  76 # add mon address
  77 [mon.a]
  78 mon addr = ${mon_ip}:12046
  79 EOL
  80
  81 # create mon
  82 rm -rf "${mon_dir:?}/"*
  83 mkdir -p ${mon_dir}
  84 mkdir -p ${pid_dir}
  85 rm -f /etc/ceph/ceph.client.admin.keyring
  86
  87 ceph-authtool --create-keyring --gen-key --name=mon. ${base_dir}/keyring --cap mon 'allow *'
  88 ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' ${base_dir}/keyring
  89
  90 monmaptool --create --clobber --add a ${mon_ip}:12046 --print ${base_dir}/monmap $set_min_mon_release
  91
  92 sh -c "ulimit -c unlimited && exec ceph-mon --mkfs -c ${ceph_conf} -i a --monmap=${base_dir}/monmap --keyring=${base_dir}/keyring --mon-data=${mon_dir}"
  93
  94 if [ $update_config = true ]; then
  95         sed -i 's/mon addr = /mon addr = v2:/g' $ceph_conf
  96 fi
  97
  98 cp ${base_dir}/keyring ${mon_dir}/keyring
  99
 100 cp $ceph_conf /etc/ceph/ceph.conf
 101
 102 cp ${base_dir}/keyring /etc/ceph/keyring
 103 cp ${base_dir}/keyring /etc/ceph/ceph.client.admin.keyring
 104 chmod a+r /etc/ceph/ceph.client.admin.keyring
 105
 106 ceph-run sh -c "ulimit -n 16384 && ulimit -c unlimited && exec ceph-mon -c ${ceph_conf} -i a --keyring=${base_dir}/keyring --pid-file=${base_dir}/pid/root@$(hostname).pid --mon-data=${mon_dir}" || true
 107
 108 # after ceph-mon creation, ceph -s should work.
 109 if [ $update_config = true ]; then
 110         # start to get whole log.
 111         ceph-conf --name mon.a --show-config-value log_file
 112
 113         # add fsid to ceph config file.
 114         fsid=$(ceph -s | grep id | awk '{print $2}')
 115         sed -i 's/perf = true/perf = true\n\tfsid = '$fsid' \n/g' $ceph_conf
 116
 117         # unify the filesystem with the old versions.
 118         sed -i 's/perf = true/perf = true\n\tosd objectstore = filestore\n/g' $ceph_conf
 119         cat ${ceph_conf}
 120 fi
 121
 122 # create osd
 123
 124 i=0
 125
 126 mkdir -p ${mnt_dir}
 127
 128 uuid=$(uuidgen)
 129 ceph -c ${ceph_conf} osd create ${uuid} $i
 130 ceph-osd -c ${ceph_conf} -i $i --mkfs --mkkey --osd-uuid ${uuid} ${ceph_osd_extra_config}
 131 ceph -c ${ceph_conf} osd crush add osd.${i} 1.0 host=$(hostname) root=default
 132 ceph -c ${ceph_conf} -i ${mnt_dir}/osd-device-${i}-data/keyring auth add osd.${i} osd "allow *" mon "allow profile osd" mgr "allow *"
 133
 134 # start osd
 135 pkill -9 ceph-osd || true
 136 sleep 2
 137
 138 mkdir -p ${pid_dir}
 139 env -i TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 ceph-osd -c ${ceph_conf} -i 0 --pid-file=${pid_dir}/ceph-osd.0.pid