ceph搭建

实验环境介绍

系统版本：buntu 18.04.5 LTS

内核参数：4.15.0-112-generic

ceph版本： pacific/16.2.5

主机分配：
#部署服务器ceph-deploy
192.168.192.171
#1个ceph-mgr 管理服务器后续拓展2-4台
192.168.192.172
#1台ceph 集群Mon 监视服务器，每台服务器可以和ceph 集群的cluster 网络通信。后续拓展为3台
192.168.192.172
#四台服务器作为ceph 集群OSD 存储服务器，每台服务器支持两个网络，public 网络针对客户端访问，cluster 网络用于集群管理及数据同步，每台三块或以上的磁盘
192.168.192.173-175/192.168.227.173-175 ceph-node01-03
#磁盘划分
/dev/sdb /dev/sdc /dev/sdd #50G

1，所有节点配置ubuntu源和ceph源

cat >/etc/apt/sources.list<<EOF
# 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic main
EOF

wget -q -O- 'http://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc' | sudo apt-key add -
#更新仓库源
apt update

2，所有节点安装常用软件

apt install iproute2  ntpdate  tcpdump telnet traceroute nfs-kernel-server nfs-common  lrzsz tree  openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute  gcc openssh-server lrzsz tree  openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute iotop unzip zip openjdk-8-jdk -y

3，所有节点内核配置

cat >/etc/sysctl.conf <<EOF
# Controls source route verification 
net.ipv4.conf.default.rp_filter = 1 
net.ipv4.ip_nonlocal_bind = 1 
net.ipv4.ip_forward = 1 
  
# Do not accept source routing 
net.ipv4.conf.default.accept_source_route = 0 
  
# Controls the System Request debugging functionality of the kernel 
kernel.sysrq = 0 
  
# Controls whether core dumps will append the PID to the core filename. 
# Useful for debugging multi-threaded 
applications. kernel.core_uses_pid = 1 
  
# Controls the use of TCP syncookies 
net.ipv4.tcp_syncookies = 1 
  
# Disable netfilter on bridges. 
net.bridge.bridge-nf-call-ip6tables = 0 
net.bridge.bridge-nf-call-iptables = 0 
net.bridge.bridge-nf-call-arptables = 0 
  
# Controls the default maxmimum size of a mesage queue 
kernel.msgmnb = 65536 
  
# # Controls the maximum size of a message, in bytes 
kernel.msgmax = 65536 
  
# Controls the maximum shared segment size, in bytes 
kernel.shmmax = 68719476736 
  
# # Controls the maximum number of shared memory segments, in pages 
kernel.shmall = 4294967296 
  
# TCP kernel paramater 
net.ipv4.tcp_mem = 786432 1048576 1572864 
net.ipv4.tcp_rmem = 4096        87380   4194304 
net.ipv4.tcp_wmem = 4096        16384   4194304 n
et.ipv4.tcp_window_scaling = 1 
net.ipv4.tcp_sack = 1 
  
# socket buffer 
net.core.wmem_default = 8388608 
net.core.rmem_default = 8388608 
net.core.rmem_max = 16777216 
net.core.wmem_max = 16777216 
net.core.netdev_max_backlog = 262144 
net.core.somaxconn = 20480 
net.core.optmem_max = 81920 
  
# TCP conn 
net.ipv4.tcp_max_syn_backlog = 262144 
net.ipv4.tcp_syn_retries = 3 
net.ipv4.tcp_retries1 = 3 
net.ipv4.tcp_retries2 = 15 
  
# tcp conn reuse 
net.ipv4.tcp_timestamps = 0 
net.ipv4.tcp_tw_reuse = 0 
net.ipv4.tcp_tw_recycle = 0 
net.ipv4.tcp_fin_timeout = 1 
  
  
net.ipv4.tcp_max_tw_buckets = 20000 
net.ipv4.tcp_max_orphans = 3276800 
net.ipv4.tcp_synack_retries = 1 
net.ipv4.tcp_syncookies = 1 
  
# keepalive conn 
net.ipv4.tcp_keepalive_time = 300 
net.ipv4.tcp_keepalive_intvl = 30 
net.ipv4.tcp_keepalive_probes = 3 
net.ipv4.ip_local_port_range = 10001    65000 
  
# swap 
vm.overcommit_memory = 0 
vm.swappiness = 10 
  
#net.ipv4.conf.eth1.rp_filter = 0 
#net.ipv4.conf.lo.arp_ignore = 1 
#net.ipv4.conf.lo.arp_announce = 2 
#net.ipv4.conf.all.arp_ignore = 1 
#net.ipv4.conf.all.arp_announce = 2 
EOF

4，所有节点文件权限配置

cat > /etc/security/limits.conf <<EOF
root                soft    core            unlimited 
root                hard    core            unlimited 
root                soft    nproc           1000000 
root                hard    nproc           1000000 
root                soft    nofile          1000000 
root                hard    nofile          1000000 
root                soft    memlock         32000 
root                hard    memlock         32000 
root                soft    msgqueue        8192000 
root                hard    msgqueue        8192000 
  
 
*                soft    core            unlimited 
*                hard    core            unlimited 
*                soft    nproc           1000000 
*                hard    nproc           1000000 
*                soft    nofile          1000000 
*                hard    nofile          1000000 
*                soft    memlock         32000 
*                hard    memlock         32000 
*                soft    msgqueue        8192000 
*                hard    msgqueue        8192000 
EOF

5，所有节点时间同步

#安装cron并启动
apt install cron -y
systemctl status cron.service
#同步时间
/usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w
#每5分钟同步一次时间
 echo "*/5 * * * * /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w" >> /var/spool/cron/crontabs/root

6，所有节点/etc/hosts配置

root@ceph-node01:~# cat /etc/hosts
127.0.0.1    localhost
127.0.1.1    ubuntu.example.local    ubuntu

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
192.168.192.171 ceph-deploy
192.168.192.172 ceph-mon01-mgr01
192.168.192.173 ceph-node01
192.168.192.174 ceph-node02
192.168.192.175 ceph-node03

7.所有节点安装python2

做ceph初始化时，需要python2.7

apt install python2.7 -y
ln -sv /usr/bin/python2.7 /usr/bin/python2

部署ceph

推荐使用指定的普通用户部署和运行ceph 集群，普通用户只要能以非交互方式执行命令执行一些特权命令即可，新版的ceph-deploy 可以指定包含root 的在内只要可以执行命令的用户，不过仍然推荐使用普通用户，比如ceph、cephuser、cephadmin 这样的用户去管理ceph 集群。

#因为前面安装的ceph-common会更改ceph用户的家目录，建议使用其它用户来部署，如cephadmin用户
#所有节点添加cephadmin用户
groupadd -r -g 2021 cephadmin && useradd -r -m -s /bin/bash -u 2021 -g 2021 cephadmin && echo cephadmin:123456 | chpasswd
#允许ceph 用户以 执行特权命令
echo "cephadmin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers

1 配置免密登录

1 安装sshpass
cephadmin@ceph-deploy:/tmp$ sudo apt install sshpass
 
2 ceph-deploy节点使用ceph用户分发密钥脚本
cat >>/tmp/ssh_fenfa.sh<<EOF
#!/bin/bash
#目标主机列表
IP="
192.168.192.x
"
for node in ${IP};do
 sshpass -p 123456 ssh-copy-id cephadmin@${node}  -o StrictHostKeyChecking=no &> /dev/null
  if [ $? -eq 0 ];then
    echo "${node}----> 密钥分发success完成"
  else
    echo "${node}----> 密钥分发false失败"
  fi
done
EOF

2 在ceph-deploy节点部署ceph-deploy工具包

cephadmin@ceph-deploy:~# sudo apt-cache madison ceph-deploy
ceph-deploy |      2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main amd64 Packages
ceph-deploy |      2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main i386 Packages
ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe amd64 Packages
ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe i386 Packages
 
cephadmin@ceph-deploy:~# sudo apt install ceph-deploy

3 初始化mon节点

在管理节点初始化mon节点

cephadmin@ceph-deploy:~$ mkdir ceph-cluster #保存当前集群的初始化配置信息<br>cephadmin@ceph-deploy:~$ cd ceph-cluster/
cephadmin@ceph-deploy:~/ceph-cluster$

root@ceph-mon01-mgr01:~# apt install ceph-mon -y

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy new --cluster-network 192.168.227.0/24 --public-network 192.168.192.0/24 ceph-mon01-mgr01
cephadmin@ceph-deploy:~/ceph-cluster$ ls
ceph.conf  ceph-deploy-ceph.log  ceph.mon.keyring
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf 
[global]
fsid = d2cca32b-57dc-409f-9605-b19a373ce759
public_network = 192.168.192.0/24
cluster_network = 192.168.227.0/24
mon_initial_members = ceph-mon01-mgr01
mon_host = 192.168.192.172
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mon create-initial

root@ceph-mon01-mgr01:~# ps -ef|grep ceph-mon
ceph      28898      1  0 14:47 ?        00:00:02 /usr/bin/ceph-mon -f --cluster ceph --id ceph-mon01-mgr01 --setuser ceph --setgroup ceph

root@ceph-mon01-mgr01:~# id ceph
uid=64045(ceph) gid=64045(ceph) groups=64045(ceph)

在当前目录下就会产生五个keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ll *keyring
-rw------- 1 root root 113 Nov  6 14:48 ceph.bootstrap-mds.keyring
-rw------- 1 root root 113 Nov  6 14:48 ceph.bootstrap-mgr.keyring
-rw------- 1 root root 113 Nov  6 14:48 ceph.bootstrap-osd.keyring
-rw------- 1 root root 113 Nov  6 14:48 ceph.bootstrap-rgw.keyring
-rw------- 1 root root 151 Nov  6 14:47 ceph.client.admin.keyring

4,初始化node节点

root@ceph-deploy:~#sudo ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node01 ceph-node02 ceph-node03

[ceph-node03][DEBUG ] ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)

5,分发admin密钥到node节点并授权

ceph-deploy ceph-node01-03 4台执行 #安装ceph 的公共组件
cephadmin@ceph-nodeX:~$ sudo apt autoremove
cephadmin@ceph-nodeX:~$ sudo apt install ceph-common -y

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy admin ceph-node01 ceph-node02 ceph-node03

cephadmin@ceph-node0X:~$ ls  /etc/ceph/
ceph.client.admin.keyring  ceph.conf  rbdmap  tmpL4Cqwh

root@ceph-node0X:~# sudo setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring

6，部署ceph-mgr节点

root@ceph-mon01-mgr01:~#sudo apt install ceph-mgr -y

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mgr create ceph-mon01-mgr01

cephadmin@ceph-mon01-mgr01:~$ ps -ef|grep ceph-mgr
ceph      36889      1 17 21:53 ?        00:00:04 /usr/bin/ceph-mgr -f --cluster ceph --id ceph-mon01-mgr01 --setuser ceph --setgroup ceph

7,ceph-deploy管理ceph集群环境设置禁用非安全模式通信

cephadmin@ceph-deploy:~/ceph-cluster$ sudo apt install  ceph-common -y

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy admin ceph-deploy
cephadmin@ceph-deploy:~/ceph-cluster$ sudo setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring
cephadmin@ceph-deploy:~/ceph-cluster$ ls /etc/ceph/
ceph.client.admin.keyring  ceph.conf  rbdmap  tmp181KRo

cephadmin@ceph-node02:~$ ceph config set mon auth_allow_insecure_global_id_reclaim false
cephadmin@ceph-node02:~$ ceph -s
  cluster:
    id:     d2cca32b-57dc-409f-9605-b19a373ce759
    health: HEALTH_WARN
            mon is allowing insecure global_id reclaim
            OSD count 0 < osd_pool_default_size 3
 
  services:
    mon: 1 daemons, quorum ceph-mon01-mgr01 (age 7h)
    mgr: ceph-mon01-mgr01(active, since 3m)
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:

cephadmin@ceph-node02:~$ ceph versions
{
    "mon": {
        "ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)": 1
    },
    "mgr": {
        "ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)": 1
    },
    "osd": {},
    "mds": {},
    "overall": {
        "ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)": 2
    }
}

8，准备osd节点

#前面执行了node节点的初始化，如下步骤，后面的操作可以不执行
ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node01 ceph-node02 ceph-node03 ceph-node04

#擦除磁盘之前通过deploy 节点对node 节点执行安装ceph 基本运行环境。
#在ceph-deploy上操作

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy install --release pacific ceph-node01-03

9，列出node节点磁盘

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy disk list ceph-node01
[ceph-node01][INFO  ] Disk /dev/sda: 120 GiB, 128849018880 bytes, 251658240 sectors
[ceph-node01][INFO  ] Disk /dev/sdb: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO  ] Disk /dev/sdc: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO  ] Disk /dev/sdd: 50 GiB, 53687091200 bytes, 104857600 sectors

10，使用ceph-deploy disk zap 擦除ceph node的ceph数据磁盘

在ceph-deploy上执行

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node01  /dev/sdb
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node01  /dev/sdc
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node01  /dev/sdd
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node02  /dev/sdb
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node02  /dev/sdc
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node02  /dev/sdd
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node03  /dev/sdb
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node02  /dev/sdc
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy  disk zap ceph-node02  /dev/sdd

11,添加OSD 在ceph-deploy上执行

cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node01 --data /dev/sdb
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node01 --data /dev/sdc
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node01 --data /dev/sdd
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node02 --data /dev/sdb
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node02 --data /dev/sdc
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node02 --data /dev/sdd
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node03 --data /dev/sdb
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node03 --data /dev/sdc
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy osd create ceph-node03 --data /dev/sdd

12.验证

cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     d2cca32b-57dc-409f-9605-b19a373ce759
    health: HEALTH_OK
 
  services:
    mon: 1 daemons, quorum ceph-mon01-mgr01 (age 7h)
    mgr: ceph-mon01-mgr01(active, since 18m)
    osd: 9 osds: 9 up (since 14s), 9 in (since 23s)
 
  data:
    pools:   1 pools, 128 pgs
    objects: 0 objects, 0 B
    usage:   65 MiB used, 450 GiB / 450 GiB avail
    pgs:     128 active+clean

cephadmin@ceph-node01:~$ ps -ef|grep osd
ceph       13952       1  0 22:08 ?        00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
ceph       15708       1  1 22:09 ?        00:00:02 /usr/bin/ceph-osd -f --cluster ceph --id 1 --setuser ceph --setgroup ceph
ceph       17443       1  1 22:09 ?        00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 2 --setuser ceph --setgroup ceph
cephadmin@ceph-node02:~$ ps -ef|grep osd
ceph       13853       1  1 22:10 ?        00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 3 --setuser ceph --setgroup ceph
ceph       15608       1  1 22:10 ?        00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 4 --setuser ceph --setgroup ceph
ceph       17354       1  1 22:10 ?        00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 5 --setuser ceph --setgroup ceph
root@ceph-node03:~# ps -ef|grep osd
ceph       12379       1  1 22:11 ?        00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 6 --setuser ceph --setgroup ceph
ceph       14126       1  0 22:11 ?        00:00:00 /usr/bin/ceph-osd -f --cluster ceph --id 7 --setuser ceph --setgroup ceph
ceph       15860       1  0 22:11 ?        00:00:00 /usr/bin/ceph-osd -f --cluster ceph --id 8 --setuser ceph --setgroup ceph

默认就已经为自启动, node 节点添加完成后,开源测试node 服务器重启后，OSD 是否会自动启动
cephadmin@ceph-node01:~$ sudo systemctl enable ceph-osd@0 ceph-osd@01 ceph-osd@02 ceph-osd@03 ceph-osd@04 ceph-osd@05 ceph-osd@06 ceph-osd@07 ceph-osd@08

13 ceph-deploy命令

$ ceph-deploy --help
new：开始部署一个新的ceph 存储集群，并生成CLUSTER.conf 集群配置文件和keyring
认证文件。
install: 在远程主机上安装ceph 相关的软件包, 可以通过--release 指定安装的版本。
rgw：管理RGW 守护程序(RADOSGW,对象存储网关)。
mgr：管理MGR 守护程序(ceph-mgr,Ceph Manager DaemonCeph 管理器守护程序)。
mds：管理MDS 守护程序(Ceph Metadata Server，ceph 源数据服务器)。
mon：管理MON 守护程序(ceph-mon,ceph 监视器)。
gatherkeys：从指定获取提供新节点的验证keys，这些keys 会在添加新的MON/OSD/MD加入的时候使用。
disk：管理远程主机磁盘。
osd：在远程主机准备数据磁盘，即将指定远程主机的指定磁盘添加到ceph 集群作为osd
使用。
repo： 远程主机仓库管理。
admin：推送ceph 集群配置文件和client.admin 认证文件到远程主机。
config：将ceph.conf 配置文件推送到远程主机或从远程主机拷贝。
uninstall：从远端主机删除安装包。
purgedata：从/var/lib/ceph 删除ceph 数据,会删除/etc/ceph 下的内容。
purge: 删除远端主机的安装包和所有数据。
forgetkeys：从本地主机删除所有的验证keyring, 包括client.admin, monitor, bootstrap 等
认证文件。
pkg： 管理远端主机的安装包。
calamari：安装并配置一个calamari web 节点，calamari 是一个web 监控平台。

14 关闭重启

cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd set noout

关闭流量

ceph osd set noout
ceph osd set norecover
ceph osd set norebalance
ceph osd set nobackfill
ceph osd set nodown
ceph osd set pause

先关闭node节点后关闭mon节点

先开启mon节点后开启node节点取消标签

ceph搭建

实验环境介绍

推荐阅读