openstack pike版使用ceph作後端儲存

節點分佈
10.1.1.1 controller
10.1.1.2 compute
10.1.1.3 middleware
10.1.1.4 network
10.1.1.5 compute2
10.1.1.6 compute3
10.1.1.7 cinder
##分散式儲存
後端儲存用的是ceph,mon_host = 10.1.1.2,10.1.1.5,10.1.1.6
##給cinder建立資料庫，服務以及endpoint
mysql -u root -p
create database cinder;
grant all privileges on cinder.* to 'cinder'@'localhost' identified by '123456';
grant all privileges on cinder.* to 'cinder'@'%' identified by '123456';


cat admin-openrc 
export OS_USER_DOMAIN_ID=default
export OS_PROJECT_DOMAIN_ID=default
export OS_USERNAME=admin
export OS_PROJECT_NAME=admin
export OS_PASSWORD=123456
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_AUTH_URL=
source admin-openrc
建立cinder使用者
openstack user create --domain default --password-prompt cinder
cinder 使用者加入admin組
openstack role add --project service --user cinder admin
建立service
openstack service create --name cinderv2 --description "OpentStack Block Storage" volumev2
openstack service create --name cinderv3 --description "OpentStack Block Storage" volumev3
建立API endpoint
openstack endpoint create --region RegionOne volumev2 public %\(tenant_id\)s
openstack endpoint create --region RegionOne volumev2 internal %\(tenant_id\)s
openstack endpoint create --region RegionOne volumev2 admin %\(tenant_id\)s


openstack endpoint create --region RegionOne volumev3 public %\(tenant_id\)s
openstack endpoint create --region RegionOne volumev3 internal %\(tenant_id\)s
openstack endpoint create --region RegionOne volumev3 admin %\(tenant_id\)s


建立ceph pool
在ceph 上執行如下命令
ceph osd pool create volumes 128
ceph osd pool create images 128
ceph osd pool vms 128


ceph 使用者授權
因為後端儲存用的是ceph,所以要給ceph客戶端授權,以便ceph使用者能訪問相應的ceph pool,使用到ceph的有glance,cinder,nova-compute
ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms,allow rwx pool=images'
ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=images'


ceph auth list
client.cinder
key: AQDQEWdaNU9YGBAAcEhKd6KQKHN9HeFIIS4+fw==
caps: [mon] allow r
caps: [osd] allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms,allow rwx pool=images
client.glance
key: AQD4EWdaTdZjJhAAuj8CvNY59evhiGtEa9wLzw==
caps: [mon] allow r
caps: [osd] allow class-read object_prefix rbd_children,allow rwx pool=images


在controller,cinder,compute節點建/etc/ceph目錄
將給controller,cinder,compute節點，建立授權檔案
ceph auth get-or-create client.glance |ssh controller sudo tee /etc/ceph/ceph.client.glance.keyring
ceph auth get-or-create client.cinder |ssh cinder sudo tee /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder |ssh compute sudo tee /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder |ssh compute2 sudo tee /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder |ssh compute3 sudo tee /etc/ceph/ceph.client.cinder.keyring
給ceph.client.glance.keyring 賦予glance許可權ceph.client.cinder.keyring賦予cinder許可權
chown glance.glance /etc/ceph/ceph.client.glance.keyring
chown cinder.cinder /etc/ceph/ceph.client.cinder.keyring
將ceph的配置檔案/etc/ceph/ceph.conf,複製一份到glance,cinder和compute節點的/etc/ceph目錄
安裝和配置元件
cinder 節點
yum install -y openstack-cinder python-ceph ceph-common python-rbd
在/etc/ceph目錄下要有如下檔案
[root@cinder ~]# ll /etc/ceph/
total 12
-rw-r--r-- 1 cinder cinder  64 1月  26 15:52 ceph.client.cinder.keyring
-rw-r--r-- 1 root   root   263 1月  26 15:53 ceph.conf


cp /etc/cinder/cinder.conf{,.bak}
>/etc/cinder/cinder.conf


cat /etc/cindr/cinder.conf

[DEFAULT]
auth_strategy = keystone
transport_url = rabbit://openstack:123456@middleware
log_dir = /var/log/cinder/api.log
enabled_backends = ceph


[database]
connection = mysql+pymysql://cinder:123456@middleware/cinder


[keystone_authtoken]
auth_uri =
auth_url =
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = cinder
password = 123456


[oslo_concurrency]
lock_path = /var/lib/cinder/tmp




[ceph]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
volume_backend_name = ceph
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
rbd_store_chunk_size = 4
rados_connect_timeout = -1
rbd_user = cinder
rbd_secret_uuid = f85def47-c1ac-46fe-a1d5-c0139c46d91a

重啟cinder服務
systemctl restart openstack-api.service
systemctl restart openstack-scheduler.service
systemctl restart openstack-volume.service




glance節點
安裝ceph客戶端
yum install -y python-ceph ceph-common python-rbd
在/etc/ceph目錄下，要有如下檔案
[root@controller ~]# ll /etc/ceph/
total 12
-rw-r--r-- 1 glance glance  64 1月  23 19:31 ceph.client.glance.keyring
-rw-r--r-- 1 root   root   416 1月  24 10:32 ceph.conf
有關ceph的配置
/etc/glance/glance.conf
[DEFAULT]
#enable image locaions and take advantage of copy-on-write cloning for images
show_image_direct_url = true
[glance_store]
stores = rbd
default_store = rbd
rbd_store_pool = images
rbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
rbd_store_chunk_size = 8
重啟glance服務
systemctl restart openstack-glance-api.service


compute節點
安裝ceph客戶端
yum install -y python-ceph ceph-common python-rbd
uuidgen生產uid,和/etc/cinder/cinder.conf中的uuid保持一致
f85def47-c1ac-46fe-a1d5-c0139c46d91a
建立secret檔案
cat secret.xml 
<secret ephemeral='no' private='no'>
  <uuid>f85def47-c1ac-46fe-a1d5-c0139c46d91a</uuid>
  <usage type='ceph'>
    <name>client.cinder secret</name>
  </usage>
</secret>
定義secret
sudo virsh secret-define --file secret.xml
sudo virsh secret-set-value --secret f85def47-c1ac-46fe-a1d5-c0139c46d91a --base64 $(cat ceph.client.cinder.keyring |awk '/key/{print $3}')


virsh secret-list
 UUID                                  Usage
--------------------------------------------------------------------------------
 f85def47-c1ac-46fe-a1d5-c0139c46d91a  ceph client.cinder secret


/etc/nova/nova.conf配置
[libvirt]
virt_type = qemu
cpu_mode = none
images_type = rbd
images_rbd_pool = vms
images_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
rbd_secret_uuid = f85def47-c1ac-46fe-a1d5-c0139c46d91a
disk_cachemodes="network=writeback"
live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
inject_password = false
inject_key = false
inject_partition = -2
在/etc/ceph目錄下要有如下檔案
[root@cinder ~]# ll /etc/ceph/
total 12
-rw-r--r-- 1 cinder cinder  64 1月  26 15:52 ceph.client.cinder.keyring
-rw-r--r-- 1 root   root   263 1月  26 15:53 ceph.conf
重啟nova-compute服務
systemctl restart openstack-nova-compute.service