Ceph原始碼編譯與打包

原始碼打包docker 7u機器：

lvjian100081200005.et15sqa

git clone --recursive

#checkout 出某一個版本

git clone –b v10.2.2

cd ceph

git status

git submodule update --init --recursive

git checkout master

#ceph相關，獲取依賴的方法：

ceph.spec | grep -i require | sort -nk1 | uniq

sudo yum install libudev-devel libblkid-devel libuuid-devel libaio-devel fuse-devel \

xfsprogs-devel expat-devel fcgi-devel libatomic_ops-devel python-libs python-babel python-sphinx gperftools-devel \

boost-devel libunwind-devel libcurl-devel nss-devel Cython keyutils-libs-devel openssl-devel openldap-devel glibc-devel glibc-static -b current

#leveldb和rocksdb相關

sudo yum install snappy-devel bzip2-devel gflags-devel leveldb-devel

//

// main

//

namespace gogle = gflags;

int main(int argc, char** argv) {

google::SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +

" [OPTIONS]...");

google::ParseCommandLineFlags(&argc, &argv, false);

sudo yum install devtoolset-2-gcc devtoolset-2-gcc-c++ -b current

sudo yum install devtoolset-2-binutils -b current

source /opt/rh/devtoolset-2/enable

export CC=/opt/rh/devtoolset-2/root/usr/bin/gcc

export CPP=/opt/rh/devtoolset-2/root/usr/bin/cpp

export CXX=/opt/rh/devtoolset-2/root/usr/bin/c++

make –j2

./do_cmake.sh

cd build

cmake -DCMAKE_INSTALL_PREFIX=/opt/ceph -DCMAKE_C_FLAGS="-O0 -g3” ..

make –j24

./configure \

--prefix=/opt/ceph --sbindir=/opt/ceph/sbin --localstatedir=/opt/ceph/var --sysconfdir=/opt/ceph/etc \

--with-debug $with_profiler --with-nss --without-cryptopp --with-radosgw \

./configure \

--prefix=/opt/ceph --sbindir=/opt/ceph/sbin --localstatedir=/opt/ceph/var --sysconfdir=/opt/ceph/etc \

--with-debug $with_profiler --with-nss --without-cryptopp --with-radosgw \

./autogen.sh

./configure或者./do_autogen.sh -J -L -d 1

make –j24

make install DESTDIR=/opt/ceph

#build RPM package，build RPM時遇到的依賴

sudo yum install valgrind-devel sharutils libxml2-devel libxslt-devel -b current

sudo yum install python-nose python-requests python-virtualenv python-devel python-setuptools Cython python-pip binutils-devel -b current

sudo yum install redhat-lsb-core yum-utils –b current

sudo yum install java-devel junit hdparm -b current

setuptools

#以下幾個包在7u上沒有，需要手動下載

sudo rpm –ivh jq-1.5-4.fc25.x86_64.rpm

sudo rpm –ivh yasm-1.3.0-3.fc24.x86_64.rpm

sudo rpm –ivh xmlstarlet-1.6.1-6.fc24.x86_64.rpm

sudo rpm –ivh oniguruma-6.0.0-1.fc25.x86_64.rpm

sudo rpm –ivh fcgi-devel-2.4.0-15.3.x86_64.rpm

sudo rpm –ivh lib64fcgi0-2.4.0-15.mga4.x86_64.rpm

sudo rpm –ivh fcgi-2.4.0-15.3.x86_64.rpm

sudo rpm –ivh jemalloc-4.2.1-1.fc26.x86_64.rpm

sudo rpm –ivh jemalloc-devel-4.2.1-1.fc26.x86_64.rpm

./make_dist.sh

關閉debug package

echo “%debug_package %{nil}” >>.rpmmacros

修改ceph.spec檔案:

%global _prefix /opt/ceph

%global _libexecdir %{_exec_prefix}/lib

%global __libtool_path ^%{buildroot}%{_libdir}/.*.la$

%global _sysconfdir %{_prefix}%{_sysconfdir}

%global _localstatedir %{_prefix}%{_localstatedir}

%debug_package %{nil}

#mv %{buildroot}/sbin/mount.ceph %{buildroot}/usr/sbin/mount.ceph

#mv %{buildroot}/sbin/mount.fuse.ceph %{buildroot}/usr/sbin/mount.fuse.ceph

mv %{buildroot}/sbin/mount.ceph %{buildroot}%{_prefix}/sbin/mount.ceph

mv %{buildroot}/sbin/mount.fuse.ceph %{buildroot}%{_prefix}/sbin/mount.fuse.ceph

mv %{buildroot}/usr/sbin/ceph-disk %{buildroot}%{_prefix}/sbin/ceph-disk

mv %{buildroot}/usr/bin/ceph-detect-init %{buildroot}%{_prefix}/bin/ceph-detect-init

rpmbuild –ba ceph.spec

t-abs ceph.spec

sudo rpm -ivh librados2-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh libradosstriper1-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh python-rados-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-radosgw-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-base-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-common-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh librbd1-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh rbd-fuse-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh rbd-mirror-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh rbd-nbd-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh python-rbd-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh librgw2-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh libcephfs1-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-fuse-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh python-cephfs-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-libs-compat-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh python-ceph-compat-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-osd-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-mds-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-mon-10.2.2-0.rhel7.x86_64.rpm

#devel相關的包，可以不安裝

sudo rpm -ivh librados2-devel-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh librgw2-devel-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh libradosstriper1-devel-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh librbd1-devel-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh libcephfs1-devel-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-devel-compat-10.2.2-0.rhel7.x86_64.rpm

#測試，debug以及java相關的包，可以不安裝

sudo rpm -ivh ceph-test-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh ceph-debuginfo-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh libcephfs_jni1-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh libcephfs_jni1-devel-10.2.2-0.rhel7.x86_64.rpm

sudo rpm -ivh cephfs-java-10.2.2-0.rhel7.x86_64.rpm

#起動vstart

./vstart.sh –d –n –x –l

./bin/ceph –s

filestore_max_xattr_value_size_other = 4086

osd max object name len = 1024

osd max object namespace len = 256

init-ceph start osd.*

$rados -p rbd bench 30 write

Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 30 seconds or 0 objects

Object prefix: benchmark_data_lvjian100081200005.et15sqa_44248

sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)

0 0 0 0 0 0 - 0

1 16 38 22 87.9956 88 0.233292 0.179826

2 16 38 22 43.9967 0 - 0.179826

3 16 38 22 29.3311 0 - 0.179826

4 16 38 22 21.9984 0 - 0.179826

5 16 38 22 17.5987 0 - 0.179826

6 16 82 66 43.9966 35.2 0.136914 1.34638

7 16 82 66 37.7113 0 - 1.34638

8 16 82 66 32.9974 0 - 1.34638

9 16 82 66 29.331 0 - 1.34638

10 16 82 66 26.3979 0 - 1.34638

11 16 113 97 35.2699 24.8 0.16824 1.77199

12 16 113 97 32.3307 0 - 1.77199

13 16 113 97 29.8437 0 - 1.77199

14 16 113 97 27.712 0 - 1.77199

15 16 113 97 25.8646 0 - 1.77199

16 16 137 121 30.2476 19.2 0.0883345 2.09566

17 16 138 122 28.7035 4 0.164464 2.07983

18 16 138 122 27.1089 0 - 2.07983

19 16 138 122 25.6821 0 - 2.07983

2016-10-18 14:04:01.045958 min lat: 0.0330393 max lat: 5.18978 avg lat: 2.07983

sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)

20 16 138 122 24.398 0 - 2.07983

21 16 149 133 25.3312 11 5.10087 2.3338

22 16 172 156 28.3612 92 0.181053 2.16421

23 16 172 156 27.1281 0 - 2.16421

24 16 172 156 25.9978 0 - 2.16421

25 16 172 156 24.9579 0 - 2.16421

26 16 172 156 23.9979 0 - 2.16421

27 16 203 187 27.7013 24.8 0.133892 2.24739

28 16 203 187 26.712 0 - 2.24739

29 16 203 187 25.7909 0 - 2.24739

30 16 203 187 24.9312 0 - 2.24739

31 16 203 187 24.127 0 - 2.24739

Total time run: 31.372199

Total writes made: 204

Write size: 4194304

Object size: 4194304

Bandwidth (MB/sec): 26.0103

Stddev Bandwidth: 23.3087

Max bandwidth (MB/sec): 92

Min bandwidth (MB/sec): 0

Average IOPS: 6

Stddev IOPS: 5

Max IOPS: 23

Min IOPS: 0

Average Latency(s): 2.46023

Stddev Latency(s): 2.47093

Max latency(s): 5.30316

Min latency(s): 0.0224793

[jianshu.ljs@lvjian100081200005.et15sqa /home/jianshu.ljs/ceph_10.2.2/src]

$rbd create rbd_test_image_01 --size 1000

[jianshu.ljs@lvjian100081200005.et15sqa /home/jianshu.ljs/ceph_10.2.2/src]

$rbd ls

rbd_test_image_01

[jianshu.ljs@lvjian100081200005.et15sqa /home/jianshu.ljs/ceph_10.2.2/src]

$rbd info rbd_test_image_01

rbd image 'rbd_test_image_01':

size 1000 MB in 250 objects

order 22 (4096 kB objects)

block_name_prefix: rbd_data.102e74b0dc51

format: 2

features: layering, exclusive-lock, object-map, fast-diff, deep-flatten

flags: