diff --git a/scripts/autotest_common.sh b/scripts/autotest_common.sh index b5fd63702..77075a3c1 100755 --- a/scripts/autotest_common.sh +++ b/scripts/autotest_common.sh @@ -19,6 +19,7 @@ fi : ${SPDK_TEST_ISCSI=1}; export SPDK_TEST_ISCSI : ${SPDK_TEST_NVME=1}; export SPDK_TEST_NVME : ${SPDK_TEST_NVMF=1}; export SPDK_TEST_NVMF +: ${SPDK_TEST_RBD=1}; export SPDK_TEST_RBD : ${SPDK_TEST_VHOST=1}; export SPDK_TEST_VHOST : ${SPDK_TEST_BLOCKDEV=1}; export SPDK_TEST_BLOCKDEV : ${SPDK_TEST_IOAT=1}; export SPDK_TEST_IOAT @@ -251,12 +252,19 @@ function rbd_setup() { export RBD_NAME=foo (cd $CEPH_DIR && ../src/vstart.sh -d -n -x -l) /usr/local/bin/rbd create $RBD_NAME --size 1000 + elif hash ceph; then + export RBD_POOL=rbd + export RBD_NAME=foo + (cd $rootdir/scripts/ceph && ./start.sh) + rbd create $RBD_NAME --size 1000 fi } function rbd_cleanup() { if [ -d $CEPH_DIR ]; then (cd $CEPH_DIR && ../src/stop.sh || true) + elif hash ceph; then + (cd $rootdir/scripts/ceph && ./stop.sh || true) fi } diff --git a/scripts/ceph/ceph.conf b/scripts/ceph/ceph.conf new file mode 100644 index 000000000..13095d988 --- /dev/null +++ b/scripts/ceph/ceph.conf @@ -0,0 +1,67 @@ +[global] + debug_lockdep = 0/0 + debug_context = 0/0 + debug_crush = 0/0 + debug_buffer = 0/0 + debug_timer = 0/0 + debug_filer = 0/0 + debug_objecter = 0/0 + debug_rados = 0/0 + debug_rbd = 0/0 + debug_ms = 0/0 + debug_monc = 0/0 + debug_tp = 0/0 + debug_auth = 0/0 + debug_finisher = 0/0 + debug_heartbeatmap = 0/0 + debug_perfcounter = 0/0 + debug_asok = 0/0 + debug_throttle = 0/0 + debug_mon = 0/0 + debug_paxos = 0/0 + debug_rgw = 0/0 + + perf = true + mutex_perf_counter = false + throttler_perf_counter = false + rbd cache = false + mon_allow_pool_delete = true + + osd_pool_default_size = 1 + +[mon] + mon_max_pool_pg_num=166496 + mon_osd_max_split_count = 10000 + mon_pg_warn_max_per_osd = 10000 + +[mon.a] + mon addr = 127.0.0.1:12046 + +[osd] + osd_op_threads = 64 + filestore_queue_max_ops=5000 + filestore_queue_committing_max_ops=5000 + journal_max_write_entries=1000 + journal_queue_max_ops=3000 + objecter_inflight_ops=102400 + filestore_wbthrottle_enable=false + filestore_queue_max_bytes=1048576000 + filestore_queue_committing_max_bytes=1048576000 + journal_max_write_bytes=1048576000 + journal_queue_max_bytes=1048576000 + ms_dispatch_throttle_bytes=1048576000 + objecter_infilght_op_bytes=1048576000 + filestore_max_sync_interval=10 + osd_client_message_size_cap = 0 + osd_client_message_cap = 0 + osd_enable_op_tracker = false + filestore_fd_cache_size = 10240 + filestore_fd_cache_shards = 64 + filestore_op_threads = 16 + osd_op_num_shards = 48 + osd_op_num_threads_per_shard = 2 + osd_pg_object_context_cache_count = 10240 + filestore_odsync_write = True + journal_dynamic_throttle = True + +[osd.0] diff --git a/scripts/ceph/start.sh b/scripts/ceph/start.sh new file mode 100755 index 000000000..a7f76c868 --- /dev/null +++ b/scripts/ceph/start.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# create mon + +set -x +set -e + +fsid=`uuidgen` +base_dir=`pwd` +mon_ip=127.0.0.1 +mon_dir=${base_dir}/ceph/mon.a/ +pid_dir=${base_dir}/ceph/pid +ceph_conf=${base_dir}/ceph.conf +ceph_conf_temp=${base_dir}/ceph.conf.tmp +mnt_dir=${base_dir}/ceph/mnt +dev_backend=/dev/ceph +image=${base_dir}/../../../output/ceph_raw.img +dev=/dev/loop200 +# partition osd + +cp $ceph_conf $ceph_conf_temp + +if [ ! -e $image ]; then + fallocate -l 10G $image +fi + +mknod ${dev_backend} b 7 200 || true +losetup ${dev_backend} ${image} || true + +PARTED="parted -s" +SGDISK="sgdisk" + +echo "Partitioning ${dev}" +${PARTED} ${dev} mktable gpt +sleep 2 +${PARTED} ${dev} mkpart primary 0% 5GiB +${PARTED} ${dev} mkpart primary 5GiB 100% + + +partno=0 +echo "Setting name on ${dev}" +${SGDISK} -c 1:osd-device-${partno}-journal ${dev} +${SGDISK} -c 2:osd-device-${partno}-data ${dev} +kpartx ${dev} + +# prep osds + +mnt_pt=${mnt_dir}/osd-device-0-data/ +mkdir -p ${mnt_pt} +mkfs.xfs -f /dev/disk/by-partlabel/osd-device-0-data +mount /dev/disk/by-partlabel/osd-device-0-data ${mnt_pt} +echo -e "\tosd data = ${mnt_pt}" >> "$ceph_conf" +echo -e "\tosd journal = /dev/disk/by-partlabel/osd-device-0-journal" >> "$ceph_conf" + + +# create mon +rm -rf ${mon_dir}/* +mkdir -p ${mon_dir} +mkdir -p ${pid_dir} + +ceph-authtool --create-keyring --gen-key --name=mon. ${base_dir}/ceph/keyring --cap mon 'allow *' +ceph-authtool --gen-key --name=client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' ${base_dir}/ceph/keyring + +monmaptool --create --clobber --add a ${mon_ip}:12046 --print ${base_dir}/ceph/monmap + +sh -c "ulimit -c unlimited && exec ceph-mon --mkfs -c ${ceph_conf} -i a --monmap=${base_dir}/ceph/monmap --keyring=${base_dir}/ceph/keyring --mon-data=${mon_dir}" + +cp ${base_dir}/ceph/keyring ${mon_dir}/keyring + +cp $ceph_conf /etc/ceph/ceph.conf + +cp ${base_dir}/ceph/keyring /etc/ceph/keyring + +ceph-run sh -c "ulimit -n 16384 && ulimit -c unlimited && exec ceph-mon -c ${ceph_conf} -i a --keyring=${base_dir}/ceph/keyring --pid-file=${base_dir}/ceph/pid/root@`hostname`.pid --mon-data=${mon_dir}" || true + +# create osd + +i=0 + +mkdir -p ${mnt_dir} + +uuid=`uuidgen` +ceph -c ${ceph_conf} osd create ${uuid} $i +ceph-osd -c ${ceph_conf} -i $i --mkfs --mkkey --osd-uuid ${uuid} +ceph -c ${ceph_conf} osd crush add osd.${i} 1.0 host=`hostname` root=default +ceph -c ${ceph_conf} -i ${mnt_dir}/osd-device-${i}-data/keyring auth add osd.${i} osd "allow *" mon "allow profile osd" mgr "allow" + +# start osd +pkill -9 ceph-osd || true +sleep 2 + +mkdir -p ${pid_dir} +env -i TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 ceph-osd -c ${ceph_conf} -i 0 --pid-file=${pid_dir}/ceph-osd.0.pid + +rm -f $ceph_conf +mv $ceph_conf_temp $ceph_conf diff --git a/scripts/ceph/stop.sh b/scripts/ceph/stop.sh new file mode 100755 index 000000000..c9cf0eab7 --- /dev/null +++ b/scripts/ceph/stop.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -x + +base_dir=`pwd` +home_folder=${base_dir}/ceph +image=${base_dir}/../../../output/ceph_raw.img +pkill -9 ceph +sleep 3 +umount /dev/loop200p2 +rm -rf $home_folder +rm $image diff --git a/test/iscsi_tgt/iscsi_tgt.sh b/test/iscsi_tgt/iscsi_tgt.sh index 559974499..cc3baa2e0 100755 --- a/test/iscsi_tgt/iscsi_tgt.sh +++ b/test/iscsi_tgt/iscsi_tgt.sh @@ -36,7 +36,9 @@ if [ $RUN_NIGHTLY -eq 1 ]; then run_test ./test/iscsi_tgt/ip_migration/ip_migration.sh fi run_test ./test/iscsi_tgt/ext4test/ext4test.sh -run_test ./test/iscsi_tgt/rbd/rbd.sh +if [ $SPDK_TEST_RBD -eq 1 ]; then + run_test ./test/iscsi_tgt/rbd/rbd.sh +fi trap - SIGINT SIGTERM EXIT kill_stub