commit 6a0feecaef478c2764cf138a5c0cdc4e46cce080 Author: Chinasubbareddy Mallavarapu Date: Sat Oct 10 21:32:49 2020 -0500 [ceph-osd] Fix the sync issue between osds when using shared disk for metadata This is to fix the sync between ceph osds when they are using shared disk for metadata as they are having conflict while preparing the metadata disk. we are adding a lock when first osd preparing the sahred metadata disk so that other osd will wait for the lock, also adding udev settle in few places to get latest tags on lvm devices. Change-Id: I018bd12a3f02cf8cd3486b9c97e14b138b5dac76 diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml index 0646ccb..abd9da7 100644 --- a/ceph-osd/Chart.yaml +++ b/ceph-osd/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph OSD name: ceph-osd -version: 0.1.4 +version: 0.1.5 home: https://github.com/ceph/ceph ... diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl index 0290502..8476e9a 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl @@ -80,6 +80,12 @@ function locked() { "$@" flock -u "${lock_fd}" } +function global_locked() { + exec {global_lock_fd}>/var/lib/ceph/tmp/init-osd-global.lock || exit 1 + flock -w 600 --verbose "${global_lock_fd}" + "$@" + flock -u "${global_lock_fd}" +} function crush_create_or_move { local crush_location=${1} diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl index 6cd9d5b..4bde715 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl @@ -185,6 +185,7 @@ function prep_device { local device_type=$3 local data_disk=$4 local vg_name lv_name VG DEVICE_OSD_ID logical_devices logical_volume + udev_settle vg_name=$(get_vg_name_from_device ${BLOCK_DEVICE}) lv_name=$(get_lv_name_from_device ${data_disk} ${device_type}) VG=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]') @@ -219,6 +220,7 @@ function prep_device { elif [[ "${device_type}" == "wal" ]]; then BLOCK_WAL="${VG}/${lv_name}" fi + udev_settle } function osd_disk_prepare { @@ -379,6 +381,7 @@ function osd_disk_prepare { fi if [[ ${CEPH_DISK_USED} -eq 1 ]]; then + udev_settle CLI_OPTS="${CLI_OPTS} --data ${OSD_DEVICE}" ceph-volume simple scan --force ${OSD_DEVICE}$(sgdisk --print ${OSD_DEVICE} | grep "F800" | awk '{print $1}') elif [[ ${CEPH_LVM_PREPARE} -eq 1 ]] || [[ ${DISK_ZAPPED} -eq 1 ]]; then @@ -411,12 +414,12 @@ function osd_disk_prepare { block_wal_string=$(echo ${BLOCK_WAL} | awk -F "/" '{print $2 "-" $3}') fi if [[ ${BLOCK_DB} && ${BLOCK_WAL} ]]; then - prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" - prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" + global_locked prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" + global_locked prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then - prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" + global_locked prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then - prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" + global_locked prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" fi else if pvdisplay ${OSD_DEVICE} | awk '/VG Name/{print $3}' | grep "ceph"; then