#!/bin/bash -e # set -E set -o pipefail if [ ! -x "$PWD/minio" ]; then echo "minio executable binary not found in current directory" exit 1 fi WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) function start_minio_3_node() { export MINIO_ROOT_USER=minio export MINIO_ROOT_PASSWORD=minio123 export MINIO_ERASURE_SET_DRIVE_COUNT=6 export MINIO_CI_CD=1 start_port=$1 args="" for i in $(seq 1 3); do args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/" done "${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 & pid1=$! disown ${pid1} "${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 & pid2=$! disown $pid2 "${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 & pid3=$! disown $pid3 export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))" /tmp/mc ready myminio # Wait for all drives to be online and formatted while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].state | select(. != "ok")' | wc -l) -gt 0 ]; do sleep 1; done # Wait for all drives to be healed while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].healing | select(. != null) | select(. == true)' | wc -l) -gt 0 ]; do sleep 1; done # Wait for Status: in MinIO output while true; do rv=$(check_online) if [ "$rv" != "1" ]; then # success break fi # Check if we should retry retry=$((retry + 1)) if [ $retry -le 20 ]; then sleep 5 continue fi # Failure for i in $(seq 1 3); do echo "server$i log:" cat "${WORK_DIR}/dist-minio-server$i.log" done pkill -9 minio echo "FAILED" purge "$WORK_DIR" exit 1 done if ! ps -p $pid1 1>&2 >/dev/null; then echo "server1 log:" cat "${WORK_DIR}/dist-minio-server1.log" echo "FAILED" purge "$WORK_DIR" exit 1 fi if ! ps -p $pid2 1>&2 >/dev/null; then echo "server2 log:" cat "${WORK_DIR}/dist-minio-server2.log" echo "FAILED" purge "$WORK_DIR" exit 1 fi if ! ps -p $pid3 1>&2 >/dev/null; then echo "server3 log:" cat "${WORK_DIR}/dist-minio-server3.log" echo "FAILED" purge "$WORK_DIR" exit 1 fi if ! pkill minio; then for i in $(seq 1 3); do echo "server$i log:" cat "${WORK_DIR}/dist-minio-server$i.log" done echo "FAILED" purge "$WORK_DIR" exit 1 fi sleep 1 if pgrep minio; then # forcibly killing, to proceed further properly. if ! pkill -9 minio; then echo "no minio process running anymore, proceed." fi fi } function check_online() { if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then echo "1" fi } function purge() { echo rm -rf "$1" } function __init__() { echo "Initializing environment" mkdir -p "$WORK_DIR" mkdir -p "$MINIO_CONFIG_DIR" ## version is purposefully set to '3' for minio to migrate configuration file echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" if [ ! -f /tmp/mc ]; then wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x /tmp/mc fi } function perform_test() { start_minio_3_node $2 echo "Testing Distributed Erasure setup healing of drives" echo "Remove the contents of the disks belonging to '${1}' erasure set" rm -rf ${WORK_DIR}/${1}/*/ set -x start_minio_3_node $2 } function main() { # use same ports for all tests start_port=$(shuf -i 10000-65000 -n 1) perform_test "2" ${start_port} perform_test "1" ${start_port} perform_test "3" ${start_port} } (__init__ "$@" && main "$@") rv=$? purge "$WORK_DIR" exit "$rv"