2021-04-18 19:41:13 +00:00
|
|
|
// Copyright (c) 2015-2021 MinIO, Inc.
|
|
|
|
//
|
|
|
|
// This file is part of MinIO Object Storage stack
|
|
|
|
//
|
|
|
|
// This program is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Affero General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
2016-08-31 02:22:27 +00:00
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
2020-08-04 21:55:53 +00:00
|
|
|
"context"
|
2022-01-24 19:28:45 +00:00
|
|
|
"errors"
|
2017-06-17 18:20:12 +00:00
|
|
|
"fmt"
|
2020-01-15 02:45:17 +00:00
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2019-10-30 07:04:39 +00:00
|
|
|
"sync"
|
2016-08-31 02:22:27 +00:00
|
|
|
"time"
|
|
|
|
|
2020-05-14 03:25:29 +00:00
|
|
|
"github.com/dustin/go-humanize"
|
2021-06-01 21:59:40 +00:00
|
|
|
xhttp "github.com/minio/minio/internal/http"
|
|
|
|
"github.com/minio/minio/internal/logger"
|
2016-08-31 02:22:27 +00:00
|
|
|
)
|
|
|
|
|
2020-07-17 17:08:04 +00:00
|
|
|
var printEndpointError = func() func(Endpoint, error, bool) {
|
2019-10-30 07:04:39 +00:00
|
|
|
var mutex sync.Mutex
|
2020-07-17 17:08:04 +00:00
|
|
|
printOnce := make(map[Endpoint]map[string]int)
|
2018-02-16 01:45:57 +00:00
|
|
|
|
2020-07-17 17:08:04 +00:00
|
|
|
return func(endpoint Endpoint, err error, once bool) {
|
2018-09-14 04:42:50 +00:00
|
|
|
reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
|
2020-04-09 16:30:02 +00:00
|
|
|
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
|
2019-10-30 07:04:39 +00:00
|
|
|
mutex.Lock()
|
|
|
|
defer mutex.Unlock()
|
2020-07-17 17:08:04 +00:00
|
|
|
|
2018-02-16 01:45:57 +00:00
|
|
|
m, ok := printOnce[endpoint]
|
|
|
|
if !ok {
|
2020-07-17 17:08:04 +00:00
|
|
|
m = make(map[string]int)
|
|
|
|
m[err.Error()]++
|
2018-02-16 01:45:57 +00:00
|
|
|
printOnce[endpoint] = m
|
2020-07-17 17:08:04 +00:00
|
|
|
if once {
|
|
|
|
logger.LogAlwaysIf(ctx, err)
|
|
|
|
return
|
|
|
|
}
|
2017-12-28 17:32:48 +00:00
|
|
|
}
|
2020-07-17 17:08:04 +00:00
|
|
|
// Once is set and we are here means error was already
|
|
|
|
// printed once.
|
|
|
|
if once {
|
2018-02-16 01:45:57 +00:00
|
|
|
return
|
2017-12-28 17:32:48 +00:00
|
|
|
}
|
2020-07-17 17:08:04 +00:00
|
|
|
// once not set, check if same error occurred 3 times in
|
|
|
|
// a row, then make sure we print it to call attention.
|
|
|
|
if m[err.Error()] > 2 {
|
|
|
|
logger.LogAlwaysIf(ctx, fmt.Errorf("Following error has been printed %d times.. %w", m[err.Error()], err))
|
|
|
|
// Reduce the count to introduce further delay in printing
|
|
|
|
// but let it again print after the 2th attempt
|
|
|
|
m[err.Error()]--
|
|
|
|
m[err.Error()]--
|
|
|
|
}
|
|
|
|
m[err.Error()]++
|
2017-12-28 17:32:48 +00:00
|
|
|
}
|
2018-02-16 01:45:57 +00:00
|
|
|
}()
|
2016-08-31 02:22:27 +00:00
|
|
|
|
2021-06-05 16:10:32 +00:00
|
|
|
// Cleans up tmp directory of the local disk.
|
2022-05-12 22:24:58 +00:00
|
|
|
func bgFormatErasureCleanupTmp(diskPath string) {
|
2021-06-05 16:10:32 +00:00
|
|
|
// Need to move temporary objects left behind from previous run of minio
|
2021-06-16 23:19:18 +00:00
|
|
|
// server to a unique directory under `minioMetaTmpBucket-old` to clean
|
2021-06-05 16:10:32 +00:00
|
|
|
// up `minioMetaTmpBucket` for the current run.
|
|
|
|
//
|
2021-06-16 23:19:18 +00:00
|
|
|
// /disk1/.minio.sys/tmp-old/
|
2021-06-05 16:10:32 +00:00
|
|
|
// |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
|
|
|
|
// |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
|
|
|
|
//
|
|
|
|
// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
|
|
|
|
// temporary objects from one of the previous runs of minio server.
|
2022-01-24 19:28:45 +00:00
|
|
|
tmpID := mustGetUUID()
|
|
|
|
tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", tmpID)
|
2021-06-05 16:10:32 +00:00
|
|
|
if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket),
|
2022-01-24 19:28:45 +00:00
|
|
|
tmpOld); err != nil && !errors.Is(err, errFileNotFound) {
|
2021-06-05 16:10:32 +00:00
|
|
|
logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
|
|
|
|
pathJoin(diskPath, minioMetaTmpBucket),
|
|
|
|
tmpOld,
|
|
|
|
osErrToFileErr(err)))
|
2016-11-23 23:48:10 +00:00
|
|
|
}
|
2019-01-25 21:33:28 +00:00
|
|
|
|
2022-01-02 17:15:06 +00:00
|
|
|
if err := mkdirAll(pathJoin(diskPath, minioMetaTmpDeletedBucket), 0o777); err != nil {
|
2021-06-05 16:10:32 +00:00
|
|
|
logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate",
|
|
|
|
pathJoin(diskPath, minioMetaTmpBucket),
|
|
|
|
err))
|
2018-03-15 20:55:23 +00:00
|
|
|
}
|
2022-01-24 19:28:45 +00:00
|
|
|
|
2022-10-31 14:27:50 +00:00
|
|
|
// Remove the entire folder in case there are leftovers that didn't get cleaned up before restart.
|
|
|
|
go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
|
2022-01-24 19:28:45 +00:00
|
|
|
// Renames and schedules for purging all bucket metacache.
|
2022-05-12 22:24:58 +00:00
|
|
|
go renameAllBucketMetacache(diskPath)
|
2018-03-15 20:55:23 +00:00
|
|
|
}
|
|
|
|
|
2018-04-04 04:58:48 +00:00
|
|
|
// Following error message is added to fix a regression in release
|
|
|
|
// RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3. This
|
|
|
|
// migration failed to capture '.This' field properly which indicates
|
|
|
|
// the disk UUID association. Below error message is returned when
|
|
|
|
// we see this situation in format.json, for more info refer
|
|
|
|
// https://github.com/minio/minio/issues/5667
|
2020-06-13 03:04:01 +00:00
|
|
|
var errErasureV3ThisEmpty = fmt.Errorf("Erasure format version 3 has This field empty")
|
2018-04-04 04:58:48 +00:00
|
|
|
|
2021-01-08 03:16:18 +00:00
|
|
|
// isServerResolvable - checks if the endpoint is resolvable
|
2020-01-15 02:45:17 +00:00
|
|
|
// by sending a naked HTTP request with liveness checks.
|
2021-01-25 18:01:27 +00:00
|
|
|
func isServerResolvable(endpoint Endpoint, timeout time.Duration) error {
|
2020-01-15 02:45:17 +00:00
|
|
|
serverURL := &url.URL{
|
|
|
|
Scheme: endpoint.Scheme,
|
|
|
|
Host: endpoint.Host,
|
2020-10-30 19:20:28 +00:00
|
|
|
Path: pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
|
2020-01-15 02:45:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
httpClient := &http.Client{
|
2022-03-17 23:20:10 +00:00
|
|
|
Transport: globalInternodeTransport,
|
2020-01-15 02:45:17 +00:00
|
|
|
}
|
|
|
|
|
2021-01-25 18:01:27 +00:00
|
|
|
ctx, cancel := context.WithTimeout(GlobalContext, timeout)
|
2020-08-04 21:55:53 +00:00
|
|
|
|
2020-09-08 21:22:04 +00:00
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL.String(), nil)
|
|
|
|
if err != nil {
|
2020-10-30 19:20:28 +00:00
|
|
|
cancel()
|
2020-09-08 21:22:04 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := httpClient.Do(req)
|
2020-10-30 19:20:28 +00:00
|
|
|
cancel()
|
2020-01-15 02:45:17 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-10-30 19:20:28 +00:00
|
|
|
xhttp.DrainBody(resp.Body)
|
2020-01-15 02:45:17 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-06-13 03:04:01 +00:00
|
|
|
// connect to list of endpoints and load all Erasure disk formats, validate the formats are correct
|
2018-04-04 04:58:48 +00:00
|
|
|
// and are in quorum, if no formats are found attempt to initialize all of them for the first
|
|
|
|
// time. additionally make sure to close all the disks used in this attempt.
|
2022-02-09 11:14:22 +00:00
|
|
|
func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID, distributionAlgo string) (storageDisks []StorageAPI, format *formatErasureV3, err error) {
|
2018-09-10 23:21:59 +00:00
|
|
|
// Initialize all storage disks
|
2022-10-31 14:27:50 +00:00
|
|
|
storageDisks, errs := initStorageDisksWithErrors(endpoints, true)
|
2020-01-15 02:45:17 +00:00
|
|
|
|
2020-04-04 01:06:31 +00:00
|
|
|
defer func(storageDisks []StorageAPI) {
|
|
|
|
if err != nil {
|
2022-05-30 17:58:37 +00:00
|
|
|
closeStorageDisks(storageDisks...)
|
2020-04-04 01:06:31 +00:00
|
|
|
}
|
|
|
|
}(storageDisks)
|
|
|
|
|
2019-09-27 23:47:12 +00:00
|
|
|
for i, err := range errs {
|
2022-05-30 17:58:37 +00:00
|
|
|
if err != nil && !errors.Is(err, errXLBackend) {
|
|
|
|
if errors.Is(err, errDiskNotFound) && verboseLogging {
|
2022-10-28 21:11:20 +00:00
|
|
|
if globalEndpoints.NEndpoints() > 1 {
|
|
|
|
logger.Error("Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
|
|
|
|
} else {
|
|
|
|
logger.Fatal(err, "Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
|
|
|
|
}
|
2021-05-15 19:56:58 +00:00
|
|
|
} else {
|
2022-10-28 21:11:20 +00:00
|
|
|
if globalEndpoints.NEndpoints() > 1 {
|
|
|
|
logger.Error("Unable to use the drive %s: %v", endpoints[i], err)
|
|
|
|
} else {
|
|
|
|
logger.Fatal(errInvalidArgument, "Unable to use the drive %s: %v", endpoints[i], err)
|
|
|
|
}
|
2020-01-15 02:45:17 +00:00
|
|
|
}
|
2019-09-27 23:47:12 +00:00
|
|
|
}
|
|
|
|
}
|
2018-04-04 04:58:48 +00:00
|
|
|
|
2021-12-03 17:25:17 +00:00
|
|
|
if err := checkDiskFatalErrs(errs); err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
2018-12-04 18:25:56 +00:00
|
|
|
// Attempt to load all `format.json` from all disks.
|
2020-06-13 03:04:01 +00:00
|
|
|
formatConfigs, sErrs := loadFormatErasureAll(storageDisks, false)
|
2018-12-04 18:25:56 +00:00
|
|
|
// Check if we have
|
|
|
|
for i, sErr := range sErrs {
|
2020-08-04 01:17:48 +00:00
|
|
|
// print the error, nonetheless, which is perhaps unhandled
|
2022-05-30 17:58:37 +00:00
|
|
|
if !errors.Is(sErr, errUnformattedDisk) && !errors.Is(sErr, errDiskNotFound) && verboseLogging {
|
2020-03-23 14:32:18 +00:00
|
|
|
if sErr != nil {
|
2021-12-03 17:25:17 +00:00
|
|
|
logger.Error("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)
|
2020-03-23 14:32:18 +00:00
|
|
|
}
|
2020-03-08 19:12:07 +00:00
|
|
|
}
|
2018-12-04 18:25:56 +00:00
|
|
|
}
|
|
|
|
|
2018-04-04 04:58:48 +00:00
|
|
|
// Pre-emptively check if one of the formatted disks
|
|
|
|
// is invalid. This function returns success for the
|
|
|
|
// most part unless one of the formats is not consistent
|
2020-06-13 03:04:01 +00:00
|
|
|
// with expected Erasure format. For example if a user is
|
|
|
|
// trying to pool FS backend into an Erasure set.
|
2021-01-29 19:40:55 +00:00
|
|
|
if err = checkFormatErasureValues(formatConfigs, storageDisks, setDriveCount); err != nil {
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2018-04-04 04:58:48 +00:00
|
|
|
}
|
|
|
|
|
2023-02-06 17:26:09 +00:00
|
|
|
// Return error when quorum unformatted disks - indicating we are
|
|
|
|
// waiting for first server to be online.
|
|
|
|
unformattedDisks := quorumUnformattedDisks(sErrs)
|
|
|
|
if unformattedDisks && !firstDisk {
|
|
|
|
return nil, nil, errNotFirstDisk
|
|
|
|
}
|
|
|
|
|
2018-04-12 22:43:38 +00:00
|
|
|
// All disks report unformatted we should initialized everyone.
|
2023-02-06 17:26:09 +00:00
|
|
|
if unformattedDisks && firstDisk {
|
2021-01-06 17:35:47 +00:00
|
|
|
logger.Info("Formatting %s pool, %v set(s), %v drives per set.",
|
|
|
|
humanize.Ordinal(poolCount), setCount, setDriveCount)
|
2020-01-17 23:39:07 +00:00
|
|
|
|
2019-04-02 17:50:13 +00:00
|
|
|
// Initialize erasure code format on disks
|
2021-01-19 18:01:31 +00:00
|
|
|
format, err = initFormatErasure(GlobalContext, storageDisks, setCount, setDriveCount, deploymentID, distributionAlgo, sErrs)
|
2019-04-02 17:50:13 +00:00
|
|
|
if err != nil {
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2019-04-02 17:50:13 +00:00
|
|
|
}
|
2020-05-14 03:25:29 +00:00
|
|
|
|
2019-04-02 17:50:13 +00:00
|
|
|
// Assign globalDeploymentID on first run for the
|
|
|
|
// minio server managing the first disk
|
|
|
|
globalDeploymentID = format.ID
|
2020-03-27 21:48:30 +00:00
|
|
|
return storageDisks, format, nil
|
2019-12-24 00:31:03 +00:00
|
|
|
}
|
|
|
|
|
2020-08-18 21:37:26 +00:00
|
|
|
// Mark all root disks down
|
|
|
|
markRootDisksAsDown(storageDisks, sErrs)
|
|
|
|
|
2018-04-04 04:58:48 +00:00
|
|
|
// Following function is added to fix a regressions which was introduced
|
|
|
|
// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
|
|
|
|
// This migration failed to capture '.This' field properly which indicates
|
|
|
|
// the disk UUID association. Below function is called to handle and fix
|
|
|
|
// this regression, for more info refer https://github.com/minio/minio/issues/5667
|
2020-06-13 03:04:01 +00:00
|
|
|
if err = fixFormatErasureV3(storageDisks, endpoints, formatConfigs); err != nil {
|
2021-05-15 19:56:58 +00:00
|
|
|
logger.LogIf(GlobalContext, err)
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2018-04-04 04:58:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If any of the .This field is still empty, we return error.
|
2020-06-13 03:04:01 +00:00
|
|
|
if formatErasureV3ThisEmpty(formatConfigs) {
|
|
|
|
return nil, nil, errErasureV3ThisEmpty
|
2018-04-04 04:58:48 +00:00
|
|
|
}
|
|
|
|
|
2020-06-13 03:04:01 +00:00
|
|
|
format, err = getFormatErasureInQuorum(formatConfigs)
|
2018-04-04 04:58:48 +00:00
|
|
|
if err != nil {
|
2021-05-15 19:56:58 +00:00
|
|
|
logger.LogIf(GlobalContext, err)
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2018-04-04 04:58:48 +00:00
|
|
|
}
|
|
|
|
|
2018-07-19 03:17:35 +00:00
|
|
|
if format.ID == "" {
|
2019-11-13 20:17:45 +00:00
|
|
|
// Not a first disk, wait until first disk fixes deploymentID
|
|
|
|
if !firstDisk {
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, errNotFirstDisk
|
2019-11-13 20:17:45 +00:00
|
|
|
}
|
2020-06-13 03:04:01 +00:00
|
|
|
if err = formatErasureFixDeploymentID(endpoints, storageDisks, format); err != nil {
|
2021-05-15 19:56:58 +00:00
|
|
|
logger.LogIf(GlobalContext, err)
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2018-07-19 03:17:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-19 22:47:03 +00:00
|
|
|
globalDeploymentID = format.ID
|
2018-07-19 03:17:35 +00:00
|
|
|
|
2020-06-13 03:04:01 +00:00
|
|
|
if err = formatErasureFixLocalDeploymentID(endpoints, storageDisks, format); err != nil {
|
2021-05-15 19:56:58 +00:00
|
|
|
logger.LogIf(GlobalContext, err)
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2018-07-19 03:17:35 +00:00
|
|
|
}
|
2019-12-24 00:31:03 +00:00
|
|
|
|
2020-03-27 21:48:30 +00:00
|
|
|
return storageDisks, format, nil
|
2018-04-04 04:58:48 +00:00
|
|
|
}
|
|
|
|
|
2018-02-16 01:45:57 +00:00
|
|
|
// Format disks before initialization of object layer.
|
2021-01-19 18:01:31 +00:00
|
|
|
func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID, distributionAlgo string) ([]StorageAPI, *formatErasureV3, error) {
|
2020-08-27 02:29:35 +00:00
|
|
|
if len(endpoints) == 0 || setCount == 0 || setDriveCount == 0 {
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, errInvalidArgument
|
2016-11-02 15:51:06 +00:00
|
|
|
}
|
2018-02-16 01:45:57 +00:00
|
|
|
|
2016-12-07 18:22:00 +00:00
|
|
|
// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
|
|
|
|
// All times are rounded to avoid showing milli, micro and nano seconds
|
|
|
|
formatStartTime := time.Now().Round(time.Second)
|
|
|
|
getElapsedTime := func() string {
|
|
|
|
return time.Now().Round(time.Second).Sub(formatStartTime).String()
|
|
|
|
}
|
|
|
|
|
2022-01-20 21:03:15 +00:00
|
|
|
var tries int
|
2022-02-09 11:14:22 +00:00
|
|
|
var verboseLogging bool
|
|
|
|
storageDisks, format, err := connectLoadInitFormats(verboseLogging, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
|
2022-01-20 21:03:15 +00:00
|
|
|
if err == nil {
|
|
|
|
return storageDisks, format, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
tries++ // tried already once
|
|
|
|
|
2020-01-08 21:36:54 +00:00
|
|
|
// Wait on each try for an update.
|
2022-02-04 20:21:21 +00:00
|
|
|
ticker := time.NewTicker(150 * time.Millisecond)
|
2020-01-08 21:36:54 +00:00
|
|
|
defer ticker.Stop()
|
2022-01-20 21:03:15 +00:00
|
|
|
|
2016-11-02 22:27:36 +00:00
|
|
|
for {
|
|
|
|
select {
|
2020-01-08 21:36:54 +00:00
|
|
|
case <-ticker.C:
|
2022-02-09 11:14:22 +00:00
|
|
|
// Only log once every 10 iterations, then reset the tries count.
|
|
|
|
verboseLogging = tries >= 10
|
|
|
|
if verboseLogging {
|
2022-02-04 20:21:21 +00:00
|
|
|
tries = 1
|
|
|
|
}
|
|
|
|
|
2022-02-09 11:14:22 +00:00
|
|
|
storageDisks, format, err := connectLoadInitFormats(verboseLogging, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
|
2018-04-04 04:58:48 +00:00
|
|
|
if err != nil {
|
2020-01-08 21:36:54 +00:00
|
|
|
tries++
|
2018-04-04 04:58:48 +00:00
|
|
|
switch err {
|
|
|
|
case errNotFirstDisk:
|
|
|
|
// Fresh setup, wait for first server to be up.
|
2022-08-04 23:10:08 +00:00
|
|
|
logger.Info("Waiting for the first server to format the drives (elapsed %s)\n", getElapsedTime())
|
2017-04-18 17:35:17 +00:00
|
|
|
continue
|
2018-04-12 22:43:38 +00:00
|
|
|
case errFirstDiskWait:
|
|
|
|
// Fresh setup, wait for other servers to come up.
|
2022-08-04 23:10:08 +00:00
|
|
|
logger.Info("Waiting for all other servers to be online to format the drives (elapses %s)\n", getElapsedTime())
|
2018-04-12 22:43:38 +00:00
|
|
|
continue
|
2020-06-13 03:04:01 +00:00
|
|
|
case errErasureReadQuorum:
|
2018-04-04 04:58:48 +00:00
|
|
|
// no quorum available continue to wait for minimum number of servers.
|
2022-08-04 23:10:08 +00:00
|
|
|
logger.Info("Waiting for a minimum of %d drives to come online (elapsed %s)\n",
|
2021-08-23 08:13:47 +00:00
|
|
|
len(endpoints)/2, getElapsedTime())
|
|
|
|
continue
|
|
|
|
case errErasureWriteQuorum:
|
|
|
|
// no quorum available continue to wait for minimum number of servers.
|
2022-08-04 23:10:08 +00:00
|
|
|
logger.Info("Waiting for a minimum of %d drives to come online (elapsed %s)\n",
|
2021-08-23 08:13:47 +00:00
|
|
|
(len(endpoints)/2)+1, getElapsedTime())
|
2018-04-04 04:58:48 +00:00
|
|
|
continue
|
2020-06-13 03:04:01 +00:00
|
|
|
case errErasureV3ThisEmpty:
|
2018-04-04 04:58:48 +00:00
|
|
|
// need to wait for this error to be healed, so continue.
|
|
|
|
continue
|
|
|
|
default:
|
|
|
|
// For all other unhandled errors we exit and fail.
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, err
|
2017-04-18 17:35:17 +00:00
|
|
|
}
|
2016-12-11 23:18:55 +00:00
|
|
|
}
|
2020-03-27 21:48:30 +00:00
|
|
|
return storageDisks, format, nil
|
2018-02-06 23:07:17 +00:00
|
|
|
case <-globalOSSignalCh:
|
2020-03-27 21:48:30 +00:00
|
|
|
return nil, nil, fmt.Errorf("Initializing data volumes gracefully stopped")
|
2016-11-02 22:27:36 +00:00
|
|
|
}
|
|
|
|
}
|
2016-08-31 02:22:27 +00:00
|
|
|
}
|