self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-07-21 18:35:07 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Implement bucket expansion (#8509)

Browse source
This commit is contained in:
Harshavardhana 2019-11-19 17:42:27 -08:00 committed by kannappanr
parent 3a34d98db8
commit 347b29d059
63 changed files with 2208 additions and 1166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/admin-handlers_test.go
View file

@ -61,7 +61,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
// Initialize boot time
globalBootTime = UTCNow()
globalEndpoints = mustGetNewEndpointList(xlDirs...)
globalEndpoints = mustGetZoneEndpoints(xlDirs...)
// Set globalIsXL to indicate that the setup uses an erasure
// code backend.
@ -113,7 +113,7 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
if err != nil {
return nil, nil, err
}
endpoints := mustGetNewEndpointList(xlDirs...)
endpoints := mustGetNewEndpoints(xlDirs...)
format, err := waitForFormatXL(true, endpoints, 1, 16)
if err != nil {
removeRoots(xlDirs)

157
cmd/admin-server-info.go
View file

@ -30,27 +30,28 @@ import (
cpuhw "github.com/shirou/gopsutil/cpu"
)
// getLocalMemUsage - returns ServerMemUsageInfo for only the
// local endpoints from given list of endpoints
func getLocalMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
// getLocalMemUsage - returns ServerMemUsageInfo for all zones, endpoints.
func getLocalMemUsage(endpointZones EndpointZones, r *http.Request) ServerMemUsageInfo {
var memUsages []mem.Usage
var historicUsages []mem.Usage
seenHosts := set.NewStringSet()
for _, endpoint := range endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
seenHosts.Add(endpoint.Host)
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
seenHosts.Add(endpoint.Host)
// Only proceed for local endpoints
if endpoint.IsLocal {
memUsages = append(memUsages, mem.GetUsage())
historicUsages = append(historicUsages, mem.GetHistoricUsage())
// Only proceed for local endpoints
if endpoint.IsLocal {
memUsages = append(memUsages, mem.GetUsage())
historicUsages = append(historicUsages, mem.GetHistoricUsage())
}
}
}
addr := r.Host
if globalIsDistXL {
addr = GetLocalPeer(endpoints)
addr = GetLocalPeer(endpointZones)
}
return ServerMemUsageInfo{
Addr: addr,
@ -59,27 +60,28 @@ func getLocalMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInf
}
}
// getLocalCPULoad - returns ServerCPULoadInfo for only the
// local endpoints from given list of endpoints
func getLocalCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
// getLocalCPULoad - returns ServerCPULoadInfo for all zones, endpoints.
func getLocalCPULoad(endpointZones EndpointZones, r *http.Request) ServerCPULoadInfo {
var cpuLoads []cpu.Load
var historicLoads []cpu.Load
seenHosts := set.NewStringSet()
for _, endpoint := range endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
seenHosts.Add(endpoint.Host)
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
seenHosts.Add(endpoint.Host)
// Only proceed for local endpoints
if endpoint.IsLocal {
cpuLoads = append(cpuLoads, cpu.GetLoad())
historicLoads = append(historicLoads, cpu.GetHistoricLoad())
// Only proceed for local endpoints
if endpoint.IsLocal {
cpuLoads = append(cpuLoads, cpu.GetLoad())
historicLoads = append(historicLoads, cpu.GetHistoricLoad())
}
}
}
addr := r.Host
if globalIsDistXL {
addr = GetLocalPeer(endpoints)
addr = GetLocalPeer(endpointZones)
}
return ServerCPULoadInfo{
Addr: addr,
@ -88,26 +90,27 @@ func getLocalCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo
}
}
// getLocalDrivesPerf - returns ServerDrivesPerfInfo for only the
// local endpoints from given list of endpoints
func getLocalDrivesPerf(endpoints EndpointList, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
// getLocalDrivesPerf - returns ServerDrivesPerfInfo for all zones, endpoints.
func getLocalDrivesPerf(endpointZones EndpointZones, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
var dps []disk.Performance
for _, endpoint := range endpoints {
// Only proceed for local endpoints
if endpoint.IsLocal {
if _, err := os.Stat(endpoint.Path); err != nil {
// Since this drive is not available, add relevant details and proceed
dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
continue
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
// Only proceed for local endpoints
if endpoint.IsLocal {
if _, err := os.Stat(endpoint.Path); err != nil {
// Since this drive is not available, add relevant details and proceed
dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
continue
}
dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
dp.Path = endpoint.Path
dps = append(dps, dp)
}
dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
dp.Path = endpoint.Path
dps = append(dps, dp)
}
}
addr := r.Host
if globalIsDistXL {
addr = GetLocalPeer(endpoints)
addr = GetLocalPeer(endpointZones)
}
return madmin.ServerDrivesPerfInfo{
Addr: addr,
@ -116,31 +119,32 @@ func getLocalDrivesPerf(endpoints EndpointList, size int64, r *http.Request) mad
}
}
// getLocalCPUInfo - returns ServerCPUHardwareInfo only for the
// local endpoints from given list of endpoints
func getLocalCPUInfo(endpoints EndpointList, r *http.Request) madmin.ServerCPUHardwareInfo {
// getLocalCPUInfo - returns ServerCPUHardwareInfo for all zones, endpoints.
func getLocalCPUInfo(endpointZones EndpointZones, r *http.Request) madmin.ServerCPUHardwareInfo {
var cpuHardwares []cpuhw.InfoStat
seenHosts := set.NewStringSet()
for _, endpoint := range endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
// Add to the list of visited hosts
seenHosts.Add(endpoint.Host)
// Only proceed for local endpoints
if endpoint.IsLocal {
cpuHardware, err := cpuhw.Info()
if err != nil {
return madmin.ServerCPUHardwareInfo{
Error: err.Error(),
}
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
// Add to the list of visited hosts
seenHosts.Add(endpoint.Host)
// Only proceed for local endpoints
if endpoint.IsLocal {
cpuHardware, err := cpuhw.Info()
if err != nil {
return madmin.ServerCPUHardwareInfo{
Error: err.Error(),
}
}
cpuHardwares = append(cpuHardwares, cpuHardware...)
}
cpuHardwares = append(cpuHardwares, cpuHardware...)
}
}
addr := r.Host
if globalIsDistXL {
addr = GetLocalPeer(endpoints)
addr = GetLocalPeer(endpointZones)
}
return madmin.ServerCPUHardwareInfo{
@ -149,31 +153,32 @@ func getLocalCPUInfo(endpoints EndpointList, r *http.Request) madmin.ServerCPUHa
}
}
// getLocalNetworkInfo - returns ServerNetworkHardwareInfo only for the
// local endpoints from given list of endpoints
func getLocalNetworkInfo(endpoints EndpointList, r *http.Request) madmin.ServerNetworkHardwareInfo {
// getLocalNetworkInfo - returns ServerNetworkHardwareInfo for all zones, endpoints.
func getLocalNetworkInfo(endpointZones EndpointZones, r *http.Request) madmin.ServerNetworkHardwareInfo {
var networkHardwares []net.Interface
seenHosts := set.NewStringSet()
for _, endpoint := range endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
// Add to the list of visited hosts
seenHosts.Add(endpoint.Host)
// Only proceed for local endpoints
if endpoint.IsLocal {
networkHardware, err := net.Interfaces()
if err != nil {
return madmin.ServerNetworkHardwareInfo{
Error: err.Error(),
}
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
// Add to the list of visited hosts
seenHosts.Add(endpoint.Host)
// Only proceed for local endpoints
if endpoint.IsLocal {
networkHardware, err := net.Interfaces()
if err != nil {
return madmin.ServerNetworkHardwareInfo{
Error: err.Error(),
}
}
networkHardwares = append(networkHardwares, networkHardware...)
}
networkHardwares = append(networkHardwares, networkHardware...)
}
}
addr := r.Host
if globalIsDistXL {
addr = GetLocalPeer(endpoints)
addr = GetLocalPeer(endpointZones)
}
return madmin.ServerNetworkHardwareInfo{

2
cmd/background-heal-ops.go
View file

@ -65,7 +65,7 @@ func (h *healRoutine) run() {
// Wait at max 10 minute for an inprogress request before proceeding to heal
waitCount := 600
// Any requests in progress, delay the heal.
for (globalHTTPServer.GetRequestCount() >= int32(globalXLSetCount*globalXLSetDriveCount)) &&
for (globalHTTPServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
waitCount > 0 {
waitCount--
time.Sleep(1 * time.Second)

76
cmd/background-newdisks-heal-ops.go
View file

@ -44,7 +44,7 @@ func monitorLocalDisksAndHeal() {
break
}
sets, ok := objAPI.(*xlSets)
z, ok := objAPI.(*xlZones)
if !ok {
return
}
@ -66,21 +66,24 @@ func monitorLocalDisksAndHeal() {
for {
time.Sleep(defaultMonitorNewDiskInterval)
localDisksToHeal := []Endpoint{}
for _, endpoint := range globalEndpoints {
if !endpoint.IsLocal {
localDisksInZoneHeal := make([]Endpoints, len(z.zones))
for i, ep := range globalEndpoints {
localDisksToHeal := Endpoints{}
for _, endpoint := range ep.Endpoints {
if !endpoint.IsLocal {
continue
}
// Try to connect to the current endpoint
// and reformat if the current disk is not formatted
_, _, err := connectEndpoint(endpoint)
if err == errUnformattedDisk {
localDisksToHeal = append(localDisksToHeal, endpoint)
}
}
if len(localDisksToHeal) == 0 {
continue
}
// Try to connect to the current endpoint
// and reformat if the current disk is not formatted
_, _, err := connectEndpoint(endpoint)
if err == errUnformattedDisk {
localDisksToHeal = append(localDisksToHeal, endpoint)
}
}
if len(localDisksToHeal) == 0 {
continue
localDisksInZoneHeal[i] = localDisksToHeal
}
// Reformat disks
@ -88,31 +91,36 @@ func monitorLocalDisksAndHeal() {
// Ensure that reformatting disks is finished
bgSeq.sourceCh <- nopHeal
var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
// Compute the list of erasure set to heal
var erasureSetToHeal []int
for _, endpoint := range localDisksToHeal {
// Load the new format of this passed endpoint
_, format, err := connectEndpoint(endpoint)
if err != nil {
logger.LogIf(ctx, err)
continue
}
// Calculate the set index where the current endpoint belongs
setIndex, _, err := findDiskIndex(sets.format, format)
if err != nil {
logger.LogIf(ctx, err)
continue
}
for i, localDisksToHeal := range localDisksInZoneHeal {
var erasureSetToHeal []int
for _, endpoint := range localDisksToHeal {
// Load the new format of this passed endpoint
_, format, err := connectEndpoint(endpoint)
if err != nil {
logger.LogIf(ctx, err)
continue
}
// Calculate the set index where the current endpoint belongs
setIndex, _, err := findDiskIndex(z.zones[i].format, format)
if err != nil {
logger.LogIf(ctx, err)
continue
}
erasureSetToHeal = append(erasureSetToHeal, setIndex)
erasureSetToHeal = append(erasureSetToHeal, setIndex)
}
erasureSetInZoneToHeal[i] = erasureSetToHeal
}
// Heal all erasure sets that need
for _, setIndex := range erasureSetToHeal {
xlObj := sets.sets[setIndex]
err := healErasureSet(ctx, setIndex, xlObj)
if err != nil {
logger.LogIf(ctx, err)
for i, erasureSetToHeal := range erasureSetInZoneToHeal {
for _, setIndex := range erasureSetToHeal {
err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex])
if err != nil {
logger.LogIf(ctx, err)
}
}
}
}

22
cmd/bucket-handlers.go
View file

@ -383,12 +383,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
deleteObjectsFn = api.CacheAPI().DeleteObjects
}
type delObj struct {
origIndex int
name string
}
var objectsToDelete []delObj
var objectsToDelete = map[string]int{}
var dErrs = make([]APIErrorCode, len(deleteObjects.Objects))
for index, object := range deleteObjects.Objects {
@ -400,13 +395,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
continue
}
objectsToDelete = append(objectsToDelete, delObj{index, object.ObjectName})
// Avoid duplicate objects, we use map to filter them out.
if _, ok := objectsToDelete[object.ObjectName]; !ok {
objectsToDelete[object.ObjectName] = index
}
}
toNames := func(input []delObj) (output []string) {
toNames := func(input map[string]int) (output []string) {
output = make([]string, len(input))
for i := range input {
output[i] = input[i].name
for name, index := range input {
output[index] = name
}
return
}
@ -417,8 +415,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
return
}
for i, obj := range objectsToDelete {
dErrs[obj.origIndex] = toAPIErrorCode(ctx, errs[i])
for _, index := range objectsToDelete {
dErrs[index] = toAPIErrorCode(ctx, errs[index])
}
// Collect deleted objects and errors if any.

6
cmd/bucket-handlers_test.go
View file

@ -331,7 +331,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
shouldPass: false,
},
// Test case -3.
// Setting invalid delimiter, expecting the HTTP response status to be http.StatusNotImplemented.
// Delimiter unsupported, but response is empty.
{
bucket: bucketName,
prefix: "",
@ -341,8 +341,8 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
maxUploads: "0",
accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey,
expectedRespStatus: http.StatusNotImplemented,
shouldPass: false,
expectedRespStatus: http.StatusOK,
shouldPass: true,
},
// Test case - 4.
// Setting Invalid prefix and marker combination.

14
cmd/config-encrypted.go
View file

@ -44,20 +44,6 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
var encrypted bool
var err error
// Construct path to config/transaction.lock for locking
transactionConfigPrefix := minioConfigPrefix + "/transaction.lock"
// Make sure to hold lock for entire migration to avoid
// such that only one server should migrate the entire config
// at a given time, this big transaction lock ensures this
// appropriately. This is also true for rotation of encrypted
// content.
objLock := objAPI.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
if err := objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.Unlock()
// Migrating Config backend needs a retry mechanism for
// the following reasons:
// - Read quorum is lost just after the initialization

13
cmd/config.go
View file

@ -287,19 +287,6 @@ func initConfig(objAPI ObjectLayer) error {
}
}
// Construct path to config/transaction.lock for locking
transactionConfigPrefix := minioConfigPrefix + "/transaction.lock"
// Hold lock only by one server and let that server alone migrate
// all the config as necessary, this is to ensure that
// redundant locks are not held for each migration - this allows
// for a more predictable behavior while debugging.
objLock := objAPI.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
if err := objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.Unlock()
// Migrates ${HOME}/.minio/config.json or config.json.deprecated
// to '<export_path>/.minio.sys/config/config.json'
// ignore if the file doesn't exist.

4
cmd/consolelogger.go
View file

@ -44,8 +44,8 @@ type HTTPConsoleLoggerSys struct {
// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
// the console logging pub sub system
func NewConsoleLogger(ctx context.Context, endpoints EndpointList) *HTTPConsoleLoggerSys {
host, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
func NewConsoleLogger(ctx context.Context, endpointZones EndpointZones) *HTTPConsoleLoggerSys {
host, err := xnet.ParseHost(GetLocalPeer(endpointZones))
if err != nil {
logger.FatalIf(err, "Unable to start console logging subsystem")
}

102
cmd/endpoint-ellipses.go
View file

@ -58,31 +58,20 @@ func getDivisibleSize(totalSizes []uint64) (result uint64) {
return result
}
// isValidSetSize - checks whether given count is a valid set size for erasure coding.
var isValidSetSize = func(count uint64) bool {
return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1] && count%2 == 0)
}
// getSetIndexes returns list of indexes which provides the set size
// on each index, this function also determines the final set size
// The final set size has the affinity towards choosing smaller
// indexes (total sets)
func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, err error) {
func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint64) (setIndexes [][]uint64, err error) {
if len(totalSizes) == 0 || len(args) == 0 {
return nil, errInvalidArgument
}
// isValidSetSize - checks whether given count is a valid set size for erasure coding.
isValidSetSize := func(count uint64) bool {
return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1] && count%2 == 0)
}
var customSetDriveCount uint64
if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
customSetDriveCount, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, config.ErrInvalidErasureSetSize(err)
}
if !isValidSetSize(customSetDriveCount) {
return nil, config.ErrInvalidErasureSetSize(nil)
}
}
setIndexes = make([][]uint64, len(totalSizes))
for _, totalSize := range totalSizes {
// Check if totalSize has minimum range upto setSize
@ -189,7 +178,7 @@ func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 {
// Parses all arguments and returns an endpointSet which is a collection
// of endpoints following the ellipses pattern, this is what is used
// by the object layer for initializing itself.
func parseEndpointSet(args ...string) (ep endpointSet, err error) {
func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSet, err error) {
var argPatterns = make([]ellipses.ArgPattern, len(args))
for i, arg := range args {
patterns, perr := ellipses.FindEllipsesPatterns(arg)
@ -199,7 +188,7 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
argPatterns[i] = patterns
}
ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns), customSetDriveCount)
if err != nil {
return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
}
@ -215,8 +204,15 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
// For example: {1...64} is divided into 4 sets each of size 16.
// This applies to even distributed setup syntax as well.
func GetAllSets(args ...string) ([][]string, error) {
if len(args) == 0 {
return nil, errInvalidArgument
var customSetDriveCount uint64
if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
customSetDriveCount, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, config.ErrInvalidErasureSetSize(err)
}
if !isValidSetSize(customSetDriveCount) {
return nil, config.ErrInvalidErasureSetSize(nil)
}
}
var setArgs [][]string
@ -225,7 +221,7 @@ func GetAllSets(args ...string) ([][]string, error) {
// Check if we have more one args.
if len(args) > 1 {
var err error
setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))})
setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))}, customSetDriveCount)
if err != nil {
return nil, err
}
@ -239,7 +235,7 @@ func GetAllSets(args ...string) ([][]string, error) {
}
setArgs = s.Get()
} else {
s, err := parseEndpointSet(args...)
s, err := parseEndpointSet(customSetDriveCount, args...)
if err != nil {
return nil, err
}
@ -261,18 +257,60 @@ func GetAllSets(args ...string) ([][]string, error) {
// CreateServerEndpoints - validates and creates new endpoints from input args, supports
// both ellipses and without ellipses transparently.
func createServerEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, int, int, error) {
setArgs, err := GetAllSets(args...)
if err != nil {
return serverAddr, nil, -1, 0, 0, err
func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, SetupType, error) {
if len(args) == 0 {
return nil, -1, errInvalidArgument
}
var endpoints EndpointList
var endpointZones EndpointZones
var setupType SetupType
serverAddr, endpoints, setupType, err = CreateEndpoints(serverAddr, setArgs...)
if err != nil {
return serverAddr, nil, -1, 0, 0, err
if !ellipses.HasEllipses(args...) {
setArgs, err := GetAllSets(args...)
if err != nil {
return nil, -1, err
}
endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
if err != nil {
return nil, -1, err
}
endpointZones = append(endpointZones, ZoneEndpoints{
SetCount: len(setArgs),
DrivesPerSet: len(setArgs[0]),
Endpoints: endpointList,
})
globalXLSetDriveCount = len(setArgs[0])
setupType = newSetupType
return endpointZones, setupType, nil
}
return serverAddr, endpoints, setupType, len(setArgs), len(setArgs[0]), nil
// Look for duplicate args.
if _, err := GetAllSets(args...); err != nil {
return nil, -1, err
}
for _, arg := range args {
setArgs, err := GetAllSets(arg)
if err != nil {
return nil, -1, err
}
endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
if err != nil {
return nil, -1, err
}
if setupType != 0 && setupType != newSetupType {
return nil, -1, fmt.Errorf("Mixed modes of operation %s and %s are not allowed",
setupType, newSetupType)
}
if globalXLSetDriveCount != 0 && globalXLSetDriveCount != len(setArgs[0]) {
return nil, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d",
globalXLSetDriveCount, len(setArgs[0]))
}
endpointZones = append(endpointZones, ZoneEndpoints{
SetCount: len(setArgs),
DrivesPerSet: len(setArgs[0]),
Endpoints: endpointList,
})
globalXLSetDriveCount = len(setArgs[0])
setupType = newSetupType
}
return endpointZones, setupType, nil
}

44
cmd/endpoint-ellipses_test.go
View file

@ -18,7 +18,6 @@ package cmd
import (
"fmt"
"os"
"reflect"
"testing"
@ -55,7 +54,7 @@ func TestCreateServerEndpoints(t *testing.T) {
}
for i, testCase := range testCases {
_, _, _, _, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
_, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
if err != nil && testCase.success {
t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
}
@ -74,8 +73,10 @@ func TestGetDivisibleSize(t *testing.T) {
{[]uint64{8, 8, 8}, 8},
{[]uint64{24}, 24},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
gotGCD := getDivisibleSize(testCase.totalSizes)
if testCase.result != gotGCD {
t.Errorf("Expected %v, got %v", testCase.result, gotGCD)
@ -90,45 +91,43 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
args []string
totalSizes []uint64
indexes [][]uint64
envOverride string
envOverride uint64
success bool
}{
{
[]string{"data{1...64}"},
[]uint64{64},
[][]uint64{{8, 8, 8, 8, 8, 8, 8, 8}},
"8",
8,
true,
},
{
[]string{"data{1...60}"},
nil,
nil,
"8",
8,
false,
},
{
[]string{"data{1...64}"},
nil,
nil,
"-1",
64,
false,
},
{
[]string{"data{1...64}"},
nil,
nil,
"2",
2,
false,
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
if err := os.Setenv("MINIO_ERASURE_SET_DRIVE_COUNT", testCase.envOverride); err != nil {
t.Fatal(err)
}
gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes)
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, testCase.envOverride)
if err != nil && testCase.success {
t.Errorf("Expected success but failed instead %s", err)
}
@ -138,7 +137,6 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
if !reflect.DeepEqual(testCase.indexes, gotIndexes) {
t.Errorf("Expected %v, got %v", testCase.indexes, gotIndexes)
}
os.Unsetenv("MINIO_ERASURE_SET_DRIVE_COUNT")
})
}
}
@ -209,9 +207,10 @@ func TestGetSetIndexes(t *testing.T) {
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes)
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, 0)
if err != nil && testCase.success {
t.Errorf("Expected success but failed instead %s", err)
}
@ -530,9 +529,10 @@ func TestParseEndpointSet(t *testing.T) {
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
gotEs, err := parseEndpointSet(testCase.arg)
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
gotEs, err := parseEndpointSet(0, testCase.arg)
if err != nil && testCase.success {
t.Errorf("Expected success but failed instead %s", err)
}

242
cmd/endpoint.go
View file

@ -55,7 +55,6 @@ type Endpoint struct {
*url.URL
IsLocal bool
SetIndex int
HostName string
}
func (endpoint Endpoint) String() string {
@ -75,15 +74,15 @@ func (endpoint Endpoint) Type() EndpointType {
return URLEndpointType
}
// IsHTTPS - returns true if secure for URLEndpointType.
func (endpoint Endpoint) IsHTTPS() bool {
// HTTPS - returns true if secure for URLEndpointType.
func (endpoint Endpoint) HTTPS() bool {
return endpoint.Scheme == "https"
}
// UpdateIsLocal - resolves the host and updates if it is local or not.
func (endpoint *Endpoint) UpdateIsLocal() error {
if !endpoint.IsLocal {
isLocal, err := isLocalHost(endpoint.HostName)
isLocal, err := isLocalHost(endpoint.Hostname())
if err != nil {
return err
}
@ -181,35 +180,46 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
}
return Endpoint{
URL: u,
IsLocal: isLocal,
HostName: host,
URL: u,
IsLocal: isLocal,
}, nil
}
// EndpointList - list of same type of endpoint.
type EndpointList []Endpoint
// Nodes - returns number of unique servers.
func (endpoints EndpointList) Nodes() int {
uniqueNodes := set.NewStringSet()
for _, endpoint := range endpoints {
if uniqueNodes.Contains(endpoint.Host) {
continue
}
uniqueNodes.Add(endpoint.Host)
}
return len(uniqueNodes)
// ZoneEndpoints represent endpoints in a given zone
// along with its setCount and drivesPerSet.
type ZoneEndpoints struct {
SetCount int
DrivesPerSet int
Endpoints Endpoints
}
// IsHTTPS - returns true if secure for URLEndpointType.
func (endpoints EndpointList) IsHTTPS() bool {
return endpoints[0].IsHTTPS()
// EndpointZones - list of list of endpoints
type EndpointZones []ZoneEndpoints
// HTTPS - returns true if secure for URLEndpointType.
func (l EndpointZones) HTTPS() bool {
return l[0].Endpoints.HTTPS()
}
// Nodes - returns all nodes count
func (l EndpointZones) Nodes() (count int) {
for _, ep := range l {
count += len(ep.Endpoints)
}
return count
}
// Endpoints - list of same type of endpoint.
type Endpoints []Endpoint
// HTTPS - returns true if secure for URLEndpointType.
func (endpoints Endpoints) HTTPS() bool {
return endpoints[0].HTTPS()
}
// GetString - returns endpoint string of i-th endpoint (0-based),
// and empty string for invalid indexes.
func (endpoints EndpointList) GetString(i int) string {
func (endpoints Endpoints) GetString(i int) string {
if i < 0 || i >= len(endpoints) {
return ""
}
@ -217,7 +227,7 @@ func (endpoints EndpointList) GetString(i int) string {
}
// UpdateIsLocal - resolves the host and discovers the local host.
func (endpoints EndpointList) UpdateIsLocal() error {
func (endpoints Endpoints) UpdateIsLocal() error {
var epsResolved int
var foundLocal bool
resolvedList := make([]bool, len(endpoints))
@ -246,7 +256,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
// return err if not Docker or Kubernetes
// We use IsDocker() to check for Docker environment
// We use IsKubernetes() to check for Kubernetes environment
isLocal, err := isLocalHost(endpoints[i].HostName)
isLocal, err := isLocalHost(endpoints[i].Hostname())
if err != nil {
if !IsDocker() && !IsKubernetes() {
return err
@ -256,8 +266,10 @@ func (endpoints EndpointList) UpdateIsLocal() error {
// log error only if more than 1s elapsed
if timeElapsed > time.Second {
// Log the message to console about the host not being resolveable.
reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName)
reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].Hostname())
reqInfo.AppendTags("elapsedTime",
humanize.RelTime(startTime, startTime.Add(timeElapsed),
"elapsed", ""))
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err, logger.Application)
}
@ -301,8 +313,8 @@ func (endpoints EndpointList) UpdateIsLocal() error {
return nil
}
// NewEndpointList - returns new endpoint list based on input args.
func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
// NewEndpoints - returns new endpoint list based on input args.
func NewEndpoints(args ...string) (endpoints Endpoints, err error) {
var endpointType EndpointType
var scheme string
@ -335,28 +347,30 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
return endpoints, nil
}
func checkEndpointsSubOptimal(ctx *cli.Context, setupType SetupType, endpoints EndpointList) (err error) {
func checkEndpointsSubOptimal(ctx *cli.Context, setupType SetupType, endpointZones EndpointZones) (err error) {
// Validate sub optimal ordering only for distributed setup.
if setupType != DistXLSetupType {
return nil
}
var endpointOrder int
err = fmt.Errorf("Too many disk args are local, input is in sub-optimal order. Please review input args: %s", ctx.Args())
for _, endpoint := range endpoints {
if endpoint.IsLocal {
endpointOrder++
} else {
endpointOrder--
}
if endpointOrder >= 2 {
return err
for _, endpoints := range endpointZones {
for _, endpoint := range endpoints.Endpoints {
if endpoint.IsLocal {
endpointOrder++
} else {
endpointOrder--
}
if endpointOrder >= 2 {
return err
}
}
}
return nil
}
// Checks if there are any cross device mounts.
func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
func checkCrossDeviceMounts(endpoints Endpoints) (err error) {
var absPaths []string
for _, endpoint := range endpoints {
if endpoint.IsLocal {
@ -372,14 +386,14 @@ func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
}
// CreateEndpoints - validates and creates new endpoints for given args.
func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, SetupType, error) {
var endpoints EndpointList
func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, error) {
var endpoints Endpoints
var setupType SetupType
var err error
// Check whether serverAddr is valid for this host.
if err = CheckLocalServerAddr(serverAddr); err != nil {
return serverAddr, endpoints, setupType, err
return endpoints, setupType, err
}
_, serverAddrPort := mustSplitHostPort(serverAddr)
@ -389,36 +403,36 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
var endpoint Endpoint
endpoint, err = NewEndpoint(args[0][0])
if err != nil {
return serverAddr, endpoints, setupType, err
return endpoints, setupType, err
}
if err := endpoint.UpdateIsLocal(); err != nil {
return serverAddr, endpoints, setupType, err
return endpoints, setupType, err
}
if endpoint.Type() != PathEndpointType {
return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
}
endpoints = append(endpoints, endpoint)
setupType = FSSetupType
// Check for cross device mounts if any.
if err = checkCrossDeviceMounts(endpoints); err != nil {
return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
}
return serverAddr, endpoints, setupType, nil
return endpoints, setupType, nil
}
for i, iargs := range args {
var newEndpoints EndpointList
// Convert args to endpoints
var eps EndpointList
eps, err = NewEndpointList(iargs...)
var newEndpoints Endpoints
eps, err := NewEndpoints(iargs...)
if err != nil {
return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
}
// Check for cross device mounts if any.
if err = checkCrossDeviceMounts(eps); err != nil {
return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
}
for _, ep := range eps {
@ -431,54 +445,44 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
// Return XL setup when all endpoints are path style.
if endpoints[0].Type() == PathEndpointType {
setupType = XLSetupType
return serverAddr, endpoints, setupType, nil
return endpoints, setupType, nil
}
if err := endpoints.UpdateIsLocal(); err != nil {
return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
if err = endpoints.UpdateIsLocal(); err != nil {
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
}
// Here all endpoints are URL style.
endpointPathSet := set.NewStringSet()
localEndpointCount := 0
localServerAddrSet := set.NewStringSet()
localServerHostSet := set.NewStringSet()
localPortSet := set.NewStringSet()
for _, endpoint := range endpoints {
endpointPathSet.Add(endpoint.Path)
if endpoint.IsLocal {
localServerAddrSet.Add(endpoint.Host)
localServerHostSet.Add(endpoint.Hostname())
var port string
_, port, err = net.SplitHostPort(endpoint.Host)
if err != nil {
port = serverAddrPort
}
localPortSet.Add(port)
localEndpointCount++
}
}
// No local endpoint found.
if localEndpointCount == 0 {
return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
}
// Check whether same path is not used in endpoints of a host on different port.
{
pathIPMap := make(map[string]set.StringSet)
for _, endpoint := range endpoints {
var host string
host, _, err = net.SplitHostPort(endpoint.Host)
if err != nil {
host = endpoint.Host
}
host := endpoint.Hostname()
hostIPSet, _ := getHostIP(host)
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
if !IPSet.Intersection(hostIPSet).IsEmpty() {
return serverAddr, endpoints, setupType,
return endpoints, setupType,
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
}
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
@ -496,42 +500,25 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
continue
}
if localPathSet.Contains(endpoint.Path) {
return serverAddr, endpoints, setupType,
return endpoints, setupType,
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
}
localPathSet.Add(endpoint.Path)
}
}
// Check whether serverAddrPort matches at least in one of port used in local endpoints.
{
if !localPortSet.Contains(serverAddrPort) {
if len(localPortSet) > 1 {
return serverAddr, endpoints, setupType,
config.ErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
}
return serverAddr, endpoints, setupType,
config.ErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
}
}
// All endpoints are pointing to local host
if len(endpoints) == localEndpointCount {
// If all endpoints have same port number, then this is XL setup using URL style endpoints.
if len(localPortSet) == 1 {
if len(localServerAddrSet) > 1 {
// TODO: Even though all endpoints are local, the local host is referred by different IP/name.
// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
//
// In this case, we bind to 0.0.0.0 ie to all interfaces.
// The actual way to do is bind to only IPs in uniqueLocalHosts.
serverAddr = net.JoinHostPort("", serverAddrPort)
if len(localServerHostSet) > 1 {
return endpoints, setupType,
config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
}
endpointPaths := endpointPathSet.ToSlice()
endpoints, _ = NewEndpointList(endpointPaths...)
endpoints, _ := NewEndpoints(endpointPaths...)
setupType = XLSetupType
return serverAddr, endpoints, setupType, nil
return endpoints, setupType, nil
}
// Even though all endpoints are local, but those endpoints use different ports.
@ -539,24 +526,20 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
} else {
// This is DistXL setup.
// Check whether local server address are not 127.x.x.x
for _, localServerAddr := range localServerAddrSet.ToSlice() {
host, _, err := net.SplitHostPort(localServerAddr)
if err != nil {
host = localServerAddr
}
ipList, err := getHostIP(host)
logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
for _, localHost := range localServerHostSet.ToSlice() {
ipList, err := getHostIP(localHost)
logger.FatalIf(err, "unexpected error when resolving host '%s'", localHost)
// Filter ipList by IPs those start with '127.' or '::1'
loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
return strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
return net.ParseIP(ip).IsLoopback()
}, "")
// If loop back IP is found and ipList contains only loop back IPs, then error out.
if len(loopBackIPs) > 0 && len(loopBackIPs) == len(ipList) {
err = fmt.Errorf("'%s' resolves to loopback address is not allowed for distributed XL", localServerAddr)
return serverAddr, endpoints, setupType, err
err = fmt.Errorf("'%s' resolves to loopback address is not allowed for distributed XL",
localHost)
return endpoints, setupType, err
}
}
}
@ -580,7 +563,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
// Error out if we have less than 2 unique servers.
if len(uniqueArgs.ToSlice()) < 2 && setupType == DistXLSetupType {
err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)
return serverAddr, endpoints, setupType, err
return endpoints, setupType, err
}
publicIPs := env.Get(config.EnvPublicIPs, "")
@ -589,7 +572,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
}
setupType = DistXLSetupType
return serverAddr, endpoints, setupType, nil
return endpoints, setupType, nil
}
// GetLocalPeer - returns local peer value, returns globalMinioAddr
@ -597,14 +580,16 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
// the first element from the set of peers which indicate that
// they are local. There is always one entry that is local
// even with repeated server endpoints.
func GetLocalPeer(endpoints EndpointList) (localPeer string) {
func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
peerSet := set.NewStringSet()
for _, endpoint := range endpoints {
if endpoint.Type() != URLEndpointType {
continue
}
if endpoint.IsLocal && endpoint.Host != "" {
peerSet.Add(endpoint.Host)
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if endpoint.Type() != URLEndpointType {
continue
}
if endpoint.IsLocal && endpoint.Host != "" {
peerSet.Add(endpoint.Host)
}
}
}
if peerSet.IsEmpty() {
@ -620,23 +605,24 @@ func GetLocalPeer(endpoints EndpointList) (localPeer string) {
}
// GetRemotePeers - get hosts information other than this minio service.
func GetRemotePeers(endpoints EndpointList) []string {
func GetRemotePeers(endpointZones EndpointZones) []string {
peerSet := set.NewStringSet()
for _, endpoint := range endpoints {
if endpoint.Type() != URLEndpointType {
continue
}
peer := endpoint.Host
if endpoint.IsLocal {
if _, port := mustSplitHostPort(peer); port == globalMinioPort {
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if endpoint.Type() != URLEndpointType {
continue
}
peer := endpoint.Host
if endpoint.IsLocal {
if _, port := mustSplitHostPort(peer); port == globalMinioPort {
continue
}
}
peerSet.Add(peer)
}
peerSet.Add(peer)
}
return peerSet.ToSlice()
}
@ -664,6 +650,10 @@ func updateDomainIPs(endPoints set.StringSet) {
ipList = ipList.Union(IPsWithPort)
}
globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
return !(strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1") || strings.HasPrefix(ip, "[::1]"))
host, _, err := net.SplitHostPort(ip)
if err != nil {
host = ip
}
return !net.ParseIP(host).IsLoopback()
}, "")
}

170
cmd/endpoint_test.go
View file

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2017 MinIO, Inc.
* MinIO Cloud Storage, (C) 2017,2018,2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,6 +19,7 @@ package cmd
import (
"flag"
"fmt"
"net"
"net/url"
"reflect"
"strings"
@ -35,19 +36,19 @@ func TestSubOptimalEndpointInput(t *testing.T) {
tests := []struct {
setupType SetupType
ctx *cli.Context
endpoints EndpointList
endpoints EndpointZones
isErr bool
}{
{
setupType: DistXLSetupType,
ctx: cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
endpoints: mustGetNewEndpointList(args1...),
endpoints: mustGetZoneEndpoints(args1...),
isErr: false,
},
{
setupType: DistXLSetupType,
ctx: cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
endpoints: mustGetNewEndpointList(args2...),
endpoints: mustGetZoneEndpoints(args2...),
isErr: false,
},
}
@ -90,11 +91,11 @@ func TestNewEndpoint(t *testing.T) {
{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
{"http://localhost/path", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
{"https://example.org/path", Endpoint{URL: u2, IsLocal: false, HostName: "example.org"}, URLEndpointType, nil},
{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true, HostName: "127.0.0.1"}, URLEndpointType, nil},
{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false, HostName: "192.168.253.200"}, URLEndpointType, nil},
{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
{"https://example.org/path", Endpoint{URL: u2, IsLocal: false}, URLEndpointType, nil},
{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false}, URLEndpointType, nil},
{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
{SlashSeparator, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
@ -136,7 +137,7 @@ func TestNewEndpoint(t *testing.T) {
}
}
func TestNewEndpointList(t *testing.T) {
func TestNewEndpoints(t *testing.T) {
testCases := []struct {
args []string
expectedErr error
@ -159,7 +160,7 @@ func TestNewEndpointList(t *testing.T) {
}
for _, testCase := range testCases {
_, err := NewEndpointList(testCase.args...)
_, err := NewEndpoints(testCase.args...)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
@ -175,7 +176,7 @@ func TestNewEndpointList(t *testing.T) {
func TestCreateEndpoints(t *testing.T) {
// Filter ipList by IPs those do not start with '127.'.
nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
return !strings.HasPrefix(ip, "127.")
return !net.ParseIP(ip).IsLoopback()
}, "")
if len(nonLoopBackIPs) == 0 {
t.Fatalf("No non-loop back IP address found for this host")
@ -257,120 +258,111 @@ func TestCreateEndpoints(t *testing.T) {
serverAddr string
args [][]string
expectedServerAddr string
expectedEndpoints EndpointList
expectedEndpoints Endpoints
expectedSetupType SetupType
expectedErr error
}{
{"localhost", [][]string{}, "", EndpointList{}, -1, fmt.Errorf("address localhost: missing port in address")},
{"localhost", [][]string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")},
// FS Setup
{"localhost:9000", [][]string{{"http://localhost/d1"}}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
{":443", [][]string{{"d1"}}, ":443", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{"./d1"}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{`\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{`.\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
{"localhost:9000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
{"localhost:10000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},
{"localhost:9000", [][]string{{"http://localhost/d1"}}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
{":443", [][]string{{"d1"}}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{"./d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{`\d1`}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", [][]string{{`.\d1`}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
// XL Setup with PathEndpointType
{":1234", [][]string{{"/d1", "/d2", "d3", "d4"}}, ":1234",
EndpointList{
Endpoints{
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "d3"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "d4"}, IsLocal: true},
}, XLSetupType, nil},
// XL Setup with URLEndpointType
{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", EndpointList{
{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
}, XLSetupType, nil},
// XL Setup with URLEndpointType having mixed naming to local host.
{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, ":10000", EndpointList{
{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, ":10000", Endpoints{
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
}, XLSetupType, nil},
{":9001", [][]string{{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}}, "", EndpointList{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
}, XLSetupType, fmt.Errorf("all local endpoints should not have different hostname/ips")},
{":9001", [][]string{{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
{":9000", [][]string{{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}}, "", EndpointList{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
{":9000", [][]string{{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}}, "", EndpointList{}, -1, fmt.Errorf("'localhost' resolves to loopback address is not allowed for distributed XL")},
{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}}, "", Endpoints{}, -1, fmt.Errorf("'localhost' resolves to loopback address is not allowed for distributed XL")},
// DistXL type
{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0], HostName: nonLoopBackIP},
Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1], HostName: nonLoopBackIP},
Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2], HostName: "example.org"},
Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3], HostName: "example.com"},
{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
}, DistXLSetupType, nil},
{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0], HostName: nonLoopBackIP},
Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1], HostName: nonLoopBackIP},
Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2], HostName: "example.org"},
Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3], HostName: "example.com"},
{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
}, DistXLSetupType, nil},
{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", EndpointList{
Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0], HostName: nonLoopBackIP},
Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1], HostName: "example.org"},
Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2], HostName: "example.com"},
Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3], HostName: "example.net"},
{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", Endpoints{
Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
}, DistXLSetupType, nil},
{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", EndpointList{
Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0], HostName: nonLoopBackIP},
Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1], HostName: "example.org"},
Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2], HostName: "example.com"},
Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3], HostName: "example.net"},
{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", Endpoints{
Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
}, DistXLSetupType, nil},
{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", EndpointList{
Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0], HostName: nonLoopBackIP},
Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1], HostName: nonLoopBackIP},
Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2], HostName: nonLoopBackIP},
Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3], HostName: nonLoopBackIP},
{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", Endpoints{
Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
}, DistXLSetupType, nil},
// DistXL Setup using only local host.
{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", EndpointList{
Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0], HostName: "localhost"},
Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1], HostName: "localhost"},
Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2], HostName: "127.0.0.1"},
Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3], HostName: nonLoopBackIP},
{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", Endpoints{
Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
}, DistXLSetupType, nil},
}
for i, testCase := range testCases {
for _, testCase := range testCases {
testCase := testCase
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
t.Run("", func(t *testing.T) {
endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
if err == nil && testCase.expectedErr != nil {
t.Errorf("error: expected = %v, got = <nil>", testCase.expectedErr)
}
if err == nil {
if testCase.expectedErr != nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else {
if serverAddr != testCase.expectedServerAddr {
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
}
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
}
if setupType != testCase.expectedSetupType {
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
}
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
t.Errorf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
}
} else if testCase.expectedErr == nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
} else if err.Error() != testCase.expectedErr.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
if setupType != testCase.expectedSetupType {
t.Errorf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
}
}
if err != nil && testCase.expectedErr == nil {
t.Errorf("error: expected = <nil>, got = %v, testCase: %v", err, testCase)
}
})
}
@ -403,13 +395,13 @@ func TestGetLocalPeer(t *testing.T) {
}
for i, testCase := range testCases {
endpoints, _ := NewEndpointList(testCase.endpointArgs...)
if !endpoints[0].IsLocal {
if err := endpoints.UpdateIsLocal(); err != nil {
zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
if !zendpoints[0].Endpoints[0].IsLocal {
if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
}
remotePeer := GetLocalPeer(endpoints)
remotePeer := GetLocalPeer(zendpoints)
if remotePeer != testCase.expectedResult {
t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
}
@ -435,13 +427,13 @@ func TestGetRemotePeers(t *testing.T) {
}
for _, testCase := range testCases {
endpoints, _ := NewEndpointList(testCase.endpointArgs...)
if !endpoints[0].IsLocal {
if err := endpoints.UpdateIsLocal(); err != nil {
zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
if !zendpoints[0].Endpoints[0].IsLocal {
if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
}
remotePeers := GetRemotePeers(endpoints)
remotePeers := GetRemotePeers(zendpoints)
if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
}

8
cmd/format-xl.go
View file

@ -471,7 +471,7 @@ func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (stri
}
// formatXLFixDeploymentID - Add deployment id if it is not present.
func formatXLFixDeploymentID(endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
func formatXLFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
// Attempt to load all `format.json` from all disks.
var sErrs []error
formats, sErrs := loadFormatXLAll(storageDisks)
@ -515,7 +515,7 @@ func formatXLFixDeploymentID(endpoints EndpointList, storageDisks []StorageAPI,
}
// Update only the valid local disks which have not been updated before.
func formatXLFixLocalDeploymentID(endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) error {
func formatXLFixLocalDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatXLV3) error {
// If this server was down when the deploymentID was updated
// then we make sure that we update the local disks with the deploymentID.
for index, storageDisk := range storageDisks {
@ -655,7 +655,7 @@ func closeStorageDisks(storageDisks []StorageAPI) {
// Initialize storage disks for each endpoint.
// Errors are returned for each endpoint with matching index.
func initStorageDisksWithErrors(endpoints EndpointList) ([]StorageAPI, []error) {
func initStorageDisksWithErrors(endpoints Endpoints) ([]StorageAPI, []error) {
// Bootstrap disks.
storageDisks := make([]StorageAPI, len(endpoints))
g := errgroup.WithNErrs(len(endpoints))
@ -695,7 +695,7 @@ func formatXLV3ThisEmpty(formats []*formatXLV3) bool {
}
// fixFormatXLV3 - fix format XL configuration on all disks.
func fixFormatXLV3(storageDisks []StorageAPI, endpoints EndpointList, formats []*formatXLV3) error {
func fixFormatXLV3(storageDisks []StorageAPI, endpoints Endpoints, formats []*formatXLV3) error {
for i, format := range formats {
if format == nil || !endpoints[i].IsLocal {
continue

4
cmd/format-xl_test.go
View file

@ -83,7 +83,7 @@ func TestFixFormatV3(t *testing.T) {
for _, xlDir := range xlDirs {
defer os.RemoveAll(xlDir)
}
endpoints := mustGetNewEndpointList(xlDirs...)
endpoints := mustGetNewEndpoints(xlDirs...)
storageDisks, errs := initStorageDisksWithErrors(endpoints)
for _, err := range errs {
@ -593,7 +593,7 @@ func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
if err != nil {
b.Fatal(err)
}
endpoints := mustGetNewEndpointList(fsDirs...)
endpoints := mustGetNewEndpoints(fsDirs...)
b.RunParallel(func(pb *testing.PB) {
endpoints := endpoints
for pb.Next() {

42
cmd/global-heal.go
View file

@ -80,16 +80,6 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
// healErasureSet lists and heals all objects in a specific erasure set
func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
// Hold a lock for healing the erasure set
zeroDuration := time.Millisecond
zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
erasureSetHealLock := xlObj.nsMutex.NewNSLock(ctx, xlObj.getLockers(),
"system", fmt.Sprintf("erasure-set-heal-%d", setIndex))
if err := erasureSetHealLock.GetLock(zeroDynamicTimeout); err != nil {
return err
}
defer erasureSetHealLock.Unlock()
buckets, err := xlObj.ListBuckets(ctx)
if err != nil {
return err
@ -123,11 +113,11 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
}
// Healing leader will take the charge of healing all erasure sets
func execLeaderTasks(sets *xlSets) {
func execLeaderTasks(z *xlZones) {
ctx := context.Background()
// Hold a lock so only one server performs auto-healing
leaderLock := sets.NewNSLock(ctx, minioMetaBucket, "leader")
leaderLock := z.NewNSLock(ctx, minioMetaBucket, "leader")
for {
err := leaderLock.GetLock(leaderLockTimeout)
if err == nil {
@ -136,18 +126,30 @@ func execLeaderTasks(sets *xlSets) {
time.Sleep(leaderTick)
}
// Hold a lock for healing the erasure set
zeroDuration := time.Millisecond
zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
lastScanTime := time.Now() // So that we don't heal immediately, but after one month.
for {
if time.Since(lastScanTime) < healInterval {
time.Sleep(healTick)
continue
}
// Heal set by set
for i, set := range sets.sets {
err := healErasureSet(ctx, i, set)
if err != nil {
logger.LogIf(ctx, err)
continue
for _, zone := range z.zones {
// Heal set by set
for i, set := range zone.sets {
setLock := z.zones[0].NewNSLock(ctx, "system", fmt.Sprintf("erasure-set-heal-%d", i))
if err := setLock.GetLock(zeroDynamicTimeout); err != nil {
logger.LogIf(ctx, err)
continue
}
if err := healErasureSet(ctx, i, set); err != nil {
setLock.Unlock()
logger.LogIf(ctx, err)
continue
}
setLock.Unlock()
}
}
lastScanTime = time.Now()
@ -165,12 +167,12 @@ func startGlobalHeal() {
break
}
sets, ok := objAPI.(*xlSets)
zones, ok := objAPI.(*xlZones)
if !ok {
return
}
execLeaderTasks(sets)
execLeaderTasks(zones)
}
func initGlobalHeal() {

8
cmd/globals.go
View file

@ -103,9 +103,6 @@ var globalCLIContext = struct {
}{}
var (
// Indicates the total number of erasure coded sets configured.
globalXLSetCount int
// Indicates set drive count.
globalXLSetDriveCount int
@ -130,9 +127,6 @@ var (
// This flag is set to 'us-east-1' by default
globalServerRegion = globalMinioDefaultRegion
// Maximum size of internal objects parts
globalPutPartSize = int64(64 * 1024 * 1024)
// MinIO local server address (in `host:port` format)
globalMinioAddr = ""
// MinIO default port, can be changed through command line.
@ -173,7 +167,7 @@ var (
// registered listeners
globalConsoleSys *HTTPConsoleLoggerSys
globalEndpoints EndpointList
globalEndpoints EndpointZones
// Global server's network statistics
globalConnStats = newConnStats()

20
cmd/healthcheck-handler.go
View file

@ -77,26 +77,26 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
}
// For FS and Erasure backend, check if local disks are up.
var totalLocalDisks int
var erroredDisks int
for _, endpoint := range globalEndpoints {
// Check only if local disks are accessible, we do not have
// to reach to rest of the other servers in a distributed setup.
if endpoint.IsLocal {
totalLocalDisks++
for _, ep := range globalEndpoints {
for _, endpoint := range ep.Endpoints {
// Check only if local disks are accessible, we do not have
// to reach to rest of the other servers in a distributed setup.
if !endpoint.IsLocal {
continue
}
// Attempt a stat to backend, any error resulting
// from this Stat() operation is considered as backend
// is not available, count them as errors.
if _, err := os.Stat(endpoint.Path); err != nil {
if _, err := os.Stat(endpoint.Path); err != nil && os.IsNotExist(err) {
logger.LogIf(ctx, err)
erroredDisks++
}
}
}
// If all exported local disks have errored, we simply let kubernetes
// take us down.
if totalLocalDisks == erroredDisks {
// Any errored disks, we let orchestrators take us down.
if erroredDisks > 0 {
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
return
}

9
cmd/local-locker.go
View file

@ -65,6 +65,10 @@ func (d *errorLocker) Close() error {
return nil
}
func (d *errorLocker) IsOnline() bool {
return false
}
// localLocker implements Dsync.NetLocker
type localLocker struct {
mutex sync.Mutex
@ -193,6 +197,11 @@ func (l *localLocker) Close() error {
return nil
}
// Local locker is always online.
func (l *localLocker) IsOnline() bool {
return true
}
func newLocker(endpoint Endpoint) *localLocker {
return &localLocker{
endpoint: endpoint,

34
cmd/lock-rest-server.go
View file

@ -128,24 +128,26 @@ func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request)
}
// registerLockRESTHandlers - register lock rest router.
func registerLockRESTHandlers(router *mux.Router, endpoints EndpointList) {
func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
queries := restQueries(lockRESTUID, lockRESTSource, lockRESTResource)
for _, endpoint := range endpoints {
if !endpoint.IsLocal {
continue
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if !endpoint.IsLocal {
continue
}
lockServer := &lockRESTServer{
ll: newLocker(endpoint),
}
subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler)).Queries(queries...)
globalLockServers[endpoint] = lockServer.ll
}
lockServer := &lockRESTServer{
ll: newLocker(endpoint),
}
subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)
subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler)).Queries(queries...)
globalLockServers[endpoint] = lockServer.ll
}
// If none of the routes match add default error handler routes

12
cmd/metrics.go
View file

@ -90,11 +90,13 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
}
storageAPIs := []StorageAPI{}
for _, endpoint := range globalEndpoints {
if endpoint.IsLocal {
// Construct storageAPIs.
sAPI, _ := newStorageAPI(endpoint)
storageAPIs = append(storageAPIs, sAPI)
for _, ep := range globalEndpoints {
for _, endpoint := range ep.Endpoints {
if endpoint.IsLocal {
// Construct storageAPIs.
sAPI, _ := newStorageAPI(endpoint)
storageAPIs = append(storageAPIs, sAPI)
}
}
}

10
cmd/namespace-lock.go
View file

@ -223,14 +223,12 @@ type localLockInstance struct {
// NewNSLock - returns a lock instance for a given volume and
// path. The returned lockInstance object encapsulates the nsLockMap,
// volume, path and operation ID.
func (n *nsLockMap) NewNSLock(ctx context.Context, lockers []dsync.NetLocker, volume, path string) RWLocker {
func (n *nsLockMap) NewNSLock(ctx context.Context, lockersFn func() []dsync.NetLocker, volume, path string) RWLocker {
opsID := mustGetUUID()
if n.isDistXL {
sync, err := dsync.New(lockers)
if err != nil {
logger.CriticalIf(ctx, err)
}
return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), sync), volume, path, opsID}
return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), &dsync.Dsync{
GetLockersFn: lockersFn,
}), volume, path, opsID}
}
return &localLockInstance{ctx, n, volume, path, opsID}
}

7
cmd/net.go
View file

@ -120,10 +120,10 @@ func (n byLastOctetValue) Less(i, j int) bool {
// This case is needed when all ips in the list
// have same last octets, Following just ensures that
// 127.0.0.1 is moved to the end of the list.
if n[i].String() == "127.0.0.1" {
if n[i].IsLoopback() {
return false
}
if n[j].String() == "127.0.0.1" {
if n[j].IsLoopback() {
return true
}
return []byte(n[i].To4())[3] > []byte(n[j].To4())[3]
@ -171,7 +171,8 @@ func getAPIEndpoints() (apiEndpoints []string) {
}
for _, ip := range ipList {
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(ip, globalMinioPort)))
endpoint := fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(ip, globalMinioPort))
apiEndpoints = append(apiEndpoints, endpoint)
}
return apiEndpoints

34
cmd/notification.go
View file

@ -697,17 +697,6 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
// Construct path to listener.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig)
transactionConfigFile := configFile + "/transaction.lock"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid data race between readConfig()
// and saveConfig().
objLock := objAPI.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
if err := objLock.GetRLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.RUnlock()
configData, e := readConfig(ctx, objAPI, configFile)
if e != nil && !IsErrIgnored(e, errDiskNotFound, errConfigNotFound) {
return e
@ -1180,7 +1169,7 @@ func (sys *NotificationSys) NetworkInfo() []madmin.ServerNetworkHardwareInfo {
}
// NewNotificationSys - creates new notification system object.
func NewNotificationSys(endpoints EndpointList) *NotificationSys {
func NewNotificationSys(endpoints EndpointZones) *NotificationSys {
// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
return &NotificationSys{
targetList: event.NewTargetList(),
@ -1338,16 +1327,6 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
// Construct path to listener.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig)
transactionConfigFile := configFile + "/transaction.lock"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid data race between readConfig()
// and saveConfig().
objLock := objAPI.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
if err := objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.Unlock()
configData, err := readConfig(ctx, objAPI, configFile)
if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
@ -1389,17 +1368,6 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
// Construct path to listener.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig)
transactionConfigFile := configFile + "/transaction.lock"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid data race between readConfig()
// and saveConfig().
objLock := objAPI.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
if err := objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.Unlock()
configData, err := readConfig(ctx, objAPI, configFile)
if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
return err

10
cmd/object-api-datatypes.go
View file

@ -170,6 +170,16 @@ type ListPartsInfo struct {
EncodingType string // Not supported yet.
}
// Lookup - returns if uploadID is valid
func (lm ListMultipartsInfo) Lookup(uploadID string) bool {
for _, upload := range lm.Uploads {
if upload.UploadID == uploadID {
return true
}
}
return false
}
// ListMultipartsInfo - represnets bucket resources for incomplete multipart uploads.
type ListMultipartsInfo struct {
// Together with upload-id-marker, this parameter specifies the multipart upload

6
cmd/object-api-errors.go
View file

@ -427,6 +427,12 @@ func (e BackendDown) Error() string {
return "Backend down"
}
// isErrBucketNotFound - Check if error type is BucketNotFound.
func isErrBucketNotFound(err error) bool {
_, ok := err.(BucketNotFound)
return ok
}
// isErrObjectNotFound - Check if error type is ObjectNotFound.
func isErrObjectNotFound(err error) bool {
_, ok := err.(ObjectNotFound)

9
cmd/object-api-input-checks.go
View file

@ -74,15 +74,6 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker, delimiter st
Object: prefix,
}
}
// Verify if delimiter is anything other than '/', which we do not support.
if delimiter != "" && delimiter != SlashSeparator {
logger.LogIf(ctx, UnsupportedDelimiter{
Delimiter: delimiter,
})
return UnsupportedDelimiter{
Delimiter: delimiter,
}
}
// Verify if marker has prefix.
if marker != "" && !hasPrefix(marker, prefix) {
logger.LogIf(ctx, InvalidMarkerPrefixCombination{

16
cmd/object-api-multipart_test.go
View file

@ -1128,10 +1128,9 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
{"volatile-bucket-1", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
{"volatile-bucket-2", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
{"volatile-bucket-3", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
// Valid, existing bucket, but sending invalid delimeter values (Test number 8-9).
// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
{bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
{bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
// Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9).
{bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true},
{bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true},
// Testing for failure cases with both perfix and marker (Test number 10).
// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
{bucketNames[0], "asia", "europe-object", "", "", 0, ListMultipartsInfo{},
@ -1193,9 +1192,6 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
{bucketNames[1], "Asia", "", "", "", 10, listMultipartResults[23], nil, true},
// Test case with `Prefix` and `UploadIDMarker` (Test number 37).
{bucketNames[1], "min", "minio-object-1.txt", uploadIDs[1], "", 10, listMultipartResults[24], nil, true},
// Test case with `KeyMarker` and `UploadIDMarker` (Test number 38).
// {bucketNames[1], "", "minio-object-1.txt", uploadIDs[1], "", 10, listMultipartResults[24], nil, true},
// Test case for bucket with multiple objects in it.
// Bucket used : `bucketNames[2]`.
// Objects used: `objectNames[1-5]`.
@ -1217,16 +1213,10 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
// Since all available entries are listed, IsTruncated is expected to be false
// and NextMarkers are expected to empty.
{bucketNames[2], "", "", "", "", 6, listMultipartResults[31], nil, true},
// Test case with `uploadIDMarker` (Test number 46).
// {bucketNames[2], "", "", uploadIDs[6], "", 10, listMultipartResults[32], nil, true},
// Test case with `KeyMarker` (Test number 47).
{bucketNames[2], "", objectNames[3], "", "", 10, listMultipartResults[33], nil, true},
// Test case with `prefix` and `KeyMarker` (Test number 48).
{bucketNames[2], "minio-object", objectNames[1], "", "", 10, listMultipartResults[34], nil, true},
// Test case with `prefix` and `uploadIDMarker` (Test number 49).
// {bucketNames[2], globalMinioDefaultOwnerID, "", uploadIDs[4], "", 10, listMultipartResults[35], nil, true},
// Test case with `KeyMarker` and `uploadIDMarker` (Test number 50).
// {bucketNames[2], "minio-object.txt", "", uploadIDs[5], "", 10, listMultipartResults[36], nil, true},
}
for i, testCase := range testCases {

6
cmd/peer-rest-client.go
View file

@ -708,9 +708,9 @@ func (client *peerRESTClient) ConsoleLog(logCh chan interface{}, doneCh chan str
}()
}
func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
func getRemoteHosts(endpointZones EndpointZones) []*xnet.Host {
var remoteHosts []*xnet.Host
for _, hostStr := range GetRemotePeers(endpoints) {
for _, hostStr := range GetRemotePeers(endpointZones) {
host, err := xnet.ParseHost(hostStr)
if err != nil {
logger.LogIf(context.Background(), err)
@ -722,7 +722,7 @@ func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
return remoteHosts
}
func getRestClients(endpoints EndpointList) []*peerRESTClient {
func getRestClients(endpoints EndpointZones) []*peerRESTClient {
peerHosts := getRemoteHosts(endpoints)
restClients := make([]*peerRESTClient, len(peerHosts))
for i, host := range peerHosts {

10
cmd/prepare-storage.go
View file

@ -53,7 +53,7 @@ var printEndpointError = func() func(Endpoint, error) {
}()
// Migrates backend format of local disks.
func formatXLMigrateLocalEndpoints(endpoints EndpointList) error {
func formatXLMigrateLocalEndpoints(endpoints Endpoints) error {
g := errgroup.WithNErrs(len(endpoints))
for index, endpoint := range endpoints {
if !endpoint.IsLocal {
@ -81,7 +81,7 @@ func formatXLMigrateLocalEndpoints(endpoints EndpointList) error {
}
// Cleans up tmp directory of local disks.
func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
func formatXLCleanupTmpLocalEndpoints(endpoints Endpoints) error {
g := errgroup.WithNErrs(len(endpoints))
for index, endpoint := range endpoints {
if !endpoint.IsLocal {
@ -145,7 +145,7 @@ func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
}
// validate reference format against list of XL formats.
func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints EndpointList, setCount, drivesPerSet int) error {
func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints Endpoints, setCount, drivesPerSet int) error {
for i := range formats {
if formats[i] == nil {
continue
@ -174,7 +174,7 @@ var errXLV3ThisEmpty = fmt.Errorf("XL format version 3 has This field empty")
// connect to list of endpoints and load all XL disk formats, validate the formats are correct
// and are in quorum, if no formats are found attempt to initialize all of them for the first
// time. additionally make sure to close all the disks used in this attempt.
func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointList, setCount, drivesPerSet int) (*formatXLV3, error) {
func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, setCount, drivesPerSet int) (*formatXLV3, error) {
// Initialize all storage disks
storageDisks, errs := initStorageDisksWithErrors(endpoints)
defer closeStorageDisks(storageDisks)
@ -286,7 +286,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
}
// Format disks before initialization of object layer.
func waitForFormatXL(firstDisk bool, endpoints EndpointList, setCount, disksPerSet int) (format *formatXLV3, err error) {
func waitForFormatXL(firstDisk bool, endpoints Endpoints, setCount, disksPerSet int) (format *formatXLV3, err error) {
if len(endpoints) == 0 || setCount == 0 || disksPerSet == 0 {
return nil, errInvalidArgument
}

10
cmd/routers.go
View file

@ -23,15 +23,15 @@ import (
)
// Composed function registering routers for only distributed XL setup.
func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
func registerDistXLRouters(router *mux.Router, endpointZones EndpointZones) {
// Register storage rpc router only if its a distributed setup.
registerStorageRESTHandlers(router, endpoints)
registerStorageRESTHandlers(router, endpointZones)
// Register peer REST router only if its a distributed setup.
registerPeerRESTHandlers(router)
// Register distributed namespace lock.
registerLockRESTHandlers(router, endpoints)
registerLockRESTHandlers(router, endpointZones)
}
@ -79,14 +79,14 @@ var globalHandlers = []HandlerFunc{
}
// configureServer handler returns final handler for the http server.
func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
func configureServerHandler(endpointZones EndpointZones) (http.Handler, error) {
// Initialize router. `SkipClean(true)` stops gorilla/mux from
// normalizing URL path minio/minio#3256
router := mux.NewRouter().SkipClean(true)
// Initialize distributed NS lock.
if globalIsDistXL {
registerDistXLRouters(router, endpoints)
registerDistXLRouters(router, endpointZones)
}
// Add STS router always.

48
cmd/server-main.go
View file

@ -146,12 +146,13 @@ func serverHandleCmdArgs(ctx *cli.Context) {
endpoints := strings.Fields(env.Get(config.EnvEndpoints, ""))
if len(endpoints) > 0 {
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
globalEndpoints, setupType, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
} else {
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, ctx.Args()...)
globalEndpoints, setupType, err = createServerEndpoints(globalCLIContext.Addr, ctx.Args()...)
}
logger.FatalIf(err, "Invalid command line arguments")
globalMinioAddr = globalCLIContext.Addr
logger.LogIf(context.Background(), checkEndpointsSubOptimal(ctx, setupType, globalEndpoints))
globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
@ -192,7 +193,24 @@ func newAllSubsystems() {
}
func initSafeModeInit(buckets []BucketInfo) (err error) {
newObject := newObjectLayerWithoutSafeModeFn()
// Construct path to config/transaction.lock for locking
transactionConfigPrefix := minioConfigPrefix + "/transaction.lock"
// Make sure to hold lock for entire migration to avoid
// such that only one server should migrate the entire config
// at a given time, this big transaction lock ensures this
// appropriately. This is also true for rotation of encrypted
// content.
objLock := newObject.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
if err = objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer func() {
objLock.Unlock()
if err != nil {
var cerr config.Err
if errors.As(err, &cerr) {
@ -210,8 +228,6 @@ func initSafeModeInit(buckets []BucketInfo) (err error) {
}
}()
newObject := newObjectLayerWithoutSafeModeFn()
// Calls New() for all sub-systems.
newAllSubsystems()
@ -302,10 +318,10 @@ func serverMain(ctx *cli.Context) {
// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
if globalIsDistXL {
if globalEndpoints.IsHTTPS() && !globalIsSSL {
if globalEndpoints.HTTPS() && !globalIsSSL {
logger.Fatal(config.ErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server")
}
if !globalEndpoints.IsHTTPS() && globalIsSSL {
if !globalEndpoints.HTTPS() && globalIsSSL {
logger.Fatal(config.ErrCertsAndHTTPEndpoints(nil), "Unable to start the server")
}
}
@ -413,19 +429,21 @@ func serverMain(ctx *cli.Context) {
}
// Initialize object layer with the supplied disks, objectLayer is nil upon any error.
func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) {
func newObjectLayer(endpointZones EndpointZones) (newObject ObjectLayer, err error) {
// For FS only, directly use the disk.
isFS := len(endpoints) == 1
if isFS {
if endpointZones.Nodes() == 1 {
// Initialize new FS object layer.
return NewFSObjectLayer(endpoints[0].Path)
return NewFSObjectLayer(endpointZones[0].Endpoints[0].Path)
}
format, err := waitForFormatXL(endpoints[0].IsLocal, endpoints, globalXLSetCount, globalXLSetDriveCount)
if err != nil {
return nil, err
var formats []*formatXLV3
for _, ep := range endpointZones {
format, err := waitForFormatXL(ep.Endpoints[0].IsLocal, ep.Endpoints, ep.SetCount, ep.DrivesPerSet)
if err != nil {
return nil, err
}
formats = append(formats, format)
}
return newXLSets(endpoints, format, len(format.XL.Sets), len(format.XL.Sets[0]))
return newXLZones(endpointZones, formats)
}

11
cmd/server-main_test.go
View file

@ -31,8 +31,7 @@ func TestNewObjectLayer(t *testing.T) {
}
defer removeRoots(disks)
endpoints := mustGetNewEndpointList(disks...)
obj, err := newObjectLayer(endpoints)
obj, err := newObjectLayer(mustGetZoneEndpoints(disks...))
if err != nil {
t.Fatal("Unexpected object layer initialization error", err)
}
@ -51,16 +50,12 @@ func TestNewObjectLayer(t *testing.T) {
}
defer removeRoots(disks)
globalXLSetCount = 1
globalXLSetDriveCount = 16
endpoints = mustGetNewEndpointList(disks...)
obj, err = newObjectLayer(endpoints)
obj, err = newObjectLayer(mustGetZoneEndpoints(disks...))
if err != nil {
t.Fatal("Unexpected object layer initialization error", err)
}
_, ok = obj.(*xlSets)
_, ok = obj.(*xlZones)
if !ok {
t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj))
}

98
cmd/storage-rest-server.go
View file

@ -560,55 +560,57 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
}
// registerStorageRPCRouter - register storage rpc router.
func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
for _, endpoint := range endpoints {
if !endpoint.IsLocal {
continue
func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
for _, ep := range endpointZones {
for _, endpoint := range ep.Endpoints {
if !endpoint.IsLocal {
continue
}
storage, err := newPosix(endpoint.Path)
if err != nil {
logger.Fatal(config.ErrUnableToWriteInBackend(err),
"Unable to initialize posix backend")
}
server := &storageRESTServer{storage: storage}
subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(httpTraceHdrs(server.StatVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(httpTraceHdrs(server.DeleteVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(httpTraceHdrs(server.ListVolsHandler))
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodAppendFile).HandlerFunc(httpTraceHdrs(server.AppendFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteAll).HandlerFunc(httpTraceHdrs(server.WriteAllHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(httpTraceHdrs(server.CreateFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTLength)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatFile).HandlerFunc(httpTraceHdrs(server.StatFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadAll).HandlerFunc(httpTraceHdrs(server.ReadAllHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFile).HandlerFunc(httpTraceHdrs(server.ReadFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength, storageRESTBitrotAlgo, storageRESTBitrotHash)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(httpTraceHdrs(server.ReadFileStreamHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListDir).HandlerFunc(httpTraceHdrs(server.ListDirHandler)).
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount, storageRESTLeafFile)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalk).HandlerFunc(httpTraceHdrs(server.WalkHandler)).
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTMarkerPath, storageRESTRecursive, storageRESTLeafFile)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFileBulk).HandlerFunc(httpTraceHdrs(server.DeleteFileBulkHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(httpTraceHdrs(server.VerifyFile)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTBitrotAlgo, storageRESTBitrotHash, storageRESTLength, storageRESTShardSize)...)
}
storage, err := newPosix(endpoint.Path)
if err != nil {
logger.Fatal(config.ErrUnableToWriteInBackend(err),
"Unable to initialize posix backend")
}
server := &storageRESTServer{storage: storage}
subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(httpTraceHdrs(server.StatVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(httpTraceHdrs(server.DeleteVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(httpTraceHdrs(server.ListVolsHandler))
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodAppendFile).HandlerFunc(httpTraceHdrs(server.AppendFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteAll).HandlerFunc(httpTraceHdrs(server.WriteAllHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(httpTraceHdrs(server.CreateFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTLength)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatFile).HandlerFunc(httpTraceHdrs(server.StatFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadAll).HandlerFunc(httpTraceHdrs(server.ReadAllHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFile).HandlerFunc(httpTraceHdrs(server.ReadFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength, storageRESTBitrotAlgo, storageRESTBitrotHash)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(httpTraceHdrs(server.ReadFileStreamHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListDir).HandlerFunc(httpTraceHdrs(server.ListDirHandler)).
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount, storageRESTLeafFile)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalk).HandlerFunc(httpTraceHdrs(server.WalkHandler)).
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTMarkerPath, storageRESTRecursive, storageRESTLeafFile)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFileBulk).HandlerFunc(httpTraceHdrs(server.DeleteFileBulkHandler)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(httpTraceHdrs(server.VerifyFile)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTBitrotAlgo, storageRESTBitrotHash, storageRESTLength, storageRESTShardSize)...)
}
// If none of the routes match add default error handler routes

4
cmd/storage-rest_test.go
View file

@ -513,7 +513,9 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
t.Fatalf("UpdateIsLocal failed %v", err)
}
registerStorageRESTHandlers(router, EndpointList{endpoint})
registerStorageRESTHandlers(router, []ZoneEndpoints{{
Endpoints: Endpoints{endpoint},
}})
restClient := newStorageRESTClient(endpoint)
prevGlobalServerConfig := globalServerConfig
globalServerConfig = newServerConfig()

117
cmd/test-utils_test.go
View file

@ -59,8 +59,6 @@ import (
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/bpool"
"github.com/minio/minio/pkg/dsync"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/policy"
)
@ -181,13 +179,13 @@ func prepareXLSets32() (ObjectLayer, []string, error) {
return nil, nil, err
}
endpoints1 := mustGetNewEndpointList(fsDirs1...)
endpoints1 := mustGetNewEndpoints(fsDirs1...)
fsDirs2, err := getRandomDisks(16)
if err != nil {
removeRoots(fsDirs1)
return nil, nil, err
}
endpoints2 := mustGetNewEndpointList(fsDirs2...)
endpoints2 := mustGetNewEndpoints(fsDirs2...)
endpoints := append(endpoints1, endpoints2...)
fsDirs := append(fsDirs1, fsDirs2...)
@ -210,7 +208,7 @@ func prepareXL(nDisks int) (ObjectLayer, []string, error) {
if err != nil {
return nil, nil, err
}
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
obj, _, err := initObjectLayer(mustGetZoneEndpoints(fsDirs...))
if err != nil {
removeRoots(fsDirs)
return nil, nil, err
@ -302,7 +300,7 @@ func isSameType(obj1, obj2 interface{}) bool {
// defer s.Stop()
type TestServer struct {
Root string
Disks EndpointList
Disks EndpointZones
AccessKey string
SecretKey string
Server *httptest.Server
@ -331,9 +329,7 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
credentials := globalActiveCred
testServer.Obj = objLayer
for _, disk := range disks {
testServer.Disks = append(testServer.Disks, mustGetNewEndpointList(disk)...)
}
testServer.Disks = mustGetZoneEndpoints(disks...)
testServer.AccessKey = credentials.AccessKey
testServer.SecretKey = credentials.SecretKey
@ -450,7 +446,7 @@ func resetGlobalConfig() {
}
func resetGlobalEndpoints() {
globalEndpoints = EndpointList{}
globalEndpoints = EndpointZones{}
}
func resetGlobalIsXL() {
@ -525,8 +521,10 @@ func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
// Deleting the temporary backend and stopping the server.
func (testServer TestServer) Stop() {
os.RemoveAll(testServer.Root)
for _, disk := range testServer.Disks {
os.RemoveAll(disk.Path)
for _, ep := range testServer.Disks {
for _, disk := range ep.Endpoints {
os.RemoveAll(disk.Path)
}
}
testServer.Server.Close()
}
@ -1580,73 +1578,52 @@ func getRandomDisks(N int) ([]string, error) {
}
// Initialize object layer with the supplied disks, objectLayer is nil upon any error.
func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) {
func newTestObjectLayer(endpointZones EndpointZones) (newObject ObjectLayer, err error) {
// For FS only, directly use the disk.
isFS := len(endpoints) == 1
if isFS {
if endpointZones.Nodes() == 1 {
// Initialize new FS object layer.
return NewFSObjectLayer(endpoints[0].Path)
return NewFSObjectLayer(endpointZones[0].Endpoints[0].Path)
}
format, err := waitForFormatXL(endpoints[0].IsLocal, endpoints, 1, 16)
if err != nil {
return nil, err
}
storageDisks, errs := initStorageDisksWithErrors(endpoints)
for _, err = range errs {
if err != nil && err != errDiskNotFound {
var formats []*formatXLV3
for _, ep := range endpointZones {
format, err := waitForFormatXL(ep.Endpoints[0].IsLocal, ep.Endpoints, ep.SetCount, ep.DrivesPerSet)
if err != nil {
return nil, err
}
formats = append(formats, format)
}
for i, disk := range storageDisks {
disk.SetDiskID(format.XL.Sets[0][i])
}
// Initialize list pool.
listPool := NewTreeWalkPool(globalLookupTimeout)
// Initialize xl objects.
xl := &xlObjects{
listPool: listPool,
storageDisks: storageDisks,
nsMutex: newNSLock(false),
bp: bpool.NewBytePoolCap(4, blockSizeV1, blockSizeV1*2),
}
xl.getDisks = func() []StorageAPI {
return xl.storageDisks
}
xl.getLockers = func() []dsync.NetLocker {
return nil
zones, err := newXLZones(endpointZones, formats)
if err != nil {
return nil, err
}
globalConfigSys = NewConfigSys()
globalIAMSys = NewIAMSys()
globalIAMSys.Init(xl)
globalIAMSys.Init(zones)
globalPolicySys = NewPolicySys()
globalPolicySys.Init(nil, xl)
globalPolicySys.Init(nil, zones)
globalNotificationSys = NewNotificationSys(endpoints)
globalNotificationSys.Init(nil, xl)
globalNotificationSys = NewNotificationSys(endpointZones)
globalNotificationSys.Init(nil, zones)
return xl, nil
return zones, nil
}
// initObjectLayer - Instantiates object layer and returns it.
func initObjectLayer(endpoints EndpointList) (ObjectLayer, []StorageAPI, error) {
objLayer, err := newTestObjectLayer(endpoints)
func initObjectLayer(endpointZones EndpointZones) (ObjectLayer, []StorageAPI, error) {
objLayer, err := newTestObjectLayer(endpointZones)
if err != nil {
return nil, nil, err
}
var formattedDisks []StorageAPI
// Should use the object layer tests for validating cache.
if xl, ok := objLayer.(*xlObjects); ok {
formattedDisks = xl.storageDisks
if z, ok := objLayer.(*xlZones); ok {
formattedDisks = z.zones[0].GetDisks(0)()
}
// Success.
@ -2052,7 +2029,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
if err != nil {
t.Fatalf("Initialization of disks for XL setup: %s", err)
}
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(erasureDisks...))
objLayer, _, err := initObjectLayer(mustGetZoneEndpoints(erasureDisks...))
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
@ -2303,23 +2280,27 @@ func generateTLSCertKey(host string) ([]byte, []byte, error) {
return certOut.Bytes(), keyOut.Bytes(), nil
}
func mustGetNewEndpointList(args ...string) (endpoints EndpointList) {
if len(args) == 1 {
endpoint, err := NewEndpoint(args[0])
logger.FatalIf(err, "unable to create new endpoint")
endpoints = append(endpoints, endpoint)
} else {
var err error
endpoints, err = NewEndpointList(args...)
logger.FatalIf(err, "unable to create new endpoint list")
}
func mustGetZoneEndpoints(args ...string) EndpointZones {
endpoints := mustGetNewEndpoints(args...)
return []ZoneEndpoints{{
SetCount: 1,
DrivesPerSet: len(args),
Endpoints: endpoints,
}}
}
func mustGetNewEndpoints(args ...string) (endpoints Endpoints) {
endpoints, err := NewEndpoints(args...)
logger.FatalIf(err, "unable to create new endpoint list")
return endpoints
}
func getEndpointsLocalAddr(endpoints EndpointList) string {
for _, endpoint := range endpoints {
if endpoint.IsLocal && endpoint.Type() == URLEndpointType {
return endpoint.Host
func getEndpointsLocalAddr(endpointZones EndpointZones) string {
for _, endpoints := range endpointZones {
for _, endpoint := range endpoints.Endpoints {
if endpoint.IsLocal && endpoint.Type() == URLEndpointType {
return endpoint.Host
}
}
}

14
cmd/tree-walk_test.go
View file

@ -123,7 +123,7 @@ func TestTreeWalk(t *testing.T) {
if err != nil {
t.Fatalf("Unable to create tmp directory: %s", err)
}
endpoints := mustGetNewEndpointList(fsDir)
endpoints := mustGetNewEndpoints(fsDir)
disk, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -160,7 +160,7 @@ func TestTreeWalkTimeout(t *testing.T) {
if err != nil {
t.Fatalf("Unable to create tmp directory: %s", err)
}
endpoints := mustGetNewEndpointList(fsDir)
endpoints := mustGetNewEndpoints(fsDir)
disk, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -235,13 +235,13 @@ func TestListDir(t *testing.T) {
}
// Create two StorageAPIs disk1 and disk2.
endpoints := mustGetNewEndpointList(fsDir1)
endpoints := mustGetNewEndpoints(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Errorf("Unable to create StorageAPI: %s", err)
}
endpoints = mustGetNewEndpointList(fsDir2)
endpoints = mustGetNewEndpoints(fsDir2)
disk2, err := newStorageAPI(endpoints[0])
if err != nil {
t.Errorf("Unable to create StorageAPI: %s", err)
@ -300,7 +300,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
t.Fatalf("Unable to create tmp directory: %s", err)
}
endpoints := mustGetNewEndpointList(fsDir1)
endpoints := mustGetNewEndpoints(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -405,7 +405,7 @@ func TestSortedness(t *testing.T) {
t.Errorf("Unable to create tmp directory: %s", err)
}
endpoints := mustGetNewEndpointList(fsDir1)
endpoints := mustGetNewEndpoints(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -476,7 +476,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
t.Errorf("Unable to create tmp directory: %s", err)
}
endpoints := mustGetNewEndpointList(fsDir1)
endpoints := mustGetNewEndpoints(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)

11
cmd/web-handlers_test.go
View file

@ -1587,9 +1587,14 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
}
// Set faulty disks to XL backend
xl := obj.(*xlObjects)
for i, d := range xl.storageDisks {
xl.storageDisks[i] = newNaughtyDisk(d, nil, errFaultyDisk)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
xlDisks := xl.getDisks()
xl.getDisks = func() []StorageAPI {
for i, d := range xlDisks {
xlDisks[i] = newNaughtyDisk(d, nil, errFaultyDisk)
}
return xlDisks
}
// Initialize web rpc endpoint.

96
cmd/xl-sets.go
View file

@ -55,18 +55,6 @@ func (s setsStorageAPI) Close() error {
return nil
}
func (s setsDsyncLockers) Close() error {
for i := 0; i < len(s); i++ {
for _, locker := range s[i] {
if locker == nil {
continue
}
locker.Close()
}
}
return nil
}
// xlSets implements ObjectLayer combining a static list of erasure coded
// object sets. NOTE: There is no dynamic scaling allowed or intended in
// current design.
@ -89,7 +77,7 @@ type xlSets struct {
lockersMap map[Endpoint]dsync.NetLocker
// List of endpoints provided on the command line.
endpoints EndpointList
endpoints Endpoints
// Total number of sets and the number of disks per set.
setCount, drivesPerSet int
@ -123,11 +111,10 @@ func (s *xlSets) isConnected(endpoint Endpoint) bool {
if s.xlDisks[i][j].String() != endpointStr {
continue
}
if s.xlDisks[i][j].IsOnline() {
return true
if !s.xlLockers[i][j].IsOnline() {
continue
}
s.xlLockers[i][j].Close()
return false
return s.xlDisks[i][j].IsOnline()
}
}
return false
@ -282,8 +269,7 @@ func (s *xlSets) GetDisks(setIndex int) func() []StorageAPI {
const defaultMonitorConnectEndpointInterval = time.Second * 10 // Set to 10 secs.
// Initialize new set of erasure coded sets.
func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesPerSet int) (ObjectLayer, error) {
func newXLSets(endpoints Endpoints, format *formatXLV3, setCount int, drivesPerSet int) (*xlSets, error) {
lockersMap := make(map[Endpoint]dsync.NetLocker)
for _, endpoint := range endpoints {
lockersMap[endpoint] = newLockAPI(endpoint)
@ -464,13 +450,6 @@ func (s *xlSets) Shutdown(ctx context.Context) error {
// even if one of the sets fail to create buckets, we proceed to undo a
// successful operation.
func (s *xlSets) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
set := s.getHashedSet(bucket)
bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer bucketLock.Unlock()
g := errgroup.WithNErrs(len(s.sets))
// Create buckets in parallel across all sets.
@ -549,14 +528,7 @@ func (s *xlSets) getHashedSet(input string) (set *xlObjects) {
// GetBucketInfo - returns bucket info from one of the erasure coded set.
func (s *xlSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
set := s.getHashedSet(bucket)
bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
if err = bucketLock.GetRLock(globalOperationTimeout); err != nil {
return bucketInfo, err
}
defer bucketLock.RUnlock()
return s.getHashedSet(bucket).GetBucketInfo(ctx, bucket)
return s.getHashedSet("").GetBucketInfo(ctx, bucket)
}
// ListObjectsV2 lists all objects in bucket filtered by prefix
@ -635,13 +607,6 @@ func (s *xlSets) IsCompressionSupported() bool {
// even if one of the sets fail to delete buckets, we proceed to
// undo a successful operation.
func (s *xlSets) DeleteBucket(ctx context.Context, bucket string) error {
set := s.getHashedSet(bucket)
bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer bucketLock.Unlock()
g := errgroup.WithNErrs(len(s.sets))
// Delete buckets in parallel across all sets.
@ -729,7 +694,6 @@ func (s *xlSets) DeleteObject(ctx context.Context, bucket string, object string)
// objects are group by set first, and then bulk delete is invoked
// for each set, the error response of each delete will be returned
func (s *xlSets) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
type delObj struct {
// Set index associated to this object
setIndex int
@ -787,13 +751,6 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
return srcSet.CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
}
if !cpSrcDstSame {
objectDWLock := destSet.nsMutex.NewNSLock(ctx, destSet.getLockers(), destBucket, destObject)
if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
return objInfo, err
}
defer objectDWLock.Unlock()
}
putOpts := ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
return destSet.putObject(ctx, destBucket, destObject, srcInfo.PutObjReader, putOpts)
}
@ -1078,11 +1035,6 @@ func (s *xlSets) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker
// walked and merged at this layer. Resulting value through the merge process sends
// the data in lexically sorted order.
func (s *xlSets) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, heal bool) (loi ListObjectsInfo, err error) {
if delimiter != SlashSeparator && delimiter != "" {
// "heal" option passed can be ignored as the heal-listing does not send non-standard delimiter.
return s.listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys)
}
if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil {
return loi, err
}
@ -1114,6 +1066,11 @@ func (s *xlSets) listObjects(ctx context.Context, bucket, prefix, marker, delimi
maxKeys = maxObjectList
}
if delimiter != SlashSeparator && delimiter != "" {
// "heal" option passed can be ignored as the heal-listing does not send non-standard delimiter.
return s.listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys)
}
// Default is recursive, if delimiter is set then list non recursive.
recursive := true
if delimiter == SlashSeparator {
@ -1284,7 +1241,7 @@ else
fi
*/
func formatsToDrivesInfo(endpoints EndpointList, formats []*formatXLV3, sErrs []error) (beforeDrives []madmin.DriveInfo) {
func formatsToDrivesInfo(endpoints Endpoints, formats []*formatXLV3, sErrs []error) (beforeDrives []madmin.DriveInfo) {
beforeDrives = make([]madmin.DriveInfo, len(endpoints))
// Existing formats are available (i.e. ok), so save it in
// result, also populate disks to be healed.
@ -1317,14 +1274,6 @@ func formatsToDrivesInfo(endpoints EndpointList, formats []*formatXLV3, sErrs []
// Reloads the format from the disk, usually called by a remote peer notifier while
// healing in a distributed setup.
func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
// Acquire lock on format.json
set := s.getHashedSet(formatConfigFile)
formatLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), minioMetaBucket, formatConfigFile)
if err = formatLock.GetRLock(globalHealingTimeout); err != nil {
return err
}
defer formatLock.RUnlock()
storageDisks, errs := initStorageDisksWithErrors(s.endpoints)
for i, err := range errs {
if err != nil && err != errDiskNotFound {
@ -1367,7 +1316,6 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
// Close all existing disks, lockers and reconnect all the disks/lockers.
s.xlDisks.Close()
s.xlLockers.Close()
s.connectDisksAndLockers()
// Restart monitoring loop to monitor reformatted disks again.
@ -1433,17 +1381,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI) {
}
// HealFormat - heals missing `format.json` on fresh unformatted disks.
// TODO: In future support corrupted disks missing format.json but has erasure
// coded data in it.
func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
// Acquire lock on format.json
set := s.getHashedSet(formatConfigFile)
formatLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), minioMetaBucket, formatConfigFile)
if err = formatLock.GetLock(globalHealingTimeout); err != nil {
return madmin.HealResultItem{}, err
}
defer formatLock.Unlock()
storageDisks, errs := initStorageDisksWithErrors(s.endpoints)
for i, derr := range errs {
if derr != nil && derr != errDiskNotFound {
@ -1576,7 +1514,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
s.xlDisks.Close()
s.xlLockers.Close()
s.connectDisksAndLockers()
// Restart our monitoring loop to start monitoring newly formatted disks.
@ -1588,13 +1525,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
// HealBucket - heals inconsistent buckets and bucket metadata on all sets.
func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (result madmin.HealResultItem, err error) {
set := s.getHashedSet(bucket)
bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
if err = bucketLock.GetLock(globalOperationTimeout); err != nil {
return result, err
}
defer bucketLock.Unlock()
// Initialize heal result info
result = madmin.HealResultItem{
Type: madmin.HealItemBucket,
@ -1697,7 +1627,7 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj
// Wait at max 10 minute for an inprogress request before proceeding to heal
waitCount := 600
// Any requests in progress, delay the heal.
for (globalHTTPServer.GetRequestCount() >= int32(globalXLSetCount*globalXLSetDriveCount)) &&
for (globalHTTPServer.GetRequestCount() >= int32(s.setCount*s.drivesPerSet)) &&
waitCount > 0 {
waitCount--
time.Sleep(1 * time.Second)

5
cmd/xl-sets_test.go
View file

@ -75,7 +75,7 @@ func TestNewXLSets(t *testing.T) {
defer os.RemoveAll(disk)
}
endpoints := mustGetNewEndpointList(erasureDisks...)
endpoints := mustGetNewEndpoints(erasureDisks...)
_, err := waitForFormatXL(true, endpoints, 0, 16)
if err != errInvalidArgument {
t.Fatalf("Expecting error, got %s", err)
@ -113,7 +113,8 @@ func TestHashedLayer(t *testing.T) {
defer os.RemoveAll(dir)
}
objs = append(objs, obj.(*xlObjects))
z := obj.(*xlZones)
objs = append(objs, z.zones[0].sets[0])
}
sets := &xlSets{sets: objs, distributionAlgo: "CRCMOD"}

5
cmd/xl-v1-common_test.go
View file

@ -51,7 +51,8 @@ func TestXLParentDirIsObject(t *testing.T) {
t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
}
fs := obj.(*xlObjects)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
testCases := []struct {
parentIsObject bool
objectName string
@ -86,7 +87,7 @@ func TestXLParentDirIsObject(t *testing.T) {
}
for i, testCase := range testCases {
gotValue := fs.parentDirIsObject(context.Background(), bucketName, testCase.objectName)
gotValue := xl.parentDirIsObject(context.Background(), bucketName, testCase.objectName)
if testCase.parentIsObject != gotValue {
t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject)
}

2
cmd/xl-v1-healing-common.go
View file

@ -122,7 +122,7 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error)
func getLatestXLMeta(ctx context.Context, partsMetadata []xlMetaV1, errs []error) (xlMetaV1, error) {
// There should be atleast half correct entries, if not return failure
if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, globalXLSetDriveCount/2); reducedErr != nil {
if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, len(partsMetadata)/2); reducedErr != nil {
return xlMetaV1{}, reducedErr
}

13
cmd/xl-v1-healing-common_test.go
View file

@ -168,7 +168,8 @@ func TestListOnlineDisks(t *testing.T) {
bucket := "bucket"
object := "object"
data := bytes.Repeat([]byte("a"), 1024)
xlDisks := obj.(*xlObjects).storageDisks
z := obj.(*xlZones)
xlDisks := z.zones[0].sets[0].getDisks()
for i, test := range testCases {
// Prepare bucket/object backend for the tests below.
@ -266,10 +267,10 @@ func TestDisksWithAllParts(t *testing.T) {
object := "object"
// make data with more than one part
partCount := 3
data := bytes.Repeat([]byte("a"), int(globalPutPartSize)*partCount)
xl := obj.(*xlObjects)
xlDisks := xl.storageDisks
data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
xlDisks := xl.getDisks()
err = obj.MakeBucketWithLocation(ctx, "bucket", "")
if err != nil {
t.Fatalf("Failed to make a bucket %v", err)
@ -281,7 +282,7 @@ func TestDisksWithAllParts(t *testing.T) {
}
_, errs := readAllXLMetadata(ctx, xlDisks, bucket, object)
readQuorum := len(xl.storageDisks) / 2
readQuorum := len(xlDisks) / 2
if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
t.Fatalf("Failed to read xl meta data %v", reducedErr)
}

10
cmd/xl-v1-healing.go
View file

@ -679,14 +679,6 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
}
healCtx := logger.SetReqInfo(context.Background(), newReqInfo)
// Lock the object before healing. Use read lock since healing
// will only regenerate parts & xl.json of outdated disks.
objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if lerr := objectLock.GetRLock(globalHealingTimeout); lerr != nil {
return madmin.HealResultItem{}, lerr
}
defer objectLock.RUnlock()
// Healing directories handle it separately.
if hasSuffix(object, SlashSeparator) {
return xl.healObjectDir(healCtx, bucket, object, dryRun)
@ -733,7 +725,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
writeQuorum = len(storageDisks)/2 + 1
}
if !dryRun && remove {
err = xl.deleteObject(ctx, bucket, object, writeQuorum, false)
xl.deleteObject(ctx, bucket, object, writeQuorum, false)
}
}
return defaultHealResult(latestXLMeta, storageDisks, errs, bucket, object), toObjectErr(reducedErr, bucket, object)

45
cmd/xl-v1-healing_test.go
View file

@ -35,7 +35,7 @@ func TestUndoMakeBucket(t *testing.T) {
defer removeRoots(fsDirs)
// Remove format.json on 16 disks.
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
obj, _, err := initObjectLayer(mustGetZoneEndpoints(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -44,8 +44,9 @@ func TestUndoMakeBucket(t *testing.T) {
if err = obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
t.Fatal(err)
}
xl := obj.(*xlObjects)
undoMakeBucket(xl.storageDisks, bucketName)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
undoMakeBucket(xl.getDisks(), bucketName)
// Validate if bucket was deleted properly.
_, err = obj.GetBucketInfo(context.Background(), bucketName)
@ -68,7 +69,7 @@ func TestHealObjectCorrupted(t *testing.T) {
defer removeRoots(fsDirs)
// Everything is fine, should return nil
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
objLayer, _, err := initObjectLayer(mustGetZoneEndpoints(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -108,8 +109,9 @@ func TestHealObjectCorrupted(t *testing.T) {
}
// Test 1: Remove the object backend files from the first disk.
xl := objLayer.(*xlObjects)
firstDisk := xl.storageDisks[0]
z := objLayer.(*xlZones)
xl := z.zones[0].sets[0]
firstDisk := xl.getDisks()[0]
err = firstDisk.DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
if err != nil {
t.Fatalf("Failed to delete a file - %v", err)
@ -179,8 +181,8 @@ func TestHealObjectCorrupted(t *testing.T) {
// Test 4: checks if HealObject returns an error when xl.json is not found
// in more than read quorum number of disks, to create a corrupted situation.
for i := 0; i <= len(xl.storageDisks)/2; i++ {
xl.storageDisks[i].DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
for i := 0; i <= len(xl.getDisks())/2; i++ {
xl.getDisks()[i].DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
}
// Try healing now, expect to receive errDiskNotFound.
@ -207,7 +209,7 @@ func TestHealObjectXL(t *testing.T) {
defer removeRoots(fsDirs)
// Everything is fine, should return nil
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
obj, _, err := initObjectLayer(mustGetZoneEndpoints(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -247,8 +249,9 @@ func TestHealObjectXL(t *testing.T) {
}
// Remove the object backend files from the first disk.
xl := obj.(*xlObjects)
firstDisk := xl.storageDisks[0]
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
firstDisk := xl.getDisks()[0]
err = firstDisk.DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
if err != nil {
t.Fatalf("Failed to delete a file - %v", err)
@ -264,9 +267,13 @@ func TestHealObjectXL(t *testing.T) {
t.Errorf("Expected xl.json file to be present but stat failed - %v", err)
}
// Nil more than half the disks, to remove write quorum.
for i := 0; i <= len(xl.storageDisks)/2; i++ {
xl.storageDisks[i] = nil
xlDisks := xl.getDisks()
xl.getDisks = func() []StorageAPI {
// Nil more than half the disks, to remove write quorum.
for i := 0; i <= len(xlDisks)/2; i++ {
xlDisks[i] = nil
}
return xlDisks
}
// Try healing now, expect to receive errDiskNotFound.
@ -287,7 +294,7 @@ func TestHealEmptyDirectoryXL(t *testing.T) {
defer removeRoots(fsDirs)
// Everything is fine, should return nil
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
obj, _, err := initObjectLayer(mustGetZoneEndpoints(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -302,14 +309,16 @@ func TestHealEmptyDirectoryXL(t *testing.T) {
}
// Upload an empty directory
_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte{}), 0, "", ""), opts)
_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t,
bytes.NewReader([]byte{}), 0, "", ""), opts)
if err != nil {
t.Fatal(err)
}
// Remove the object backend files from the first disk.
xl := obj.(*xlObjects)
firstDisk := xl.storageDisks[0]
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
firstDisk := xl.getDisks()[0]
err = firstDisk.DeleteFile(bucket, object)
if err != nil {
t.Fatalf("Failed to delete a file - %v", err)

20
cmd/xl-v1-metadata_test.go
View file

@ -76,7 +76,9 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes
}
}
_, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
_, _, err = xl.readXLMetaStat(context.Background(), bucketName, objectName)
if err != nil {
t.Fatal(err)
}
@ -85,7 +87,7 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes
removeDiskN(disks, 7)
// Removing disk shouldn't affect reading object info.
_, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName)
_, _, err = xl.readXLMetaStat(context.Background(), bucketName, objectName)
if err != nil {
t.Fatal(err)
}
@ -94,7 +96,7 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes
os.RemoveAll(path.Join(disk, bucketName))
}
_, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName)
_, _, err = xl.readXLMetaStat(context.Background(), bucketName, objectName)
if err != errVolumeNotFound {
t.Fatal(err)
}
@ -159,9 +161,11 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t
}
}
uploadIDPath := obj.(*xlObjects).getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0])
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
uploadIDPath := xl.getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0])
_, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
_, _, err = xl.readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
if err != nil {
t.Fatal(err)
}
@ -170,17 +174,17 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t
removeDiskN(disks, 7)
// Removing disk shouldn't affect reading object parts info.
_, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
_, _, err = xl.readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
if err != nil {
t.Fatal(err)
}
for _, disk := range disks {
os.RemoveAll(path.Join(disk, bucketNames[0]))
os.RemoveAll(path.Join(disk, minioMetaMultipartBucket, obj.(*xlObjects).getMultipartSHADir(bucketNames[0], objectNames[0])))
os.RemoveAll(path.Join(disk, minioMetaMultipartBucket, xl.getMultipartSHADir(bucketNames[0], objectNames[0])))
}
_, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
_, _, err = xl.readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
if err != errFileNotFound {
t.Fatal(err)
}

71
cmd/xl-v1-multipart.go
View file

@ -262,13 +262,6 @@ func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object strin
//
// Implements S3 compatible Upload Part Copy API.
func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
// Hold read locks on source object only if we are
// going to read data from source object.
objectSRLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), srcBucket, srcObject)
if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
return pi, err
}
defer objectSRLock.RUnlock()
if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, xl); err != nil {
return pi, err
@ -303,17 +296,9 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
var partsMetadata []xlMetaV1
var errs []error
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
// pre-check upload id lock.
preUploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
if err := preUploadIDLock.GetRLock(globalOperationTimeout); err != nil {
return pi, err
}
// Validates if upload ID exists.
if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
preUploadIDLock.RUnlock()
return pi, toObjectErr(err, bucket, object, uploadID)
}
@ -324,16 +309,13 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
// get Quorum for this object
_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
if err != nil {
preUploadIDLock.RUnlock()
return pi, toObjectErr(err, bucket, object)
}
reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
if reducedErr == errXLWriteQuorum {
preUploadIDLock.RUnlock()
return pi, toObjectErr(reducedErr, bucket, object)
}
preUploadIDLock.RUnlock()
// List all online disks.
onlineDisks, modTime := listOnlineDisks(xl.getDisks(), partsMetadata, errs)
@ -403,13 +385,6 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
}
}
// post-upload check (write) lock
postUploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
if err = postUploadIDLock.GetLock(globalOperationTimeout); err != nil {
return pi, err
}
defer postUploadIDLock.Unlock()
// Validates if upload ID exists.
if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return pi, toObjectErr(err, bucket, object, uploadID)
@ -497,16 +472,6 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
if err := checkListPartsArgs(ctx, bucket, object, xl); err != nil {
return result, err
}
// Hold lock so that there is no competing
// abort-multipart-upload or complete-multipart-upload.
uploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(),
minioMetaMultipartBucket,
xl.getUploadIDLockPath(bucket, object, uploadID))
if err := uploadIDLock.GetLock(globalListingTimeout); err != nil {
return result, err
}
defer uploadIDLock.Unlock()
if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return result, toObjectErr(err, bucket, object, uploadID)
}
@ -603,27 +568,6 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
if err := checkCompleteMultipartArgs(ctx, bucket, object, xl); err != nil {
return oi, err
}
// Hold write lock on the object.
destLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if err := destLock.GetLock(globalObjectTimeout); err != nil {
return oi, err
}
defer destLock.Unlock()
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
// Hold lock so that
//
// 1) no one aborts this multipart upload
//
// 2) no one does a parallel complete-multipart-upload on this
// multipart upload
uploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
return oi, err
}
defer uploadIDLock.Unlock()
if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return oi, toObjectErr(err, bucket, object, uploadID)
@ -638,6 +582,8 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
// Calculate s3 compatible md5sum for complete multipart.
s3MD5 := getCompleteMultipartMD5(parts)
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
// Read metadata associated with the object from all disks.
partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath)
@ -820,22 +766,13 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
if err := checkAbortMultipartArgs(ctx, bucket, object, xl); err != nil {
return err
}
// Construct uploadIDPath.
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
// Hold lock so that there is no competing
// complete-multipart-upload or put-object-part.
uploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer uploadIDLock.Unlock()
// Validates if upload ID exists.
if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return toObjectErr(err, bucket, object, uploadID)
}
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
// Read metadata associated with the object from all disks.
partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath)

3
cmd/xl-v1-multipart_test.go
View file

@ -32,7 +32,8 @@ func TestXLCleanupStaleMultipartUploads(t *testing.T) {
// Defer cleanup of backend directories
defer removeRoots(fsDirs)
xl := obj.(*xlObjects)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
// Close the go-routine, we are going to
// manually start it and test in this test case.

67
cmd/xl-v1-object.go
View file

@ -123,27 +123,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
// GetObjectNInfo - returns object info and an object
// Read(Closer). When err != nil, the returned reader is always nil.
func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
var nsUnlocker = func() {}
// Acquire lock
if lockType != noLock {
lock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
switch lockType {
case writeLock:
if err = lock.GetLock(globalObjectTimeout); err != nil {
return nil, err
}
nsUnlocker = lock.Unlock
case readLock:
if err = lock.GetRLock(globalObjectTimeout); err != nil {
return nil, err
}
nsUnlocker = lock.RUnlock
}
}
if err = checkGetObjArgs(ctx, bucket, object); err != nil {
nsUnlocker()
return nil, err
}
@ -152,20 +132,18 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
if hasSuffix(object, SlashSeparator) {
var objInfo ObjectInfo
if objInfo, err = xl.getObjectInfoDir(ctx, bucket, object); err != nil {
nsUnlocker()
return nil, toObjectErr(err, bucket, object)
}
return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn)
}
var objInfo ObjectInfo
objInfo, err = xl.getObjectInfo(ctx, bucket, object)
if err != nil {
nsUnlocker()
return nil, toObjectErr(err, bucket, object)
}
fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn)
if nErr != nil {
return nil, nErr
}
@ -189,12 +167,6 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
// startOffset indicates the starting read location of the object.
// length indicates the total length of the object.
func (xl xlObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
// Lock the object before reading.
objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
return err
}
defer objectLock.RUnlock()
return xl.getObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
}
@ -368,13 +340,6 @@ func (xl xlObjects) getObjectInfoDir(ctx context.Context, bucket, object string)
// GetObjectInfo - reads object metadata and replies back ObjectInfo.
func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
// Lock the object before reading.
objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
return oi, err
}
defer objectLock.RUnlock()
if err := checkGetObjArgs(ctx, bucket, object); err != nil {
return oi, err
}
@ -497,13 +462,6 @@ func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string,
return ObjectInfo{}, err
}
// Lock the object.
objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if err := objectLock.GetLock(globalObjectTimeout); err != nil {
return objInfo, err
}
defer objectLock.Unlock()
return xl.putObject(ctx, bucket, object, data, opts)
}
@ -844,20 +802,6 @@ func (xl xlObjects) deleteObjects(ctx context.Context, bucket string, objects []
errs[i] = checkDelObjArgs(ctx, bucket, object)
}
var objectLocks = make([]RWLocker, len(objects))
for i, object := range objects {
if errs[i] != nil {
continue
}
// Acquire a write lock before deleting the object.
objectLocks[i] = xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if errs[i] = objectLocks[i].GetLock(globalOperationTimeout); errs[i] != nil {
continue
}
defer objectLocks[i].Unlock()
}
for i, object := range objects {
isObjectDirs[i] = hasSuffix(object, SlashSeparator)
}
@ -953,13 +897,6 @@ func (xl xlObjects) DeleteObjects(ctx context.Context, bucket string, objects []
// any error as it is not necessary for the handler to reply back a
// response to the client request.
func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (err error) {
// Acquire a write lock before deleting the object.
objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
if perr := objectLock.GetLock(globalOperationTimeout); perr != nil {
return perr
}
defer objectLock.Unlock()
if err = checkDelObjArgs(ctx, bucket, object); err != nil {
return err
}

82
cmd/xl-v1-object_test.go
View file

@ -125,7 +125,9 @@ func TestXLDeleteObjectsXLSet(t *testing.T) {
for _, dir := range fsDirs {
defer os.RemoveAll(dir)
}
objs = append(objs, obj.(*xlObjects))
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
objs = append(objs, xl)
}
xlSets := &xlSets{sets: objs, distributionAlgo: "CRCMOD"}
@ -192,8 +194,11 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// Cleanup backend directories
defer removeRoots(fsDirs)
xl := obj.(*xlObjects)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
// Create "bucket"
err = obj.MakeBucketWithLocation(context.Background(), "bucket", "")
@ -211,8 +216,12 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
}
// for a 16 disk setup, quorum is 9. To simulate disks not found yet
// quorum is available, we remove disks leaving quorum disks behind.
for i := range xl.storageDisks[:7] {
xl.storageDisks[i] = newNaughtyDisk(xl.storageDisks[i], nil, errFaultyDisk)
xlDisks := xl.getDisks()
xl.getDisks = func() []StorageAPI {
for i := range xlDisks[:7] {
xlDisks[i] = newNaughtyDisk(xlDisks[i], nil, errFaultyDisk)
}
return xlDisks
}
err = obj.DeleteObject(context.Background(), bucket, object)
if err != nil {
@ -226,15 +235,17 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
}
// Remove one more disk to 'lose' quorum, by setting it to nil.
xl.storageDisks[7] = nil
xl.storageDisks[8] = nil
xlDisks = xl.getDisks()
xl.getDisks = func() []StorageAPI {
xlDisks[7] = nil
xlDisks[8] = nil
return xlDisks
}
err = obj.DeleteObject(context.Background(), bucket, object)
// since majority of disks are not available, metaquorum is not achieved and hence errXLReadQuorum error
if err != toObjectErr(errXLReadQuorum, bucket, object) {
t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errXLReadQuorum, bucket, object), err)
}
// Cleanup backend directories
removeRoots(fsDirs)
}
func TestGetObjectNoQuorum(t *testing.T) {
@ -243,8 +254,11 @@ func TestGetObjectNoQuorum(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// Cleanup backend directories.
defer removeRoots(fsDirs)
xl := obj.(*xlObjects)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
// Create "bucket"
err = obj.MakeBucketWithLocation(context.Background(), "bucket", "")
@ -270,22 +284,24 @@ func TestGetObjectNoQuorum(t *testing.T) {
for i := 0; i <= f; i++ {
diskErrors[i] = nil
}
for i := range xl.storageDisks[:9] {
switch diskType := xl.storageDisks[i].(type) {
xlDisks := xl.getDisks()
for i := range xlDisks[:9] {
switch diskType := xlDisks[i].(type) {
case *naughtyDisk:
xl.storageDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk)
xlDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk)
default:
xl.storageDisks[i] = newNaughtyDisk(xl.storageDisks[i], diskErrors, errFaultyDisk)
xlDisks[i] = newNaughtyDisk(xlDisks[i], diskErrors, errFaultyDisk)
}
}
xl.getDisks = func() []StorageAPI {
return xlDisks
}
// Fetch object from store.
err = xl.GetObject(context.Background(), bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
if err != toObjectErr(errXLReadQuorum, bucket, object) {
t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err)
}
}
// Cleanup backend directories.
removeRoots(fsDirs)
}
func TestPutObjectNoQuorum(t *testing.T) {
@ -295,7 +311,11 @@ func TestPutObjectNoQuorum(t *testing.T) {
t.Fatal(err)
}
xl := obj.(*xlObjects)
// Cleanup backend directories.
defer removeRoots(fsDirs)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
// Create "bucket"
err = obj.MakeBucketWithLocation(context.Background(), "bucket", "")
@ -321,22 +341,24 @@ func TestPutObjectNoQuorum(t *testing.T) {
for i := 0; i <= f; i++ {
diskErrors[i] = nil
}
for i := range xl.storageDisks[:9] {
switch diskType := xl.storageDisks[i].(type) {
xlDisks := xl.getDisks()
for i := range xlDisks[:9] {
switch diskType := xlDisks[i].(type) {
case *naughtyDisk:
xl.storageDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk)
xlDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk)
default:
xl.storageDisks[i] = newNaughtyDisk(xl.storageDisks[i], diskErrors, errFaultyDisk)
xlDisks[i] = newNaughtyDisk(xlDisks[i], diskErrors, errFaultyDisk)
}
}
xl.getDisks = func() []StorageAPI {
return xlDisks
}
// Upload new content to same object "object"
_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
if err != toObjectErr(errXLWriteQuorum, bucket, object) {
t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err)
}
}
// Cleanup backend directories.
removeRoots(fsDirs)
}
// Tests both object and bucket healing.
@ -346,7 +368,9 @@ func TestHealing(t *testing.T) {
t.Fatal(err)
}
defer removeRoots(fsDirs)
xl := obj.(*xlObjects)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
// Create "bucket"
err = obj.MakeBucketWithLocation(context.Background(), "bucket", "")
@ -369,7 +393,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err)
}
disk := xl.storageDisks[0]
disk := xl.getDisks()[0]
xlMetaPreHeal, err := readXLMeta(context.Background(), disk, bucket, object)
if err != nil {
t.Fatal(err)
@ -438,7 +462,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err)
}
// Stat the bucket to make sure that it was created.
_, err = xl.storageDisks[0].StatVol(bucket)
_, err = xl.getDisks()[0].StatVol(bucket)
if err != nil {
t.Fatal(err)
}
@ -454,9 +478,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
var opts ObjectOptions
// make data with more than one part
partCount := 3
data := bytes.Repeat([]byte("a"), int(globalPutPartSize)*partCount)
xl := obj.(*xlObjects)
xlDisks := xl.storageDisks
data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount)
z := obj.(*xlZones)
xl := z.zones[0].sets[0]
xlDisks := xl.getDisks()
err := obj.MakeBucketWithLocation(context.Background(), bucket, globalMinioDefaultRegion)
if err != nil {

16
cmd/xl-v1-utils_test.go
View file

@ -399,19 +399,19 @@ func TestShuffleDisks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
objLayer, _, err := initObjectLayer(mustGetZoneEndpoints(disks...))
if err != nil {
removeRoots(disks)
t.Fatal(err)
}
defer removeRoots(disks)
xl := objLayer.(*xlObjects)
testShuffleDisks(t, xl)
z := objLayer.(*xlZones)
testShuffleDisks(t, z)
}
// Test shuffleDisks which returns shuffled slice of disks for their actual distribution.
func testShuffleDisks(t *testing.T, xl *xlObjects) {
disks := xl.storageDisks
func testShuffleDisks(t *testing.T, z *xlZones) {
disks := z.zones[0].GetDisks(0)()
distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
shuffledDisks := shuffleDisks(disks, distribution)
// From the "distribution" above you can notice that:
@ -444,12 +444,12 @@ func TestEvalDisks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
objLayer, _, err := initObjectLayer(mustGetZoneEndpoints(disks...))
if err != nil {
removeRoots(disks)
t.Fatal(err)
}
defer removeRoots(disks)
xl := objLayer.(*xlObjects)
testShuffleDisks(t, xl)
z := objLayer.(*xlZones)
testShuffleDisks(t, z)
}

5
cmd/xl-v1.go
View file

@ -52,16 +52,13 @@ type xlObjects struct {
// Byte pools used for temporary i/o buffers.
bp *bpool.BytePoolCap
// TODO: Deprecated only kept here for tests, should be removed in future.
storageDisks []StorageAPI
// TODO: ListObjects pool management, should be removed in future.
listPool *TreeWalkPool
}
// NewNSLock - initialize a new namespace RWLocker instance.
func (xl xlObjects) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
return xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
return xl.nsMutex.NewNSLock(ctx, xl.getLockers, bucket, object)
}
// Shutdown function for object storage interface.

1299
cmd/xl-zones.go Normal file
View file

File diff suppressed because it is too large Load diff

11
docs/distributed/DESIGN.md
View file

@ -1,5 +1,5 @@
# Distributed Server Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
This document explains the design approach, advanced use cases and limits of the MinIO distributed server.
This document explains the design approach and advanced use cases of the MinIO distributed server.
## Command-line
```
@ -127,10 +127,6 @@ Distributed erasure coded configuration with rack level redundancy 32 sets in to
minio server http://rack{1...4}-host{1...8}.example.net/export{1...16}
```
Distributed erasure coded configuration with no rack level redundancy but redundancy with in the rack we split the arguments, 32 sets in total, 16 disks per set.
```
minio server http://rack1-host{1...8}.example.net/export{1...16} http://rack2-host{1...8}.example.net/export{1...16} http://rack3-host{1...8}.example.net/export{1...16} http://rack4-host{1...8}.example.net/export{1...16}
```
## Backend `format.json` changes
`format.json` has new fields
@ -208,8 +204,3 @@ type formatXLV2 struct {
} `json:"xl"`
}
```
## Limits
- Minimum of 4 disks are needed for any erasure coded configuration.
- Maximum of 32 distinct nodes are supported in distributed configuration.

21
docs/distributed/README.md
View file

@ -12,16 +12,10 @@ Distributed MinIO provides protection against multiple node/drive failures and [
### High availability
A stand-alone MinIO server would go down if the server hosting the disks goes offline. In contrast, a distributed MinIO setup with _n_ disks will have your data safe as long as _n/2_ or more disks are online. You'll need a minimum of _(n/2 + 1)_ [Quorum](https://github.com/minio/dsync#lock-process) disks to create new objects though.
A stand-alone MinIO server would go down if the server hosting the disks goes offline. In contrast, a distributed MinIO setup with _n_ disks will have your data safe as long as _n/2_ or more disks are online. You'll need a minimum of _(n/2 + 1)_ disks to create new objects.
For example, an 16-node distributed MinIO setup with 16 disks per node would continue serving files, even if up to 8 servers are offline. But, you'll need at least 9 servers online to create new objects.
### Limits
As with MinIO in stand-alone mode, distributed MinIO has a per tenant limit of minimum of 2 and maximum of 32 servers. There are no limits on number of disks across these servers. If you need a multiple tenant setup, you can easily spin up multiple MinIO instances managed by orchestration tools like Kubernetes, Docker Swarm etc.
Note that with distributed MinIO you can play around with the number of nodes and drives as long as the limits are adhered to. For example, you can have 2 nodes with 4 drives each, 4 nodes with 4 drives each, 8 nodes with 2 drives each, 32 servers with 64 drives each and so on.
You can also use [storage classes](https://github.com/minio/minio/tree/master/docs/erasure/storage-class) to set custom data and parity distribution per object.
### Consistency Guarantees
@ -61,7 +55,18 @@ export MINIO_SECRET_KEY=<SECRET_KEY>
minio server http://host{1...32}/export{1...32}
```
__NOTE:__ `{1...n}` shown have 3 dots! Using only 2 dots `{1..32}` will be interpreted by your shell and won't be passed to minio server, affecting the erasure coding order, which may impact performance and high availability. __Always use ellipses syntax `{1...n}` (3 dots!) for optimal erasure-code distribution__
> __NOTE:__ `{1...n}` shown have 3 dots! Using only 2 dots `{1..32}` will be interpreted by your shell and won't be passed to MinIO server, affecting the erasure coding order, which may impact performance and high availability. __Always use ellipses syntax `{1...n}` (3 dots!) for optimal erasure-code distribution__
#### Expanding existing distributed setup
MinIO supports expanding distributed erasure coded clusters by specifying new set of clusters on the command-line as shown below:
```sh
export MINIO_ACCESS_KEY=<ACCESS_KEY>
export MINIO_SECRET_KEY=<SECRET_KEY>
minio server http://host{1...32}/export{1...32} http://host{33...64}/export{1...32}
```
Now the server has expanded storage of *1024* more disks in total of *2048* disks, new object upload requests automatically start using the least used cluster. This expansion strategy works endlessly, so you can perpetually expand your clusters as needed.
## 3. Test your setup
To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/docs/minio-client-quickstart-guide).

6
docs/minio-limits.md
View file

@ -4,10 +4,10 @@
|Item|Specification|
|:---|:---|
|Maximum number of servers per cluster| Unlimited|
|Maximum number of federated clusters | Unlimited|
|Maximum number of servers per cluster| no-limit|
|Maximum number of federated clusters | no-limit|
|Minimum number of servers| 02|
|Maximum number of drives per server| Unlimited|
|Maximum number of drives per server| no-limit|
|Read quorum| N/2|
|Write quorum| N/2+1|

6
go.sum
View file

@ -499,6 +499,8 @@ github.com/nats-io/go-nats-streaming v0.0.0-20161216191029-077898146bfb/go.mod h
github.com/nats-io/go-nats-streaming v0.4.2/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo=
github.com/nats-io/go-nats-streaming v0.4.4 h1:1I3lkZDRdQYXb+holjdqZ2J6xyekrD06o9Fd8rWlgP4=
github.com/nats-io/go-nats-streaming v0.4.4/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo=
github.com/nats-io/jwt v0.3.0 h1:xdnzwFETV++jNc4W1mw//qFyJGb2ABOombmZJQS4+Qo=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
@ -517,7 +519,11 @@ github.com/nats-io/nats.go v1.8.0/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ
github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ=
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
github.com/nats-io/nkeys v0.0.2 h1:+qM7QpgXnvDDixitZtQUBDY9w/s9mu1ghS+JIbsrx6M=
github.com/nats-io/nkeys v0.0.2 h1:+qM7QpgXnvDDixitZtQUBDY9w/s9mu1ghS+JIbsrx6M=
github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
github.com/nats-io/nkeys v0.1.0 h1:qMd4+pRHgdr1nAClu+2h/2a5F2TmKcCzjCDazVgRoX4=
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=

59
pkg/dsync/drwmutex.go
View file

@ -20,6 +20,7 @@ import (
"context"
"fmt"
golog "log"
"math"
"math/rand"
"os"
"path"
@ -75,7 +76,7 @@ func isLocked(uid string) bool {
func NewDRWMutex(ctx context.Context, name string, clnt *Dsync) *DRWMutex {
return &DRWMutex{
Name: name,
writeLocks: make([]string, clnt.dNodeCount),
writeLocks: make([]string, len(clnt.GetLockersFn())),
clnt: clnt,
ctx: ctx,
}
@ -133,6 +134,8 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, id, source string, isRea
doneCh, start := make(chan struct{}), time.Now().UTC()
defer close(doneCh)
restClnts := dm.clnt.GetLockersFn()
// Use incremental back-off algorithm for repeated attempts to acquire the lock
for range newRetryTimerSimple(doneCh) {
select {
@ -142,7 +145,7 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, id, source string, isRea
}
// Create temp array on stack.
locks := make([]string, dm.clnt.dNodeCount)
locks := make([]string, len(restClnts))
// Try to acquire the lock.
success := lock(dm.clnt, &locks, dm.Name, id, source, isReadLock)
@ -152,7 +155,7 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, id, source string, isRea
// If success, copy array to object
if isReadLock {
// Append new array of strings at the end
dm.readersLocks = append(dm.readersLocks, make([]string, dm.clnt.dNodeCount))
dm.readersLocks = append(dm.readersLocks, make([]string, len(restClnts)))
// and copy stack array into last spot
copy(dm.readersLocks[len(dm.readersLocks)-1], locks[:])
} else {
@ -174,12 +177,14 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, id, source string, isRea
// lock tries to acquire the distributed lock, returning true or false.
func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bool) bool {
restClnts := ds.GetLockersFn()
// Create buffered channel of size equal to total number of nodes.
ch := make(chan Granted, ds.dNodeCount)
ch := make(chan Granted, len(restClnts))
defer close(ch)
var wg sync.WaitGroup
for index, c := range ds.restClnts {
for index, c := range restClnts {
wg.Add(1)
// broadcast lock request to all nodes
@ -229,7 +234,10 @@ func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bo
done := false
timeout := time.After(DRWMutexAcquireTimeout)
for ; i < ds.dNodeCount; i++ { // Loop until we acquired all locks
dquorum := int(len(restClnts)/2) + 1
dquorumReads := int(math.Ceil(float64(len(restClnts)) / 2.0))
for ; i < len(restClnts); i++ { // Loop until we acquired all locks
select {
case grant := <-ch:
@ -238,22 +246,22 @@ func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bo
(*locks)[grant.index] = grant.lockUID
} else {
locksFailed++
if !isReadLock && locksFailed > ds.dNodeCount-ds.dquorum ||
isReadLock && locksFailed > ds.dNodeCount-ds.dquorumReads {
if !isReadLock && locksFailed > len(restClnts)-dquorum ||
isReadLock && locksFailed > len(restClnts)-dquorumReads {
// We know that we are not going to get the lock anymore,
// so exit out and release any locks that did get acquired
done = true
// Increment the number of grants received from the buffered channel.
i++
releaseAll(ds, locks, lockName, isReadLock)
releaseAll(ds, locks, lockName, isReadLock, restClnts)
}
}
case <-timeout:
done = true
// timeout happened, maybe one of the nodes is slow, count
// number of locks to check whether we have quorum or not
if !quorumMet(locks, isReadLock, ds.dquorum, ds.dquorumReads) {
releaseAll(ds, locks, lockName, isReadLock)
if !quorumMet(locks, isReadLock, dquorum, dquorumReads) {
releaseAll(ds, locks, lockName, isReadLock, restClnts)
}
}
@ -263,7 +271,7 @@ func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bo
}
// Count locks in order to determine whether we have quorum or not
quorum = quorumMet(locks, isReadLock, ds.dquorum, ds.dquorumReads)
quorum = quorumMet(locks, isReadLock, dquorum, dquorumReads)
// Signal that we have the quorum
wg.Done()
@ -271,11 +279,12 @@ func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bo
// Wait for the other responses and immediately release the locks
// (do not add them to the locks array because the DRWMutex could
// already has been unlocked again by the original calling thread)
for ; i < ds.dNodeCount; i++ {
for ; i < len(restClnts); i++ {
grantToBeReleased := <-ch
if grantToBeReleased.isLocked() {
// release lock
sendRelease(ds, ds.restClnts[grantToBeReleased.index], lockName, grantToBeReleased.lockUID, isReadLock)
sendRelease(ds, restClnts[grantToBeReleased.index], lockName,
grantToBeReleased.lockUID, isReadLock)
}
}
}(isReadLock)
@ -306,10 +315,10 @@ func quorumMet(locks *[]string, isReadLock bool, quorum, quorumReads int) bool {
}
// releaseAll releases all locks that are marked as locked
func releaseAll(ds *Dsync, locks *[]string, lockName string, isReadLock bool) {
for lock := 0; lock < ds.dNodeCount; lock++ {
func releaseAll(ds *Dsync, locks *[]string, lockName string, isReadLock bool, restClnts []NetLocker) {
for lock := 0; lock < len(restClnts); lock++ {
if isLocked((*locks)[lock]) {
sendRelease(ds, ds.restClnts[lock], lockName, (*locks)[lock], isReadLock)
sendRelease(ds, restClnts[lock], lockName, (*locks)[lock], isReadLock)
(*locks)[lock] = ""
}
}
@ -320,8 +329,9 @@ func releaseAll(ds *Dsync, locks *[]string, lockName string, isReadLock bool) {
// It is a run-time error if dm is not locked on entry to Unlock.
func (dm *DRWMutex) Unlock() {
restClnts := dm.clnt.GetLockersFn()
// create temp array on stack
locks := make([]string, dm.clnt.dNodeCount)
locks := make([]string, len(restClnts))
{
dm.m.Lock()
@ -342,11 +352,11 @@ func (dm *DRWMutex) Unlock() {
// Copy write locks to stack array
copy(locks, dm.writeLocks[:])
// Clear write locks array
dm.writeLocks = make([]string, dm.clnt.dNodeCount)
dm.writeLocks = make([]string, len(restClnts))
}
isReadLock := false
unlock(dm.clnt, locks, dm.Name, isReadLock)
unlock(dm.clnt, locks, dm.Name, isReadLock, restClnts)
}
// RUnlock releases a read lock held on dm.
@ -355,8 +365,9 @@ func (dm *DRWMutex) Unlock() {
func (dm *DRWMutex) RUnlock() {
// create temp array on stack
locks := make([]string, dm.clnt.dNodeCount)
restClnts := dm.clnt.GetLockersFn()
locks := make([]string, len(restClnts))
{
dm.m.Lock()
defer dm.m.Unlock()
@ -370,15 +381,15 @@ func (dm *DRWMutex) RUnlock() {
}
isReadLock := true
unlock(dm.clnt, locks, dm.Name, isReadLock)
unlock(dm.clnt, locks, dm.Name, isReadLock, restClnts)
}
func unlock(ds *Dsync, locks []string, name string, isReadLock bool) {
func unlock(ds *Dsync, locks []string, name string, isReadLock bool, restClnts []NetLocker) {
// We don't need to synchronously wait until we have released all the locks (or the quorum)
// (a subsequent lock will retry automatically in case it would fail to get quorum)
for index, c := range ds.restClnts {
for index, c := range restClnts {
if isLocked(locks[index]) {
// broadcast lock release to all nodes that granted the lock

38
pkg/dsync/dsync.go
View file

@ -16,45 +16,9 @@
package dsync
import (
"errors"
"math"
)
// Dsync represents dsync client object which is initialized with
// authenticated clients, used to initiate lock REST calls.
type Dsync struct {
// Number of nodes participating in the distributed locking.
dNodeCount int
// List of rest client objects, one per lock server.
restClnts []NetLocker
// Simple majority based quorum, set to dNodeCount/2+1
dquorum int
// Simple quorum for read operations, set to dNodeCount/2
dquorumReads int
}
// New - initializes a new dsync object with input restClnts.
func New(restClnts []NetLocker) (*Dsync, error) {
if len(restClnts) < 2 {
return nil, errors.New("Dsync is not designed for less than 2 nodes")
} else if len(restClnts) > 32 {
return nil, errors.New("Dsync is not designed for more than 32 nodes")
}
ds := &Dsync{}
ds.dNodeCount = len(restClnts)
// With odd number of nodes, write and read quorum is basically the same
ds.dquorum = int(ds.dNodeCount/2) + 1
ds.dquorumReads = int(math.Ceil(float64(ds.dNodeCount) / 2.0))
// Initialize node name and rest path for each NetLocker object.
ds.restClnts = make([]NetLocker, ds.dNodeCount)
copy(ds.restClnts, restClnts)
return ds, nil
GetLockersFn func() []NetLocker
}

58
pkg/dsync/dsync_private_test.go
View file

@ -1,58 +0,0 @@
/*
* Minio Cloud Storage, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// GOMAXPROCS=10 go test
package dsync
import "testing"
// Tests dsync.New
func TestNew(t *testing.T) {
nclnts := make([]NetLocker, 33)
if _, err := New(nclnts); err == nil {
t.Fatal("Should have failed")
}
nclnts = make([]NetLocker, 1)
if _, err := New(nclnts); err == nil {
t.Fatal("Should have failed")
}
nclnts = make([]NetLocker, 2)
nds, err := New(nclnts)
if err != nil {
t.Fatal("Should pass", err)
}
if nds.dquorumReads != 1 {
t.Fatalf("Unexpected read quorum values expected 1, got %d", nds.dquorumReads)
}
if nds.dquorum != 2 {
t.Fatalf("Unexpected quorum values expected 2, got %d", nds.dquorum)
}
nclnts = make([]NetLocker, 3)
nds, err = New(nclnts)
if err != nil {
t.Fatal("Should pass", err)
}
if nds.dquorumReads != nds.dquorum {
t.Fatalf("Unexpected quorum values for odd nodes we expect read %d and write %d quorum to be same", nds.dquorumReads, nds.dquorum)
}
}

11
pkg/dsync/dsync_test.go
View file

@ -78,10 +78,8 @@ func TestMain(m *testing.M) {
clnts = append(clnts, newClient(nodes[i], rpcPaths[i]))
}
var err error
ds, err = New(clnts)
if err != nil {
log.Fatalf("set nodes failed with %v", err)
ds = &Dsync{
GetLockersFn: func() []NetLocker { return clnts },
}
startRPCServers(nodes)
@ -256,11 +254,10 @@ func TestMutex(t *testing.T) {
func BenchmarkMutexUncontended(b *testing.B) {
type PaddedMutex struct {
DRWMutex
pad [128]uint8
*DRWMutex
}
b.RunParallel(func(pb *testing.PB) {
var mu PaddedMutex
var mu = PaddedMutex{NewDRWMutex(context.Background(), "", ds)}
for pb.Next() {
mu.Lock(id, source)
mu.Unlock()

8
pkg/dsync/rpc-client-impl_test.go
View file

@ -41,6 +41,14 @@ func newClient(addr, endpoint string) NetLocker {
}
}
// Close closes the underlying socket file descriptor.
func (rpcClient *ReconnectRPCClient) IsOnline() bool {
rpcClient.mutex.Lock()
defer rpcClient.mutex.Unlock()
// If rpc client has not connected yet there is nothing to close.
return rpcClient.rpc != nil
}
// Close closes the underlying socket file descriptor.
func (rpcClient *ReconnectRPCClient) Close() error {
rpcClient.mutex.Lock()

3
pkg/dsync/rpc-client-interface.go
View file

@ -56,4 +56,7 @@ type NetLocker interface {
// Close closes any underlying connection to the service endpoint
Close() error
// Is the underlying connection online? (is always true for any local lockers)
IsOnline() bool
}