minio/cmd/object-api-input-checks.go
Harshavardhana 80ca120088
remove checkBucketExist check entirely to avoid fan-out calls (#18917)
Each Put, List, Multipart operations heavily rely on making
GetBucketInfo() call to verify if bucket exists or not on
a regular basis. This has a large performance cost when there
are tons of servers involved.

We did optimize this part by vectorizing the bucket calls,
however its not enough, beyond 100 nodes and this becomes
fairly visible in terms of performance.
2024-01-30 12:43:25 -08:00

184 lines
5.9 KiB
Go

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"encoding/base64"
"runtime"
"strings"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio/internal/logger"
)
// Checks on GetObject arguments, bucket and object.
func checkGetObjArgs(ctx context.Context, bucket, object string) error {
return checkBucketAndObjectNames(ctx, bucket, object)
}
// Checks on DeleteObject arguments, bucket and object.
func checkDelObjArgs(ctx context.Context, bucket, object string) error {
return checkBucketAndObjectNames(ctx, bucket, object)
}
// Checks bucket and object name validity, returns nil if both are valid.
func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error {
// Verify if bucket is valid.
if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
return BucketNameInvalid{Bucket: bucket}
}
// Verify if object is valid.
if len(object) == 0 {
return ObjectNameInvalid{Bucket: bucket, Object: object}
}
if !IsValidObjectPrefix(object) {
return ObjectNameInvalid{Bucket: bucket, Object: object}
}
if runtime.GOOS == globalWindowsOSName && strings.Contains(object, "\\") {
// Objects cannot be contain \ in Windows and is listed as `Characters to Avoid`.
return ObjectNameInvalid{Bucket: bucket, Object: object}
}
return nil
}
// Checks for all ListObjects arguments validity.
func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string) error {
// Verify if bucket is valid.
if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
return BucketNameInvalid{Bucket: bucket}
}
// Validates object prefix validity after bucket exists.
if !IsValidObjectPrefix(prefix) {
logger.LogIf(ctx, ObjectNameInvalid{
Bucket: bucket,
Object: prefix,
})
return ObjectNameInvalid{
Bucket: bucket,
Object: prefix,
}
}
return nil
}
// Checks for all ListMultipartUploads arguments validity.
func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string) error {
if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker); err != nil {
return err
}
if uploadIDMarker != "" {
if HasSuffix(keyMarker, SlashSeparator) {
logger.LogIf(ctx, InvalidUploadIDKeyCombination{
UploadIDMarker: uploadIDMarker,
KeyMarker: keyMarker,
})
return InvalidUploadIDKeyCombination{
UploadIDMarker: uploadIDMarker,
KeyMarker: keyMarker,
}
}
_, err := base64.RawURLEncoding.DecodeString(uploadIDMarker)
if err != nil {
logger.LogIf(ctx, err)
return MalformedUploadID{
UploadID: uploadIDMarker,
}
}
}
return nil
}
// Checks for NewMultipartUpload arguments validity, also validates if bucket exists.
func checkNewMultipartArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error {
return checkObjectArgs(ctx, bucket, object, obj)
}
func checkMultipartObjectArgs(ctx context.Context, bucket, object, uploadID string, obj ObjectLayer) error {
_, err := base64.RawURLEncoding.DecodeString(uploadID)
if err != nil {
return MalformedUploadID{
UploadID: uploadID,
}
}
return checkObjectArgs(ctx, bucket, object, obj)
}
// Checks for PutObjectPart arguments validity, also validates if bucket exists.
func checkPutObjectPartArgs(ctx context.Context, bucket, object, uploadID string, obj ObjectLayer) error {
return checkMultipartObjectArgs(ctx, bucket, object, uploadID, obj)
}
// Checks for ListParts arguments validity, also validates if bucket exists.
func checkListPartsArgs(ctx context.Context, bucket, object, uploadID string, obj ObjectLayer) error {
return checkMultipartObjectArgs(ctx, bucket, object, uploadID, obj)
}
// Checks for CompleteMultipartUpload arguments validity, also validates if bucket exists.
func checkCompleteMultipartArgs(ctx context.Context, bucket, object, uploadID string, obj ObjectLayer) error {
return checkMultipartObjectArgs(ctx, bucket, object, uploadID, obj)
}
// Checks for AbortMultipartUpload arguments validity, also validates if bucket exists.
func checkAbortMultipartArgs(ctx context.Context, bucket, object, uploadID string, obj ObjectLayer) error {
return checkMultipartObjectArgs(ctx, bucket, object, uploadID, obj)
}
// Checks Object arguments validity.
func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error {
// Verify if bucket is valid.
if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
return BucketNameInvalid{Bucket: bucket}
}
if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
return err
}
// Validates object name validity after bucket exists.
if !IsValidObjectName(object) {
return ObjectNameInvalid{
Bucket: bucket,
Object: object,
}
}
return nil
}
// Checks for PutObject arguments validity.
func checkPutObjectArgs(ctx context.Context, bucket, object string) error {
// Verify if bucket is valid.
if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
return BucketNameInvalid{Bucket: bucket}
}
if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
return err
}
if len(object) == 0 ||
!IsValidObjectPrefix(object) {
return ObjectNameInvalid{
Bucket: bucket,
Object: object,
}
}
return nil
}