diff --git a/fs-utils.go b/fs-utils.go new file mode 100644 index 000000000..14ac6f1c7 --- /dev/null +++ b/fs-utils.go @@ -0,0 +1,56 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "regexp" + "unicode/utf8" +) + +// validVolname regexp. +var validVolname = regexp.MustCompile(`^.{3,63}$`) + +// isValidVolname verifies a volname name in accordance with object +// layer requirements. +func isValidVolname(volname string) bool { + return validVolname.MatchString(volname) +} + +// Keeping this as lower bound value supporting Linux, Darwin and Windows operating systems. +const pathMax = 4096 + +// isValidPath verifies if a path name is in accordance with FS limitations. +func isValidPath(path string) bool { + // TODO: Make this FSType or Operating system specific. + if len(path) > pathMax || len(path) == 0 { + return false + } + if !utf8.ValidString(path) { + return false + } + return true +} + +// isValidPrefix verifies where the prefix is a valid path. +func isValidPrefix(prefix string) bool { + // Prefix can be empty. + if prefix == "" { + return true + } + // Verify if prefix is a valid path. + return isValidPath(prefix) +} diff --git a/fs.go b/fs.go index 7a19b78f5..36ec72f4f 100644 --- a/fs.go +++ b/fs.go @@ -121,26 +121,63 @@ func checkDiskFree(diskPath string, minFreeDisk int64) (err error) { return nil } +// checkVolumeArg - will convert incoming volume names to +// corresponding valid volume names on the backend in a platform +// compatible way for all operating systems. If volume is not found +// an error is generated. +func (s fsStorage) checkVolumeArg(volume string) (string, error) { + if !isValidVolname(volume) { + return "", errInvalidArgument + } + volumeDir := filepath.Join(s.diskPath, volume) + _, err := os.Stat(volumeDir) + if err == nil { + return volumeDir, nil + } + if os.IsNotExist(err) { + var volumes []os.FileInfo + volumes, err = ioutil.ReadDir(s.diskPath) + if err != nil { + return volumeDir, errVolumeNotFound + } + for _, vol := range volumes { + if vol.IsDir() { + // Verify if lowercase version of the volume + // is equal to the incoming volume, then use the proper name. + if strings.ToLower(vol.Name()) == volume { + volumeDir = filepath.Join(s.diskPath, vol.Name()) + return volumeDir, nil + } + } + } + return volumeDir, errVolumeNotFound + } else if os.IsPermission(err) { + return volumeDir, errVolumeAccessDenied + } + return volumeDir, err +} + // Make a volume entry. func (s fsStorage) MakeVol(volume string) (err error) { - if volume == "" { - return errInvalidArgument - } - if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil { - return err - } - - volumeDir := getVolumeDir(s.diskPath, volume) - if _, err = os.Stat(volumeDir); err == nil { + volumeDir, err := s.checkVolumeArg(volume) + if err == nil { + // Volume already exists, return error. return errVolumeExists } - // Make a volume entry. - if err = os.Mkdir(volumeDir, 0700); err != nil { - return err + // Validate if disk is free. + if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil { + return e } - return nil + // If volume not found create it. + if err == errVolumeNotFound { + // Make a volume entry. + return os.Mkdir(volumeDir, 0700) + } + + // For all other errors return here. + return err } // removeDuplicateVols - remove duplicate volumes. @@ -175,9 +212,15 @@ func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) { // If not directory, ignore all file types. continue } + // Volname on case sensitive fs backends can come in as + // capitalized, but object layer cannot consume it + // directly. Convert it as we see fit. + volName := strings.ToLower(file.Name()) + // Modtime is used as created time. + createdTime := file.ModTime() volInfo := VolInfo{ - Name: file.Name(), - Created: file.ModTime(), + Name: volName, + Created: createdTime, } volsInfo = append(volsInfo, volInfo) } @@ -186,30 +229,13 @@ func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) { return volsInfo, nil } -// getVolumeDir - will convert incoming volume names to -// corresponding valid volume names on the backend in a platform -// compatible way for all operating systems. -func getVolumeDir(diskPath, volume string) string { - volumes, e := ioutil.ReadDir(diskPath) - if e != nil { - return volume - } - for _, vol := range volumes { - // Verify if lowercase version of the volume - // is equal to the incoming volume, then use the proper name. - if strings.ToLower(vol.Name()) == volume { - return filepath.Join(diskPath, vol.Name()) - } - } - return filepath.Join(diskPath, volume) -} - // StatVol - get volume info. func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) { - if volume == "" { - return VolInfo{}, errInvalidArgument + // Verify if volume is valid and it exists. + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return VolInfo{}, err } - volumeDir := getVolumeDir(s.diskPath, volume) // Stat a volume entry. var st os.FileInfo st, err = os.Stat(volumeDir) @@ -219,18 +245,23 @@ func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) { } return VolInfo{}, err } + // Modtime is used as created time since operating systems lack a + // portable way of knowing the actual created time of a directory. + createdTime := st.ModTime() return VolInfo{ - Name: st.Name(), - Created: st.ModTime(), + Name: volume, + Created: createdTime, }, nil } // DeleteVol - delete a volume. func (s fsStorage) DeleteVol(volume string) error { - if volume == "" { - return errInvalidArgument + // Verify if volume is valid and it exists. + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return err } - err := os.Remove(getVolumeDir(s.diskPath, volume)) + err = os.Remove(volumeDir) if err != nil && os.IsNotExist(err) { return errVolumeNotFound } @@ -281,23 +312,13 @@ var specialPrefixes = []string{ // List operation. func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) ([]FileInfo, bool, error) { - if volume == "" { - return nil, true, errInvalidArgument + // Verify if volume is valid and it exists. + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return nil, true, err } - var fileInfos []FileInfo - volumeDir := getVolumeDir(s.diskPath, volume) - // Verify if volume directory exists - if exists, err := isDirExist(volumeDir); !exists { - if err == nil { - return nil, true, errVolumeNotFound - } else if os.IsNotExist(err) { - return nil, true, errVolumeNotFound - } else { - return nil, true, err - } - } if marker != "" { // Verify if marker has prefix. if marker != "" && !strings.HasPrefix(marker, prefix) { @@ -323,6 +344,7 @@ func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, coun // Prefix does not exist, not an error just respond empty list response. return nil, true, nil } else if strings.Contains(err.Error(), "not a directory") { + // Prefix exists as a file. return nil, true, nil } // Rest errors should be treated as failure. @@ -375,26 +397,18 @@ func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, coun // ReadFile - read a file at a given offset. func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) { - if volume == "" || path == "" { - return nil, errInvalidArgument + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return nil, err } - volumeDir := getVolumeDir(s.diskPath, volume) - // Verify if volume directory exists - var exists bool - if exists, err = isDirExist(volumeDir); !exists { - if err == nil { - return nil, errVolumeNotFound - } else if os.IsNotExist(err) { - return nil, errVolumeNotFound - } else { - return nil, err - } - } - filePath := filepath.Join(volumeDir, path) + + filePath := filepath.Join(volumeDir, filepath.FromSlash(path)) file, err := os.Open(filePath) if err != nil { if os.IsNotExist(err) { return nil, errFileNotFound + } else if os.IsPermission(err) { + return nil, errFileAccessDenied } return nil, err } @@ -416,22 +430,12 @@ func (s fsStorage) ReadFile(volume string, path string, offset int64) (readClose // CreateFile - create a file at path. func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) { - if volume == "" || path == "" { - return nil, errInvalidArgument + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return nil, err } - if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil { - return nil, e - } - volumeDir := getVolumeDir(s.diskPath, volume) - // Verify if volume directory exists - if exists, err := isDirExist(volumeDir); !exists { - if err == nil { - return nil, errVolumeNotFound - } else if os.IsNotExist(err) { - return nil, errVolumeNotFound - } else { - return nil, err - } + if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil { + return nil, err } filePath := filepath.Join(volumeDir, path) // Verify if the file already exists and is not of regular type. @@ -445,33 +449,26 @@ func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, // StatFile - get file info. func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) { - if volume == "" || path == "" { - return FileInfo{}, errInvalidArgument - } - volumeDir := getVolumeDir(s.diskPath, volume) - // Verify if volume directory exists - var exists bool - if exists, err = isDirExist(volumeDir); !exists { - if err == nil { - return FileInfo{}, errVolumeNotFound - } else if os.IsNotExist(err) { - return FileInfo{}, errVolumeNotFound - } else { - return FileInfo{}, err - } + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return FileInfo{}, err } - filePath := filepath.Join(volumeDir, path) + filePath := filepath.Join(volumeDir, filepath.FromSlash(path)) st, err := os.Stat(filePath) if err != nil { + // File is really not found. if os.IsNotExist(err) { return FileInfo{}, errFileNotFound } + // File path cannot be verified since one of the parents is a file. if strings.Contains(err.Error(), "not a directory") { return FileInfo{}, errIsNotRegular } + // Return all errors here. return FileInfo{}, err } + // If its a directory its not a regular file. if st.Mode().IsDir() { return FileInfo{}, errIsNotRegular } @@ -486,49 +483,55 @@ func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) { } // deleteFile - delete file path if its empty. -func deleteFile(basePath, deletePath, volume, path string) error { +func deleteFile(basePath, deletePath string) error { if basePath == deletePath { return nil } // Verify if the path exists. - pathSt, e := os.Stat(deletePath) - if e != nil { - return e + pathSt, err := os.Stat(deletePath) + if err != nil { + if os.IsNotExist(err) { + return errFileNotFound + } else if os.IsPermission(err) { + return errFileAccessDenied + } + return err } if pathSt.IsDir() { // Verify if directory is empty. - empty, e := isDirEmpty(deletePath) - if e != nil { - return e + empty, err := isDirEmpty(deletePath) + if err != nil { + return err } if !empty { return nil } } // Attempt to remove path. - if e := os.Remove(deletePath); e != nil { - return e + if err := os.Remove(deletePath); err != nil { + return err } // Recursively go down the next path and delete again. - if e := deleteFile(basePath, filepath.Dir(deletePath), volume, path); e != nil { - return e + if err := deleteFile(basePath, filepath.Dir(deletePath)); err != nil { + return err } return nil } // DeleteFile - delete a file at path. func (s fsStorage) DeleteFile(volume, path string) error { - if volume == "" || path == "" { - return errInvalidArgument + volumeDir, err := s.checkVolumeArg(volume) + if err != nil { + return err } - volumeDir := getVolumeDir(s.diskPath, volume) - // Following code is needed so that we retain "/" suffix if any in - // path argument. Do not use filepath.Join() since it would strip - // off any suffixes. - filePath := s.diskPath + string(os.PathSeparator) + volume + string(os.PathSeparator) + path + // path argument. + filePath := filepath.Join(volumeDir, filepath.FromSlash(path)) + if strings.HasSuffix(filepath.FromSlash(path), string(os.PathSeparator)) { + filePath = filePath + string(os.PathSeparator) + } // Delete file and delete parent directory as well if its empty. - return deleteFile(volumeDir, filePath, volume, path) + return deleteFile(volumeDir, filePath) } diff --git a/object-api-multipart.go b/object-api-multipart.go index 2ebb1aee5..eb60fe472 100644 --- a/object-api-multipart.go +++ b/object-api-multipart.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "path" - "path/filepath" "strconv" "strings" @@ -111,10 +110,13 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke } result.IsTruncated = true newMaxUploads := 0 - prefixPath := bucket + slashPathSeparator + prefix // do not use filepath.Join so that we retain trailing '/' if any - + prefixPath := path.Join(bucket, prefix) + if strings.HasSuffix(prefix, slashPathSeparator) { + // Add back the slash separator removed after 'path.Join'. + prefixPath = prefixPath + slashPathSeparator + } if recursive { - keyMarkerPath := filepath.Join(keyMarker, uploadIDMarker) + keyMarkerPath := path.Join(keyMarker, uploadIDMarker) outerLoop: for { fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, keyMarkerPath, recursive, maxUploads-newMaxUploads) @@ -123,17 +125,17 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke } for _, fi := range fileInfos { keyMarkerPath = fi.Name - fileName := filepath.Base(fi.Name) + fileName := path.Base(fi.Name) if strings.Contains(fileName, ".") { // fileName contains partnumber and md5sum info, skip this. continue } result.Uploads = append(result.Uploads, uploadMetadata{ - Object: filepath.Dir(fi.Name), + Object: path.Dir(fi.Name), UploadID: fileName, Initiated: fi.ModTime, }) - result.NextKeyMarker = filepath.Dir(fi.Name) + result.NextKeyMarker = path.Dir(fi.Name) result.NextUploadIDMarker = fileName newMaxUploads++ if newMaxUploads == maxUploads { @@ -244,7 +246,7 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err return "", probe.NewError(e) } uploadID := uuid.String() - uploadIDFile := filepath.Join(bucket, object, uploadID) + uploadIDFile := path.Join(bucket, object, uploadID) if _, e = o.storage.StatFile(minioMetaVolume, uploadIDFile); e != nil { if e != errFileNotFound { return "", probe.NewError(e) @@ -266,7 +268,7 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err } func (o objectAPI) isUploadIDExist(bucket, object, uploadID string) (bool, error) { - st, e := o.storage.StatFile(minioMetaVolume, filepath.Join(bucket, object, uploadID)) + st, e := o.storage.StatFile(minioMetaVolume, path.Join(bucket, object, uploadID)) if e != nil { if e == errFileNotFound { return false, nil @@ -291,7 +293,7 @@ func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, si } partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex) - fileWriter, e := o.storage.CreateFile(minioMetaVolume, filepath.Join(bucket, object, partSuffix)) + fileWriter, e := o.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix)) if e != nil { if e == errVolumeNotFound { return "", probe.NewError(BucketNotFound{ @@ -356,7 +358,7 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa marker := "" nextPartNumberMarker := 0 if partNumberMarker > 0 { - fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID)+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1) + fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, path.Join(bucket, object, uploadID)+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1) if e != nil { return result, probe.NewError(e) } @@ -365,12 +367,12 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa } marker = fileInfos[0].Name } - fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID)+".", marker, false, maxParts) + fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, path.Join(bucket, object, uploadID)+".", marker, false, maxParts) if e != nil { return result, probe.NewError(InvalidPart{}) } for _, fileInfo := range fileInfos { - fileName := filepath.Base(fileInfo.Name) + fileName := path.Base(fileInfo.Name) splitResult := strings.Split(fileName, ".") partNum, e := strconv.Atoi(splitResult[1]) if e != nil { @@ -415,7 +417,7 @@ func (o objectAPI) CompleteMultipartUpload(bucket string, object string, uploadI for _, part := range parts { partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag) var fileReader io.ReadCloser - fileReader, e = o.storage.ReadFile(minioMetaVolume, filepath.Join(bucket, object, partSuffix), 0) + fileReader, e = o.storage.ReadFile(minioMetaVolume, path.Join(bucket, object, partSuffix), 0) if e != nil { return ObjectInfo{}, probe.NewError(e) } @@ -456,7 +458,8 @@ func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe } marker := "" for { - fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID), marker, false, 1000) + uploadIDFile := path.Join(bucket, object, uploadID) + fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDFile, marker, false, 1000) if e != nil { return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) } diff --git a/object-api.go b/object-api.go index ce32f6071..e0e54ae8f 100644 --- a/object-api.go +++ b/object-api.go @@ -80,10 +80,15 @@ func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) { return nil, probe.NewError(e) } for _, vol := range vols { + // StorageAPI can send volume names which are incompatible + // with buckets, handle it and skip them. if !IsValidBucketName(vol.Name) { continue } - bucketInfos = append(bucketInfos, BucketInfo{vol.Name, vol.Created}) + bucketInfos = append(bucketInfos, BucketInfo{ + Name: vol.Name, + Created: vol.Created, + }) } return bucketInfos, nil } diff --git a/object-handlers.go b/object-handlers.go index 069a73985..865a1981a 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -1012,7 +1012,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } - fmt.Println(string(completeMultipartBytes)) complMultipartUpload := &completeMultipartUpload{} if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil { errorIf(probe.NewError(e), "XML Unmarshal failed", nil) diff --git a/object-utils.go b/object-utils.go index d1df78dd7..c8173d60b 100644 --- a/object-utils.go +++ b/object-utils.go @@ -18,6 +18,7 @@ package main import ( "regexp" + "strings" "unicode/utf8" ) @@ -41,7 +42,27 @@ func IsValidBucketName(bucket string) bool { // IsValidObjectName verifies an object name in accordance with Amazon's // requirements. It cannot exceed 1024 characters and must be a valid UTF8 // string. -// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +// +// See: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +// +// You should avoid the following characters in a key name because of +// significant special handling for consistency across all +// applications. +// +// Rejects strings with following characters. +// +// - Backslash ("\") +// - Left curly brace ("{") +// - Caret ("^") +// - Right curly brace ("}") +// - Grave accent / back tick ("`") +// - Right square bracket ("]") +// - Left square bracket ("[") +// - Tilde ("~") +// - 'Greater Than' symbol (">") +// - 'Less Than' symbol ("<") +// - Vertical bar / pipe ("|") func IsValidObjectName(object string) bool { if len(object) > 1024 || len(object) == 0 { return false @@ -49,7 +70,8 @@ func IsValidObjectName(object string) bool { if !utf8.ValidString(object) { return false } - return true + // Reject unsupported characters in object name. + return !strings.ContainsAny(object, "`^*{}[]|\\\"'") } // IsValidObjectPrefix verifies whether the prefix is a valid object name. diff --git a/server_test.go b/server_test.go index 1def4cbd5..2f5d9c044 100644 --- a/server_test.go +++ b/server_test.go @@ -1172,6 +1172,7 @@ func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) { c.Assert(err, IsNil) response, err = client.Do(request) + c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) decoder := xml.NewDecoder(response.Body) diff --git a/storage-errors.go b/storage-errors.go index 143f55ef1..5b50ed9b9 100644 --- a/storage-errors.go +++ b/storage-errors.go @@ -32,3 +32,11 @@ var errIsNotRegular = errors.New("Not a regular file type.") // errVolumeNotFound - cannot find the volume. var errVolumeNotFound = errors.New("Volume not found.") + +// errVolumeAccessDenied - cannot access volume, insufficient +// permissions. +var errVolumeAccessDenied = errors.New("Volume access denied.") + +// errVolumeAccessDenied - cannot access file, insufficient +// permissions. +var errFileAccessDenied = errors.New("File access denied.")