2021-04-18 19:41:13 +00:00
|
|
|
// Copyright (c) 2015-2021 MinIO, Inc.
|
|
|
|
//
|
|
|
|
// This file is part of MinIO Object Storage stack
|
|
|
|
//
|
|
|
|
// This program is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Affero General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
2019-01-17 12:58:18 +00:00
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2020-09-04 16:45:06 +00:00
|
|
|
"context"
|
2019-01-17 12:58:18 +00:00
|
|
|
"encoding/hex"
|
2019-10-01 20:12:15 +00:00
|
|
|
"fmt"
|
2019-01-17 12:58:18 +00:00
|
|
|
"hash"
|
|
|
|
"io"
|
2022-10-12 11:28:21 +00:00
|
|
|
"strings"
|
2021-05-11 16:18:37 +00:00
|
|
|
"sync"
|
2019-01-17 12:58:18 +00:00
|
|
|
|
2021-06-01 21:59:40 +00:00
|
|
|
xhttp "github.com/minio/minio/internal/http"
|
|
|
|
"github.com/minio/minio/internal/ioutil"
|
|
|
|
"github.com/minio/minio/internal/logger"
|
2019-01-17 12:58:18 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Calculates bitrot in chunks and writes the hash into the stream.
|
|
|
|
type streamingBitrotWriter struct {
|
2021-03-30 00:00:55 +00:00
|
|
|
iow io.WriteCloser
|
|
|
|
closeWithErr func(err error) error
|
|
|
|
h hash.Hash
|
|
|
|
shardSize int64
|
2021-05-11 16:18:37 +00:00
|
|
|
canClose *sync.WaitGroup
|
2019-01-17 12:58:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
|
|
|
|
if len(p) == 0 {
|
|
|
|
return 0, nil
|
|
|
|
}
|
|
|
|
b.h.Reset()
|
|
|
|
b.h.Write(p)
|
|
|
|
hashBytes := b.h.Sum(nil)
|
2019-05-02 04:46:00 +00:00
|
|
|
_, err := b.iow.Write(hashBytes)
|
|
|
|
if err != nil {
|
2021-05-17 15:32:28 +00:00
|
|
|
b.closeWithErr(err)
|
2019-01-17 12:58:18 +00:00
|
|
|
return 0, err
|
|
|
|
}
|
2021-05-17 15:32:28 +00:00
|
|
|
n, err := b.iow.Write(p)
|
|
|
|
if err != nil {
|
|
|
|
b.closeWithErr(err)
|
|
|
|
return n, err
|
|
|
|
}
|
2021-08-27 16:16:36 +00:00
|
|
|
if n != len(p) {
|
|
|
|
err = io.ErrShortWrite
|
|
|
|
b.closeWithErr(err)
|
|
|
|
}
|
2021-05-17 15:32:28 +00:00
|
|
|
return n, err
|
2019-01-17 12:58:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (b *streamingBitrotWriter) Close() error {
|
|
|
|
err := b.iow.Close()
|
|
|
|
// Wait for all data to be written before returning else it causes race conditions.
|
|
|
|
// Race condition is because of io.PipeWriter implementation. i.e consider the following
|
|
|
|
// sequent of operations:
|
|
|
|
// 1) pipe.Write()
|
|
|
|
// 2) pipe.Close()
|
|
|
|
// Now pipe.Close() can return before the data is read on the other end of the pipe and written to the disk
|
|
|
|
// Hence an immediate Read() on the file can return incorrect data.
|
2021-03-30 00:00:55 +00:00
|
|
|
if b.canClose != nil {
|
2021-05-11 16:18:37 +00:00
|
|
|
b.canClose.Wait()
|
2021-03-30 00:00:55 +00:00
|
|
|
}
|
2019-01-17 12:58:18 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-05-17 15:32:28 +00:00
|
|
|
// newStreamingBitrotWriterBuffer returns streaming bitrot writer implementation.
|
|
|
|
// The output is written to the supplied writer w.
|
|
|
|
func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize int64) io.Writer {
|
|
|
|
return &streamingBitrotWriter{iow: ioutil.NopCloser(w), h: algo.New(), shardSize: shardSize, canClose: nil, closeWithErr: func(err error) error {
|
|
|
|
// Similar to CloseWithError on pipes we always return nil.
|
|
|
|
return nil
|
|
|
|
}}
|
2021-03-30 00:00:55 +00:00
|
|
|
}
|
|
|
|
|
2019-01-17 12:58:18 +00:00
|
|
|
// Returns streaming bitrot writer implementation.
|
2021-05-17 15:32:28 +00:00
|
|
|
func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
|
2019-01-17 12:58:18 +00:00
|
|
|
r, w := io.Pipe()
|
|
|
|
h := algo.New()
|
2021-03-18 21:09:55 +00:00
|
|
|
|
2021-05-11 16:18:37 +00:00
|
|
|
bw := &streamingBitrotWriter{iow: w, closeWithErr: w.CloseWithError, h: h, shardSize: shardSize, canClose: &sync.WaitGroup{}}
|
|
|
|
bw.canClose.Add(1)
|
2019-01-17 12:58:18 +00:00
|
|
|
go func() {
|
2019-04-30 23:27:31 +00:00
|
|
|
totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
|
|
|
|
if length != -1 {
|
|
|
|
bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
|
|
|
|
totalFileSize = bitrotSumsTotalSize + length
|
|
|
|
}
|
2021-03-18 21:09:55 +00:00
|
|
|
r.CloseWithError(disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r))
|
2021-05-11 16:18:37 +00:00
|
|
|
bw.canClose.Done()
|
2019-01-17 12:58:18 +00:00
|
|
|
}()
|
|
|
|
return bw
|
|
|
|
}
|
|
|
|
|
|
|
|
// ReadAt() implementation which verifies the bitrot hash available as part of the stream.
|
|
|
|
type streamingBitrotReader struct {
|
|
|
|
disk StorageAPI
|
2021-01-08 03:27:31 +00:00
|
|
|
data []byte
|
2021-01-03 19:27:57 +00:00
|
|
|
rc io.Reader
|
2019-01-17 12:58:18 +00:00
|
|
|
volume string
|
|
|
|
filePath string
|
|
|
|
tillOffset int64
|
|
|
|
currOffset int64
|
|
|
|
h hash.Hash
|
|
|
|
shardSize int64
|
|
|
|
hashBytes []byte
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *streamingBitrotReader) Close() error {
|
|
|
|
if b.rc == nil {
|
|
|
|
return nil
|
|
|
|
}
|
2021-01-03 19:27:57 +00:00
|
|
|
if closer, ok := b.rc.(io.Closer); ok {
|
2021-05-04 17:12:08 +00:00
|
|
|
// drain the body for connection re-use at network layer.
|
|
|
|
xhttp.DrainBody(struct {
|
|
|
|
io.Reader
|
|
|
|
io.Closer
|
|
|
|
}{
|
|
|
|
Reader: b.rc,
|
|
|
|
Closer: closeWrapper(func() error { return nil }),
|
|
|
|
})
|
2021-01-03 19:27:57 +00:00
|
|
|
return closer.Close()
|
|
|
|
}
|
|
|
|
return nil
|
2019-01-17 12:58:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
|
|
|
var err error
|
|
|
|
if offset%b.shardSize != 0 {
|
|
|
|
// Offset should always be aligned to b.shardSize
|
2019-05-02 04:46:00 +00:00
|
|
|
// Can never happen unless there are programmer bugs
|
2019-01-17 12:58:18 +00:00
|
|
|
return 0, errUnexpected
|
|
|
|
}
|
2022-10-12 11:28:21 +00:00
|
|
|
ignoredErrs := []error{
|
|
|
|
errDiskNotFound,
|
|
|
|
}
|
|
|
|
if strings.HasPrefix(b.volume, minioMetaBucket) {
|
|
|
|
ignoredErrs = append(ignoredErrs,
|
|
|
|
errFileNotFound,
|
|
|
|
errVolumeNotFound,
|
|
|
|
errFileVersionNotFound,
|
|
|
|
)
|
|
|
|
}
|
2019-01-17 12:58:18 +00:00
|
|
|
if b.rc == nil {
|
|
|
|
// For the first ReadAt() call we need to open the stream for reading.
|
|
|
|
b.currOffset = offset
|
|
|
|
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
|
2021-03-30 00:00:55 +00:00
|
|
|
if len(b.data) == 0 && b.tillOffset != streamOffset {
|
2021-01-08 03:27:31 +00:00
|
|
|
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
|
2022-10-12 11:28:21 +00:00
|
|
|
if err != nil {
|
|
|
|
if !IsErr(err, ignoredErrs...) {
|
|
|
|
logger.LogIf(GlobalContext,
|
|
|
|
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
|
|
|
|
b.disk, b.volume, b.filePath, err))
|
|
|
|
}
|
2021-05-14 23:50:47 +00:00
|
|
|
}
|
2021-01-08 03:27:31 +00:00
|
|
|
} else {
|
|
|
|
b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
|
|
|
|
}
|
2021-01-05 02:51:52 +00:00
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
2019-01-17 12:58:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if offset != b.currOffset {
|
2019-05-02 04:46:00 +00:00
|
|
|
// Can never happen unless there are programmer bugs
|
2019-01-17 12:58:18 +00:00
|
|
|
return 0, errUnexpected
|
|
|
|
}
|
|
|
|
b.h.Reset()
|
|
|
|
_, err = io.ReadFull(b.rc, b.hashBytes)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
_, err = io.ReadFull(b.rc, buf)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
b.h.Write(buf)
|
|
|
|
|
2019-02-13 12:59:36 +00:00
|
|
|
if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
|
2022-08-04 23:10:08 +00:00
|
|
|
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s - content hash does not match - expected %s, got %s",
|
2021-01-27 18:21:14 +00:00
|
|
|
b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
|
|
|
|
return 0, errFileCorrupt
|
2019-01-17 12:58:18 +00:00
|
|
|
}
|
|
|
|
b.currOffset += int64(len(buf))
|
|
|
|
return len(buf), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Returns streaming bitrot reader implementation.
|
2021-01-08 03:27:31 +00:00
|
|
|
func newStreamingBitrotReader(disk StorageAPI, data []byte, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
|
2019-01-17 12:58:18 +00:00
|
|
|
h := algo.New()
|
|
|
|
return &streamingBitrotReader{
|
2021-03-30 00:00:55 +00:00
|
|
|
disk: disk,
|
|
|
|
data: data,
|
|
|
|
volume: volume,
|
|
|
|
filePath: filePath,
|
|
|
|
tillOffset: ceilFrac(tillOffset, shardSize)*int64(h.Size()) + tillOffset,
|
|
|
|
h: h,
|
|
|
|
shardSize: shardSize,
|
|
|
|
hashBytes: make([]byte, h.Size()),
|
2019-01-17 12:58:18 +00:00
|
|
|
}
|
|
|
|
}
|