minio/cmd/bitrot.go

/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"errors"
	"hash"

	"github.com/minio/highwayhash"
	"github.com/minio/minio/cmd/logger"
	sha256 "github.com/minio/sha256-simd"
	"golang.org/x/crypto/blake2b"
)

// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")

// BitrotAlgorithm specifies a algorithm used for bitrot protection.
type BitrotAlgorithm uint

const (
	// SHA256 represents the SHA-256 hash function
	SHA256 BitrotAlgorithm = 1 + iota
	// HighwayHash256 represents the HighwayHash-256 hash function
	HighwayHash256
	// BLAKE2b512 represents the BLAKE2b-512 hash function
	BLAKE2b512
)

// DefaultBitrotAlgorithm is the default algorithm used for bitrot protection.
const (
	DefaultBitrotAlgorithm = HighwayHash256
)

var bitrotAlgorithms = map[BitrotAlgorithm]string{
	SHA256:         "sha256",
	BLAKE2b512:     "blake2b",
	HighwayHash256: "highwayhash256",
}

// New returns a new hash.Hash calculating the given bitrot algorithm.
func (a BitrotAlgorithm) New() hash.Hash {
	switch a {
	case SHA256:
		return sha256.New()
	case BLAKE2b512:
		b2, _ := blake2b.New512(nil) // New512 never returns an error if the key is nil
		return b2
	case HighwayHash256:
		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
		return hh
	default:
		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
		return nil
	}
}

// Available reports whether the given algorithm is available.
func (a BitrotAlgorithm) Available() bool {
	_, ok := bitrotAlgorithms[a]
	return ok
}

// String returns the string identifier for a given bitrot algorithm.
// If the algorithm is not supported String panics.
func (a BitrotAlgorithm) String() string {
	name, ok := bitrotAlgorithms[a]
	if !ok {
		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
	}
	return name
}

// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm.
func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier {
	return &BitrotVerifier{algorithm, checksum}
}

// BitrotVerifier can be used to verify protected data.
type BitrotVerifier struct {
	algorithm BitrotAlgorithm
	sum       []byte
}

// BitrotAlgorithmFromString returns a bitrot algorithm from the given string representation.
// It returns 0 if the string representation does not match any supported algorithm.
// The zero value of a bitrot algorithm is never supported.
func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
	for alg, name := range bitrotAlgorithms {
		if name == s {
			return alg
		}
	}
	return
}

// To read bit-rot verified data.
type bitrotReader struct {
	disk      StorageAPI
	volume    string
	filePath  string
	verifier  *BitrotVerifier // Holds the bit-rot info
	endOffset int64           // Affects the length of data requested in disk.ReadFile depending on Read()'s offset
	buf       []byte          // Holds bit-rot verified data
}

// newBitrotReader returns bitrotReader.
// Note that the buffer is allocated later in Read(). This is because we will know the buffer length only
// during the bitrotReader.Read(). Depending on when parallelReader fails-over, the buffer length can be different.
func newBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, endOffset int64, sum []byte) *bitrotReader {
	return &bitrotReader{
		disk:      disk,
		volume:    volume,
		filePath:  filePath,
		verifier:  &BitrotVerifier{algo, sum},
		endOffset: endOffset,
		buf:       nil,
	}
}

// ReadChunk returns requested data.
func (b *bitrotReader) ReadChunk(offset int64, length int64) ([]byte, error) {
	if b.buf == nil {
		b.buf = make([]byte, b.endOffset-offset)
		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
			ctx := context.Background()
			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
			logger.LogIf(ctx, err)
			return nil, err
		}
	}
	if int64(len(b.buf)) < length {
		logger.LogIf(context.Background(), errLessData)
		return nil, errLessData
	}
	retBuf := b.buf[:length]
	b.buf = b.buf[length:]
	return retBuf, nil
}

// To calculate the bit-rot of the written data.
type bitrotWriter struct {
	disk     StorageAPI
	volume   string
	filePath string
	h        hash.Hash
}

// newBitrotWriter returns bitrotWriter.
func newBitrotWriter(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm) *bitrotWriter {
	return &bitrotWriter{
		disk:     disk,
		volume:   volume,
		filePath: filePath,
		h:        algo.New(),
	}
}

// Append appends the data and while calculating the hash.
func (b *bitrotWriter) Append(buf []byte) error {
	n, err := b.h.Write(buf)
	if err != nil {
		return err
	}
	if n != len(buf) {
		logger.LogIf(context.Background(), errUnexpected)
		return errUnexpected
	}
	if err = b.disk.AppendFile(b.volume, b.filePath, buf); err != nil {
		logger.LogIf(context.Background(), err)
		return err
	}
	return nil
}

// Sum returns bit-rot sum.
func (b *bitrotWriter) Sum() []byte {
	return b.h.Sum(nil)
}