self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-10-02 22:24:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add padding to compressed+encrypted files (#15282)

Browse source 
Add up to 256 bytes of padding for compressed+encrypted files.

This will obscure the obvious cases of extremely compressible content 
and leave a similar output size for a very wide variety of inputs.

This does *not* mean the compression ratio doesn't leak information 
about the content, but the outcome space is much smaller, 
so often *less* information is leaked.
This commit is contained in:
Klaus Post 2022-07-13 07:52:15 -07:00 committed by GitHub
parent 697c9973a7
commit 0149382cdc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 64 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/bucket-handlers.go
View file

@ -865,7 +865,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return
}
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
@ -1041,7 +1041,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return
}
if objectAPI.IsEncryptionSupported() {
if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
if crypto.Requested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
if crypto.SSECopy.IsRequested(r.Header) {
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
return

9
cmd/erasure-object.go
View file

@ -976,14 +976,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
switch size := data.Size(); {
case size == 0:
buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
case size == -1:
if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
buffer = make([]byte, data.ActualSize()+256, data.ActualSize()*2+512)
} else {
buffer = er.bp.Get()
defer er.bp.Put(buffer)
}
case size >= fi.Erasure.BlockSize:
case size >= fi.Erasure.BlockSize || size == -1:
buffer = er.bp.Get()
defer er.bp.Put(buffer)
case size < fi.Erasure.BlockSize:

24
cmd/object-api-utils.go
View file

@ -74,6 +74,10 @@ const (
compReadAheadBuffers = 5
// Size of each buffer.
compReadAheadBufSize = 1 << 20
// Pad Encrypted+Compressed files to a multiple of this.
compPadEncrypted = 256
// Disable compressed file indices below this size
compMinIndexSize = 8 << 20
)
// isMinioBucket returns true if given bucket is a MinIO internal
@ -436,8 +440,7 @@ func isCompressible(header http.Header, object string) bool {
cfg := globalCompressConfig
globalCompressConfigMu.Unlock()
_, ok := crypto.IsRequested(header)
if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
if !cfg.Enabled || (crypto.Requested(header) && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
return false
}
return true
@ -985,10 +988,17 @@ func init() {
// input 'on' is always recommended such that this function works
// properly, because we do not wish to create an object even if
// client closed the stream prematurely.
func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() []byte) {
func newS2CompressReader(r io.Reader, on int64, encrypted bool) (rc io.ReadCloser, idx func() []byte) {
pr, pw := io.Pipe()
// Copy input to compressor
comp := s2.NewWriter(pw, compressOpts...)
opts := compressOpts
if encrypted {
// The values used for padding are not a security concern,
// but we choose pseudo-random numbers instead of just zeros.
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
opts = append([]s2.WriterOption{s2.WriterPadding(compPadEncrypted), s2.WriterPaddingSrc(rng)}, compressOpts...)
}
comp := s2.NewWriter(pw, opts...)
indexCh := make(chan []byte, 1)
go func() {
defer close(indexCh)
@ -1006,8 +1016,8 @@ func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() []
return
}
// Close the stream.
// If more than 8MB was written, generate index.
if cn > 8<<20 {
// If more than compMinIndexSize was written, generate index.
if cn > compMinIndexSize {
idx, err := comp.CloseIndex()
idx = s2.RemoveIndexHeaders(idx)
indexCh <- idx
@ -1048,7 +1058,7 @@ func compressSelfTest() {
}
}
const skip = 2<<20 + 511
r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)))
r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)), true)
b, err := io.ReadAll(r)
failOnErr(err)
failOnErr(r.Close())

2
cmd/object-api-utils_test.go
View file

@ -624,7 +624,7 @@ func TestS2CompressReader(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case
r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)))
r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)), false)
defer r.Close()
var rdrBuf bytes.Buffer

41
cmd/object-handlers.go
View file

@ -119,7 +119,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
return
}
if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
@ -330,7 +330,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
@ -611,7 +611,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest))
return
}
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
@ -951,7 +951,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return
}
if _, ok := crypto.IsRequested(r.Header); ok {
if crypto.Requested(r.Header) {
if globalIsGateway {
if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@ -1151,7 +1151,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
// Check if either the source is encrypted or the destination will be encrypted.
_, objectEncryption := crypto.IsRequested(r.Header)
objectEncryption := crypto.Requested(r.Header)
objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined)
var compressMetadata map[string]string
@ -1168,7 +1168,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
reader = etag.NewReader(reader, nil)
s2c, cb := newS2CompressReader(reader, actualSize)
wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption)
dstOpts.IndexCB = cb
defer s2c.Close()
reader = etag.Wrap(s2c, reader)
@ -1573,7 +1574,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
}
if _, ok := crypto.IsRequested(r.Header); ok {
if crypto.Requested(r.Header) {
if globalIsGateway {
if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@ -1737,7 +1738,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
// Set compression metrics.
var s2c io.ReadCloser
s2c, idxCb = newS2CompressReader(actualReader, actualSize)
wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
s2c, idxCb = newS2CompressReader(actualReader, actualSize, wantEncryption)
defer s2c.Close()
reader = etag.Wrap(s2c, actualReader)
@ -1796,7 +1798,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
var objectEncryptionKey crypto.ObjectKey
if objectAPI.IsEncryptionSupported() {
if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
if crypto.Requested(r.Header) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
if crypto.SSECopy.IsRequested(r.Header) {
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
return
@ -1929,7 +1931,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
return
}
if _, ok := crypto.IsRequested(r.Header); ok {
if crypto.Requested(r.Header) {
if globalIsGateway {
if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@ -2088,7 +2090,8 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
}
// Set compression metrics.
s2c, cb := newS2CompressReader(actualReader, actualSize)
wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption)
defer s2c.Close()
idxCb = cb
reader = etag.Wrap(s2c, actualReader)
@ -2144,7 +2147,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
var objectEncryptionKey crypto.ObjectKey
if objectAPI.IsEncryptionSupported() {
if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
if crypto.Requested(r.Header) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
if crypto.SSECopy.IsRequested(r.Header) {
return errInvalidEncryptionParameters
}
@ -2234,7 +2237,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
return
}
if _, ok := crypto.IsRequested(r.Header); ok {
if crypto.Requested(r.Header) {
if globalIsGateway {
if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@ -2279,7 +2282,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
encMetadata := map[string]string{}
if objectAPI.IsEncryptionSupported() {
if _, ok := crypto.IsRequested(r.Header); ok {
if crypto.Requested(r.Header) {
if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@ -2377,7 +2380,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
return
}
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
@ -2594,7 +2597,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
// Compress only if the compression is enabled during initial multipart.
var idxCb func() []byte
if isCompressed {
s2c, cb := newS2CompressReader(reader, actualPartSize)
wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
s2c, cb := newS2CompressReader(reader, actualPartSize, wantEncryption)
idxCb = cb
defer s2c.Close()
reader = etag.Wrap(s2c, reader)
@ -2709,7 +2713,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
return
}
if _, ok := crypto.IsRequested(r.Header); ok {
if crypto.Requested(r.Header) {
if globalIsGateway {
if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@ -2857,7 +2861,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
}
// Set compression metrics.
s2c, cb := newS2CompressReader(actualReader, actualSize)
wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption)
idxCb = cb
defer s2c.Close()
reader = etag.Wrap(s2c, actualReader)

4
cmd/s3-zip-handlers.go
View file

@ -66,7 +66,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
@ -357,7 +357,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest))
return
}
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}

2
internal/bucket/encryption/bucket-sse-config.go
View file

@ -128,7 +128,7 @@ type ApplyOptions struct {
// set minimal SSE-KMS headers if autoEncrypt is true and the BucketSSEConfig
// is nil.
func (b *BucketSSEConfig) Apply(headers http.Header, opts ApplyOptions) {
if _, ok := crypto.IsRequested(headers); ok {
if crypto.Requested(headers) {
return
}
if b == nil {

12
internal/crypto/header_test.go
View file

@ -29,6 +29,10 @@ import (
func TestIsRequested(t *testing.T) {
for i, test := range kmsIsRequestedTests {
_, got := IsRequested(test.Header)
if Requested(test.Header) != got {
// Test if result matches.
t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got)
}
got = got && S3KMS.IsRequested(test.Header)
if got != test.Expected {
t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
@ -36,6 +40,10 @@ func TestIsRequested(t *testing.T) {
}
for i, test := range s3IsRequestedTests {
_, got := IsRequested(test.Header)
if Requested(test.Header) != got {
// Test if result matches.
t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got)
}
got = got && S3.IsRequested(test.Header)
if got != test.Expected {
t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
@ -43,6 +51,10 @@ func TestIsRequested(t *testing.T) {
}
for i, test := range ssecIsRequestedTests {
_, got := IsRequested(test.Header)
if Requested(test.Header) != got {
// Test if result matches.
t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got)
}
got = got && SSEC.IsRequested(test.Header)
if got != test.Expected {
t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)

5
internal/crypto/sse.go
View file

@ -71,6 +71,11 @@ func IsRequested(h http.Header) (Type, bool) {
}
}
// Requested returns whether any type of encryption is requested.
func Requested(h http.Header) bool {
return S3.IsRequested(h) || S3KMS.IsRequested(h) || SSEC.IsRequested(h)
}
// UnsealObjectKey extracts and decrypts the sealed object key
// from the metadata using the SSE-Copy client key of the HTTP headers
// and returns the decrypted object key.