mirror of
https://github.com/minio/minio
synced 2024-11-05 17:34:01 +00:00
enable small and large file optimization (#11260)
- for large objects we found that 1MiB block for r/w respectively. - for small objects we found that 128KiB block for r/w respectively.
This commit is contained in:
Harshavardhana
GitHub
parent
e2579b1f5a
commit
1a5775e2e8
5 changed files with 50 additions and 35 deletions
|
|
@ -290,7 +290,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
|
|||
bucket := "bucket"
|
||||
object := "object"
|
||||
opts := ObjectOptions{}
|
||||
buf := make([]byte, 33<<10)
|
||||
buf := make([]byte, 129<<10)
|
||||
if _, err = io.ReadFull(crand.Reader, buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
|
|||
// max requests per node is calculated as
|
||||
// total_ram / ram_per_request
|
||||
// ram_per_request is 4MiB * setDriveCount + 2 * 10MiB (default erasure block size)
|
||||
apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*(writeBlockSize+readBlockSize)+blockSizeV1*2))
|
||||
apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*(blockSizeLarge+blockSizeSmall)+blockSizeV1*2))
|
||||
} else {
|
||||
apiRequestsMaxPerNode = cfg.RequestsMax
|
||||
if len(globalEndpoints.Hostnames()) > 0 {
|
||||
|
|
|
|||
|
|
@ -55,8 +55,8 @@ import (
|
|||
const (
|
||||
nullVersionID = "null"
|
||||
diskMinTotalSpace = 900 * humanize.MiByte // Min 900MiB total space.
|
||||
writeBlockSize = 4 * humanize.MiByte // Default write block size 4MiB.
|
||||
readBlockSize = 2 * humanize.MiByte // Default read block size 2MiB.
|
||||
blockSizeLarge = 1 * humanize.MiByte // Default r/w block size for larger objects.
|
||||
blockSizeSmall = 128 * humanize.KiByte // Default r/w block size for smaller objects.
|
||||
|
||||
// On regular files bigger than this;
|
||||
readAheadSize = 16 << 20
|
||||
|
|
@ -65,9 +65,10 @@ const (
|
|||
// Size of each buffer.
|
||||
readAheadBufSize = 1 << 20
|
||||
|
||||
// Small file threshold below which data accompanies metadata
|
||||
// from storage layer.
|
||||
smallFileThreshold = 32 * humanize.KiByte
|
||||
// Small file threshold below which data accompanies metadata from storage layer.
|
||||
smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs
|
||||
// For hardrives it is possible to set this to a lower value to avoid any
|
||||
// spike in latency. But currently we are simply keeping it optimal for SSDs.
|
||||
|
||||
// XL metadata file carries per object metadata.
|
||||
xlStorageFormatFile = "xl.meta"
|
||||
|
|
@ -97,8 +98,8 @@ type xlStorage struct {
|
|||
|
||||
globalSync bool
|
||||
|
||||
wpool sync.Pool
|
||||
rpool sync.Pool
|
||||
poolLarge sync.Pool
|
||||
poolSmall sync.Pool
|
||||
|
||||
rootDisk bool
|
||||
|
||||
|
|
@ -261,15 +262,15 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
|
|||
p := &xlStorage{
|
||||
diskPath: path,
|
||||
endpoint: ep,
|
||||
wpool: sync.Pool{
|
||||
poolLarge: sync.Pool{
|
||||
New: func() interface{} {
|
||||
b := disk.AlignedBlock(writeBlockSize)
|
||||
b := disk.AlignedBlock(blockSizeLarge)
|
||||
return &b
|
||||
},
|
||||
},
|
||||
rpool: sync.Pool{
|
||||
poolSmall: sync.Pool{
|
||||
New: func() interface{} {
|
||||
b := disk.AlignedBlock(readBlockSize)
|
||||
b := disk.AlignedBlock(blockSizeSmall)
|
||||
return &b
|
||||
},
|
||||
},
|
||||
|
|
@ -1180,7 +1181,7 @@ func (s *xlStorage) readAllData(volumeDir, filePath string) (buf []byte, err err
|
|||
}
|
||||
|
||||
atomic.AddInt32(&s.activeIOCount, 1)
|
||||
rd := &odirectReader{f, nil, nil, true, s, nil}
|
||||
rd := &odirectReader{f, nil, nil, true, true, s, nil}
|
||||
defer rd.Close()
|
||||
|
||||
return ioutil.ReadAll(rd)
|
||||
|
|
@ -1410,6 +1411,7 @@ type odirectReader struct {
|
|||
buf []byte
|
||||
bufp *[]byte
|
||||
freshRead bool
|
||||
smallFile bool
|
||||
s *xlStorage
|
||||
err error
|
||||
}
|
||||
|
|
@ -1420,7 +1422,11 @@ func (o *odirectReader) Read(buf []byte) (n int, err error) {
|
|||
return 0, o.err
|
||||
}
|
||||
if o.buf == nil {
|
||||
o.bufp = o.s.rpool.Get().(*[]byte)
|
||||
if o.smallFile {
|
||||
o.bufp = o.s.poolSmall.Get().(*[]byte)
|
||||
} else {
|
||||
o.bufp = o.s.poolLarge.Get().(*[]byte)
|
||||
}
|
||||
}
|
||||
if o.freshRead {
|
||||
o.buf = *o.bufp
|
||||
|
|
@ -1449,8 +1455,14 @@ func (o *odirectReader) Read(buf []byte) (n int, err error) {
|
|||
|
||||
// Close - Release the buffer and close the file.
|
||||
func (o *odirectReader) Close() error {
|
||||
o.s.rpool.Put(o.bufp)
|
||||
atomic.AddInt32(&o.s.activeIOCount, -1)
|
||||
if o.smallFile {
|
||||
o.s.poolSmall.Put(o.bufp)
|
||||
} else {
|
||||
o.s.poolLarge.Put(o.bufp)
|
||||
}
|
||||
defer func() {
|
||||
atomic.AddInt32(&o.s.activeIOCount, -1)
|
||||
}()
|
||||
return o.f.Close()
|
||||
}
|
||||
|
||||
|
|
@ -1472,6 +1484,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
|
|||
}
|
||||
|
||||
var file *os.File
|
||||
// O_DIRECT only supported if offset is zero
|
||||
if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
|
||||
file, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
|
||||
} else {
|
||||
|
|
@ -1512,7 +1525,10 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
|
|||
|
||||
atomic.AddInt32(&s.activeIOCount, 1)
|
||||
if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
|
||||
return &odirectReader{file, nil, nil, true, s, nil}, nil
|
||||
if length <= smallFileThreshold {
|
||||
return &odirectReader{file, nil, nil, true, true, s, nil}, nil
|
||||
}
|
||||
return &odirectReader{file, nil, nil, true, false, s, nil}, nil
|
||||
}
|
||||
|
||||
if offset > 0 {
|
||||
|
|
@ -1525,7 +1541,9 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
|
|||
io.Reader
|
||||
io.Closer
|
||||
}{Reader: io.LimitReader(file, length), Closer: closeWrapper(func() error {
|
||||
atomic.AddInt32(&s.activeIOCount, -1)
|
||||
defer func() {
|
||||
atomic.AddInt32(&s.activeIOCount, -1)
|
||||
}()
|
||||
return file.Close()
|
||||
})}
|
||||
|
||||
|
|
@ -1646,8 +1664,14 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
|
|||
w.Close()
|
||||
}()
|
||||
|
||||
bufp := s.wpool.Get().(*[]byte)
|
||||
defer s.wpool.Put(bufp)
|
||||
var bufp *[]byte
|
||||
if fileSize <= smallFileThreshold {
|
||||
bufp = s.poolSmall.Get().(*[]byte)
|
||||
defer s.poolSmall.Put(bufp)
|
||||
} else {
|
||||
bufp = s.poolLarge.Get().(*[]byte)
|
||||
defer s.poolLarge.Put(bufp)
|
||||
}
|
||||
|
||||
written, err := xioutil.CopyAligned(w, r, *bufp, fileSize)
|
||||
if err != nil {
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -66,7 +66,6 @@ require (
|
|||
github.com/pierrec/lz4 v2.5.2+incompatible
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.8.0
|
||||
github.com/quasilyte/go-ruleguard v0.2.1 // indirect
|
||||
github.com/rjeczalik/notify v0.9.2
|
||||
github.com/rs/cors v1.7.0
|
||||
github.com/secure-io/sio-go v0.3.0
|
||||
|
|
@ -75,8 +74,7 @@ require (
|
|||
github.com/streadway/amqp v1.0.0
|
||||
github.com/tidwall/gjson v1.3.5
|
||||
github.com/tidwall/sjson v1.0.4
|
||||
github.com/tinylib/msgp v1.1.3
|
||||
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect
|
||||
github.com/tinylib/msgp v1.1.5
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
|
||||
github.com/willf/bitset v1.1.11 // indirect
|
||||
github.com/willf/bloom v2.0.3+incompatible
|
||||
|
|
|
|||
15
go.sum
15
go.sum
|
|
@ -531,8 +531,6 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
|
|||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
|
||||
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/quasilyte/go-ruleguard v0.2.1 h1:56eRm0daAyny9UhJnmtJW/UyLZQusukBAB8oT8AHKHo=
|
||||
github.com/quasilyte/go-ruleguard v0.2.1/go.mod h1:hN2rVc/uS4bQhQKTio2XaSJSafJwqBUWWwtssT3cQmc=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
|
||||
|
|
@ -604,8 +602,8 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
|||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg=
|
||||
github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
|
||||
github.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA=
|
||||
github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=
|
||||
github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4=
|
||||
|
|
@ -626,7 +624,6 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
|
|||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
|
|
@ -707,7 +704,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
|
|
@ -722,8 +718,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -795,9 +790,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f h1:7+Nz9MyPqt2qMCTvNiRy1G0zYfkB7UCa+ayT6uVvbyI=
|
||||
golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
||||
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e h1:Z2uDrs8MyXUWJbwGc4V+nGjV4Ygo+oubBbWSVQw21/I=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
|
|||
Loading…
Reference in a new issue