mirror of
https://github.com/minio/minio
synced 2024-10-02 22:24:00 +00:00
Implement Sia Gateway (#5114)
This commit is contained in:
David G
Dee Koder
parent
d1a6c32d80
commit
f4d4ea5c36
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -9,7 +9,7 @@ site/
|
|||
/.idea/
|
||||
/Minio.iml
|
||||
**/access.log
|
||||
build
|
||||
build/
|
||||
vendor/**/*.js
|
||||
vendor/**/*.json
|
||||
release
|
||||
|
|
@ -23,3 +23,4 @@ parts/
|
|||
prime/
|
||||
snap/.snapcraft/
|
||||
stage/
|
||||
.sia_temp/
|
||||
|
|
|
|||
524
cmd/gateway-sia.go
Normal file
524
cmd/gateway-sia.go
Normal file
|
|
@ -0,0 +1,524 @@
|
|||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
const (
|
||||
siaBackend = "sia"
|
||||
)
|
||||
|
||||
type siaObjects struct {
|
||||
gatewayUnsupported
|
||||
Address string // Address and port of Sia Daemon.
|
||||
TempDir string // Temporary storage location for file transfers.
|
||||
RootDir string // Root directory to store files on Sia.
|
||||
password string // Sia password for uploading content in authenticated manner.
|
||||
}
|
||||
|
||||
func init() {
|
||||
const siaGatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [SIA_DAEMON_ADDR]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENVIRONMENT VARIABLES: (Default values in parenthesis)
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Custom access key (Do not reuse same access keys on all instances)
|
||||
MINIO_SECRET_KEY: Custom secret key (Do not reuse same secret keys on all instances)
|
||||
|
||||
SIA_TEMP_DIR: The name of the local Sia temporary storage directory. (.sia_temp)
|
||||
SIA_API_PASSWORD: API password for Sia daemon. (default is empty)
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Sia backend.
|
||||
$ {{.HelpName}}
|
||||
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
Name: siaBackend,
|
||||
Usage: "Sia Decentralized Cloud.",
|
||||
Action: siaGatewayMain,
|
||||
CustomHelpTemplate: siaGatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway sia' command line.
|
||||
func siaGatewayMain(ctx *cli.Context) {
|
||||
// Validate gateway arguments.
|
||||
host := ctx.Args().First()
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
|
||||
startGateway(ctx, &SiaGateway{host})
|
||||
}
|
||||
|
||||
// SiaGateway implements Gateway.
|
||||
type SiaGateway struct {
|
||||
host string // Sia daemon host address
|
||||
}
|
||||
|
||||
// Name implements Gateway interface.
|
||||
func (g *SiaGateway) Name() string {
|
||||
return siaBackend
|
||||
}
|
||||
|
||||
// NewGatewayLayer returns b2 gateway layer, implements GatewayLayer interface to
|
||||
// talk to B2 remote backend.
|
||||
func (g *SiaGateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||
return newSiaGatewayLayer(g.host)
|
||||
}
|
||||
|
||||
// non2xx returns true for non-success HTTP status codes.
|
||||
func non2xx(code int) bool {
|
||||
return code < 200 || code > 299
|
||||
}
|
||||
|
||||
// decodeError returns the api.Error from a API response. This method should
|
||||
// only be called if the response's status code is non-2xx. The error returned
|
||||
// may not be of type api.Error in the event of an error unmarshalling the
|
||||
// JSON.
|
||||
type siaError struct {
|
||||
// Message describes the error in English. Typically it is set to
|
||||
// `err.Error()`. This field is required.
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (s siaError) Error() string {
|
||||
return s.Message
|
||||
}
|
||||
|
||||
func decodeError(resp *http.Response) error {
|
||||
// Error is a type that is encoded as JSON and returned in an API response in
|
||||
// the event of an error. Only the Message field is required. More fields may
|
||||
// be added to this struct in the future for better error reporting.
|
||||
var apiErr siaError
|
||||
if err := json.NewDecoder(resp.Body).Decode(&apiErr); err != nil {
|
||||
return err
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// SiaMethodNotSupported - returned if call returned error.
|
||||
type SiaMethodNotSupported struct {
|
||||
method string
|
||||
}
|
||||
|
||||
func (s SiaMethodNotSupported) Error() string {
|
||||
return fmt.Sprintf("API call not recognized: %s", s.method)
|
||||
}
|
||||
|
||||
// apiGet wraps a GET request with a status code check, such that if the GET does
|
||||
// not return 2xx, the error will be read and returned. The response body is
|
||||
// not closed.
|
||||
func apiGet(addr, call, apiPassword string) (*http.Response, error) {
|
||||
req, err := http.NewRequest("GET", "http://"+addr+call, nil)
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Sia-Agent")
|
||||
if apiPassword != "" {
|
||||
req.SetBasicAuth("", apiPassword)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
resp.Body.Close()
|
||||
return nil, SiaMethodNotSupported{call}
|
||||
}
|
||||
if non2xx(resp.StatusCode) {
|
||||
err := decodeError(resp)
|
||||
resp.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// apiPost wraps a POST request with a status code check, such that if the POST
|
||||
// does not return 2xx, the error will be read and returned. The response body
|
||||
// is not closed.
|
||||
func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) {
|
||||
req, err := http.NewRequest("POST", "http://"+addr+call, strings.NewReader(vals))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Sia-Agent")
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
if apiPassword != "" {
|
||||
req.SetBasicAuth("", apiPassword)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
resp.Body.Close()
|
||||
return nil, SiaMethodNotSupported{call}
|
||||
}
|
||||
|
||||
if non2xx(resp.StatusCode) {
|
||||
err := decodeError(resp)
|
||||
resp.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// post makes an API call and discards the response. An error is returned if
|
||||
// the response status is not 2xx.
|
||||
func post(addr, call, vals, apiPassword string) error {
|
||||
resp, err := apiPost(addr, call, vals, apiPassword)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// list makes a lists all the uploaded files, decodes the json response.
|
||||
func list(addr string, apiPassword string, obj *renterFiles) error {
|
||||
resp, err := apiGet(addr, "/renter/files", apiPassword)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNoContent {
|
||||
return errors.New("Expecting a response, but API returned status code 204 No Content")
|
||||
}
|
||||
|
||||
return json.NewDecoder(resp.Body).Decode(obj)
|
||||
}
|
||||
|
||||
// get makes an API call and discards the response. An error is returned if the
|
||||
// responsee status is not 2xx.
|
||||
func get(addr, call, apiPassword string) error {
|
||||
resp, err := apiGet(addr, call, apiPassword)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// newSiaGatewayLayer returns Sia gatewaylayer
|
||||
func newSiaGatewayLayer(host string) (GatewayLayer, error) {
|
||||
sia := &siaObjects{
|
||||
Address: host,
|
||||
// RootDir uses access key directly, provides partitioning for
|
||||
// concurrent users talking to same sia daemon.
|
||||
RootDir: os.Getenv("MINIO_ACCESS_KEY"),
|
||||
TempDir: os.Getenv("SIA_TEMP_DIR"),
|
||||
password: os.Getenv("SIA_API_PASSWORD"),
|
||||
}
|
||||
|
||||
// If Address not provided on command line or ENV, default to:
|
||||
if sia.Address == "" {
|
||||
sia.Address = "127.0.0.1:9980"
|
||||
}
|
||||
|
||||
// If local Sia temp directory not specified, default to:
|
||||
if sia.TempDir == "" {
|
||||
sia.TempDir = ".sia_temp"
|
||||
}
|
||||
|
||||
var err error
|
||||
sia.TempDir, err = filepath.Abs(sia.TempDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create the temp directory with proper permissions.
|
||||
// Ignore error when dir already exists.
|
||||
if err = os.MkdirAll(sia.TempDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Println(colorBlue("\nSia Gateway Configuration:"))
|
||||
log.Println(colorBlue(" Sia Daemon API Address:") + colorBold(fmt.Sprintf(" %s\n", sia.Address)))
|
||||
log.Println(colorBlue(" Sia Temp Directory:") + colorBold(fmt.Sprintf(" %s\n", sia.TempDir)))
|
||||
return sia, nil
|
||||
}
|
||||
|
||||
// Shutdown saves any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (s *siaObjects) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageInfo is not relevant to Sia backend.
|
||||
func (s *siaObjects) StorageInfo() (si StorageInfo) {
|
||||
return si
|
||||
}
|
||||
|
||||
// MakeBucket creates a new container on Sia backend.
|
||||
func (s *siaObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBucketInfo gets bucket metadata.
|
||||
func (s *siaObjects) GetBucketInfo(bucket string) (bi BucketInfo, err error) {
|
||||
// Until Sia support buckets/directories, must return here that all buckets exist.
|
||||
bi.Name = bucket
|
||||
return bi, nil
|
||||
}
|
||||
|
||||
// ListBuckets will detect and return existing buckets on Sia.
|
||||
func (s *siaObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
sObjs, serr := s.listRenterFiles("")
|
||||
if serr != nil {
|
||||
return buckets, serr
|
||||
}
|
||||
|
||||
m := make(set.StringSet)
|
||||
|
||||
prefix := s.RootDir + "/"
|
||||
for _, sObj := range sObjs {
|
||||
if strings.HasPrefix(sObj.SiaPath, prefix) {
|
||||
siaObj := strings.TrimPrefix(sObj.SiaPath, prefix)
|
||||
idx := strings.Index(siaObj, "/")
|
||||
if idx > 0 {
|
||||
m.Add(siaObj[0:idx])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, bktName := range m.ToSlice() {
|
||||
buckets = append(buckets, BucketInfo{
|
||||
Name: bktName,
|
||||
Created: timeSentinel,
|
||||
})
|
||||
}
|
||||
|
||||
return buckets, nil
|
||||
}
|
||||
|
||||
// DeleteBucket deletes a bucket on Sia.
|
||||
func (s *siaObjects) DeleteBucket(bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *siaObjects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
|
||||
siaObjs, siaErr := s.listRenterFiles(bucket)
|
||||
if siaErr != nil {
|
||||
return loi, siaErr
|
||||
}
|
||||
|
||||
loi.IsTruncated = false
|
||||
loi.NextMarker = ""
|
||||
|
||||
root := s.RootDir + "/"
|
||||
|
||||
// FIXME(harsha) - No paginated output supported for Sia backend right now, only prefix
|
||||
// based filtering. Once list renter files API supports paginated output we can support
|
||||
// paginated results here as well - until then Listing is an expensive operation.
|
||||
for _, sObj := range siaObjs {
|
||||
name := strings.TrimPrefix(sObj.SiaPath, pathJoin(root, bucket, "/"))
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
loi.Objects = append(loi.Objects, ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: name,
|
||||
Size: int64(sObj.Filesize),
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
func (s *siaObjects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
if !isValidObjectName(object) {
|
||||
return traceError(ObjectNameInvalid{bucket, object})
|
||||
}
|
||||
|
||||
dstFile := pathJoin(s.TempDir, mustGetUUID())
|
||||
defer fsRemoveFile(dstFile)
|
||||
|
||||
var siaObj = pathJoin(s.RootDir, bucket, object)
|
||||
if err := get(s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader, size, err := fsOpenFile(dstFile, startOffset)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket, object)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// For negative length we read everything.
|
||||
if length < 0 {
|
||||
length = size - startOffset
|
||||
}
|
||||
|
||||
bufSize := int64(readSizeV1)
|
||||
if bufSize > length {
|
||||
bufSize = length
|
||||
}
|
||||
|
||||
// Reply back invalid range if the input offset and length fall out of range.
|
||||
if startOffset > size || startOffset+length > size {
|
||||
return traceError(InvalidRange{startOffset, length, size})
|
||||
}
|
||||
|
||||
// Allocate a staging buffer.
|
||||
buf := make([]byte, int(bufSize))
|
||||
|
||||
_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
func (s *siaObjects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
var siaObj = pathJoin(s.RootDir, bucket, object)
|
||||
sObjs, serr := s.listRenterFiles(bucket)
|
||||
if serr != nil {
|
||||
return objInfo, serr
|
||||
}
|
||||
|
||||
for _, sObj := range sObjs {
|
||||
if sObj.SiaPath == siaObj {
|
||||
// Metadata about sia objects is just quite minimal
|
||||
// there is nothing else sia provides other than size.
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
Size: int64(sObj.Filesize),
|
||||
IsDir: false,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return objInfo, traceError(ObjectNotFound{bucket, object})
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (s *siaObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
// Check the object's name first
|
||||
if !isValidObjectName(object) {
|
||||
return objInfo, traceError(ObjectNameInvalid{bucket, object})
|
||||
}
|
||||
|
||||
bufSize := int64(readSizeV1)
|
||||
size := data.Size()
|
||||
if size > 0 && bufSize > size {
|
||||
bufSize = size
|
||||
}
|
||||
buf := make([]byte, int(bufSize))
|
||||
|
||||
srcFile := pathJoin(s.TempDir, mustGetUUID())
|
||||
defer fsRemoveFile(srcFile)
|
||||
|
||||
if _, err = fsCreateFile(srcFile, data, buf, data.Size()); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
var siaObj = pathJoin(s.RootDir, bucket, object)
|
||||
if err = post(s.Address, "/renter/upload/"+siaObj, "source="+srcFile, s.password); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
objInfo = ObjectInfo{
|
||||
Name: object,
|
||||
Bucket: bucket,
|
||||
ModTime: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
Size: size,
|
||||
ETag: genETag(),
|
||||
}
|
||||
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// DeleteObject deletes a blob in bucket
|
||||
func (s *siaObjects) DeleteObject(bucket string, object string) error {
|
||||
// Tell Sia daemon to delete the object
|
||||
var siaObj = pathJoin(s.RootDir, bucket, object)
|
||||
return post(s.Address, "/renter/delete/"+siaObj, "", s.password)
|
||||
}
|
||||
|
||||
// siaObjectInfo represents object info stored on Sia
|
||||
type siaObjectInfo struct {
|
||||
SiaPath string `json:"siapath"`
|
||||
LocalPath string `json:"localpath"`
|
||||
Filesize uint64 `json:"filesize"`
|
||||
Available bool `json:"available"`
|
||||
Renewing bool `json:"renewing"`
|
||||
Redundancy float64 `json:"redundancy"`
|
||||
UploadProgress float64 `json:"uploadprogress"`
|
||||
}
|
||||
|
||||
// isValidObjectName returns whether or not the objectName provided is suitable for Sia
|
||||
func isValidObjectName(objectName string) bool {
|
||||
reg, _ := regexp.Compile("[^a-zA-Z0-9., _/\\\\+-]+")
|
||||
return objectName == reg.ReplaceAllString(objectName, "")
|
||||
}
|
||||
|
||||
type renterFiles struct {
|
||||
Files []siaObjectInfo `json:"files"`
|
||||
}
|
||||
|
||||
// ListObjects will return a list of existing objects in the bucket provided
|
||||
func (s *siaObjects) listRenterFiles(bucket string) (siaObjs []siaObjectInfo, err error) {
|
||||
// Get list of all renter files
|
||||
var rf renterFiles
|
||||
if err = list(s.Address, s.password, &rf); err != nil {
|
||||
return siaObjs, err
|
||||
}
|
||||
|
||||
var prefix string
|
||||
root := s.RootDir + "/"
|
||||
if bucket == "" {
|
||||
prefix = root
|
||||
} else {
|
||||
prefix = root + bucket + "/"
|
||||
}
|
||||
|
||||
for _, f := range rf.Files {
|
||||
if strings.HasPrefix(f.SiaPath, prefix) {
|
||||
siaObjs = append(siaObjs, f)
|
||||
}
|
||||
}
|
||||
|
||||
return siaObjs, nil
|
||||
}
|
||||
151
cmd/gateway-sia_test.go
Normal file
151
cmd/gateway-sia_test.go
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSianon2xx(t *testing.T) {
|
||||
for i := 0; i < 1000; i++ {
|
||||
actual := non2xx(i)
|
||||
expected := i < 200 || i > 299
|
||||
|
||||
if actual != expected {
|
||||
t.Errorf("Test case %d: non2xx(%d) returned %t but expected %t", i+1, i, actual, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiaIsValidObjectName(t *testing.T) {
|
||||
testCases := []struct {
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
input: `abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890., _/+-\`,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
input: "`",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "~",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "!",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "@",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "#",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "$",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: `%`,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "^",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "&",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "*",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "(",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: ")",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "=",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "{",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "}",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "[",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "]",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: ":",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: ";",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "?",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: ">",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "<",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: `"`,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: `'`,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
input: "|",
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
actual := isValidObjectName(tc.input)
|
||||
if actual != tc.expected {
|
||||
t.Errorf("Test case %d: isValidObjectName(%s) returned %t but expected %t", i+1, tc.input, actual, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -17,15 +17,62 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
type gatewayUnsupported struct{}
|
||||
|
||||
// CopyObjectPart - Not implemented.
|
||||
func (a gatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string,
|
||||
partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return info, traceError(NotImplemented{})
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (a gatewayUnsupported) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
|
||||
return lmi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// NewMultipartUpload upload object in multiple parts
|
||||
func (a gatewayUnsupported) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
return "", traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// CopyObjectPart copy part of object to other bucket and object
|
||||
func (a gatewayUnsupported) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64) (pi PartInfo, err error) {
|
||||
return pi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (a gatewayUnsupported) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) {
|
||||
return pi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (a gatewayUnsupported) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
|
||||
return lpi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (a gatewayUnsupported) AbortMultipartUpload(bucket string, object string, uploadID string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (a gatewayUnsupported) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
|
||||
return oi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// SetBucketPolicies sets policy on bucket
|
||||
func (a gatewayUnsupported) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// GetBucketPolicies will get policy on bucket
|
||||
func (a gatewayUnsupported) GetBucketPolicies(bucket string) (bal policy.BucketAccessPolicy, err error) {
|
||||
return bal, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies deletes all policies on bucket
|
||||
func (a gatewayUnsupported) DeleteBucketPolicies(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
|
|
@ -43,6 +90,10 @@ func (a gatewayUnsupported) HealObject(bucket, object string) (int, int, error)
|
|||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
func (a gatewayUnsupported) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
|
||||
return result, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
|
|
@ -76,3 +127,19 @@ func (a gatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Read
|
|||
metadata map[string]string) (ObjectInfo, error) {
|
||||
return ObjectInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetObject downloads object anonymously.
|
||||
func (a gatewayUnsupported) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetObjectInfo returns stat information about an object anonymously.
|
||||
func (a gatewayUnsupported) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
return objInfo, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// CopyObject copies a blob from source container to destination container.
|
||||
func (a gatewayUnsupported) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string,
|
||||
metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
return objInfo, traceError(NotImplemented{})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Minio Gateway adds Amazon S3 compatibility to third party cloud storage provider
|
|||
- [Microsoft Azure Blob Storage](https://github.com/minio/minio/blob/master/docs/gateway/azure.md)
|
||||
- [Google Cloud Storage](https://github.com/minio/minio/blob/master/docs/gateway/gcs.md) _Alpha release_
|
||||
- [Backblaze B2](https://github.com/minio/minio/blob/master/docs/gateway/b2.md) _Alpha release_
|
||||
- [Sia Decentralized Cloud Storage](https://github.com/minio/minio/blob/master/docs/gateway/sia.md) _Alpha release_
|
||||
|
||||
## Roadmap
|
||||
* Edge Caching - Disk based proxy caching support
|
||||
|
|
|
|||
60
docs/gateway/sia.md
Normal file
60
docs/gateway/sia.md
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
# Minio Sia Gateway [](https://slack.minio.io)
|
||||
Minio Sia Gateway adds Amazon S3 compatibility to Sia Decentralized Cloud Storage.
|
||||
|
||||
## What is Sia?
|
||||
Sia is a blockchain-based decentralized storage service with built-in privacy and redundancy that costs up to 10x LESS than Amazon S3 and most other cloud providers! See [sia.tech](https://sia.tech) to learn how awesome Sia truly is.
|
||||
|
||||
## Getting Started with Sia
|
||||
|
||||
### Install Sia Daemon
|
||||
To use Sia for backend storage, Minio will need access to a running Sia daemon that is:
|
||||
1. fully synchronized with the Sia network,
|
||||
2. has sufficient rental contract allowances, and
|
||||
3. has an unlocked wallet.
|
||||
|
||||
To download and install Sia for your platform, visit [sia.tech](http://sia.tech).
|
||||
|
||||
To purchase inexpensive rental contracts with Sia, you have to possess some Siacoin in your wallet. To obtain Siacoin, you will need to purchase some on an exchange such as Bittrex using bitcoin. To obtain bitcoin, you'll need to use a service such as Coinbase to buy bitcoin using a bank account or credit card. If you need help, there are many friendly people active on [Sia's Slack](http://slackin.sia.tech).
|
||||
|
||||
### Configuration
|
||||
Once you have the Sia Daemon running and synchronized, with rental allowances created, you just need to configure the Minio server to use Sia. Configuration is accomplished using environment variables, and is only necessary if the default values need to be changed. On a linux machine using bash shell, you can easily set environment variables by adding export statements to the "~/.bash_profile" file. For example:
|
||||
```
|
||||
export MY_ENV_VAR=VALUE
|
||||
```
|
||||
Just remember to reload the profile by executing: "source ~/.bash_profile" on the command prompt.
|
||||
|
||||
#### Supported Environment Variables
|
||||
Environment Variable | Description | Default Value
|
||||
--- | --- | ---
|
||||
`MINIO_ACCESS_KEY` | The access key required to access Minio. | (empty)
|
||||
`MINIO_SECRET_KEY` | The secret key required to access Minio. | (empty)
|
||||
`SIA_TEMP_DIR` | The local directory to use for temporary storage. | .sia_temp
|
||||
`SIA_API_PASSWORD` | The API password required to access the Sia daemon, if needed. | (empty)
|
||||
|
||||
### Running Minio with Sia Gateway
|
||||
```
|
||||
export MINIO_ACCESS_KEY=minioaccesskey
|
||||
export MINIO_SECRET_KEY=miniosecretkey
|
||||
minio gateway sia [SIA_DAEMON_ADDRESS]
|
||||
```
|
||||
The [SIA_DAEMON_ADDRESS] is optional, and it defaults to "127.0.0.1:9980".
|
||||
Access information should then be presented on-screen. To connect to the server and upload files using your web browser, open a web browser and point it to the address displayed for "Browser Access." Then log in using the "AccessKey" and "SecretKey" that are also displayed on-screen. You should then be able to create buckets (folders) and upload files.
|
||||
|
||||
|
||||
|
||||
### Known limitations
|
||||
|
||||
Gateway inherits the following Sia limitations:
|
||||
|
||||
- Multipart uploads are not currently supported.
|
||||
- Bucket policies are not currently supported.
|
||||
|
||||
Other limitations:
|
||||
|
||||
- Bucket notification APIs are not supported.
|
||||
|
||||
## Explore Further
|
||||
- [`mc` command-line interface](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
- [`aws` command-line interface](https://docs.minio.io/docs/aws-cli-with-minio)
|
||||
- [`minio-go` Go SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)
|
||||
|
||||
18
vendor/vendor.json
vendored
18
vendor/vendor.json
vendored
|
|
@ -494,24 +494,6 @@
|
|||
"revision": "3b8db5e93c4c02efbc313e17b2e796b0914a01fb",
|
||||
"revisionTime": "2016-12-15T19:56:52Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "CvMkf3KUUGUVHibg6G/zI7XtVbM=",
|
||||
"path": "golang.org/x/crypto/chacha20poly1305",
|
||||
"revision": "81e90905daefcd6fd217b62423c0908922eadb30",
|
||||
"revisionTime": "2017-08-25T20:24:07Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "8f1lWFSOLu7O6Jqk/e+ydH1yoms=",
|
||||
"path": "golang.org/x/crypto/chacha20poly1305/internal/chacha20",
|
||||
"revision": "81e90905daefcd6fd217b62423c0908922eadb30",
|
||||
"revisionTime": "2017-08-25T20:24:07Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "kVKE0OX1Xdw5mG7XKT86DLLKE2I=",
|
||||
"path": "golang.org/x/crypto/poly1305",
|
||||
"revision": "81e90905daefcd6fd217b62423c0908922eadb30",
|
||||
"revisionTime": "2017-08-25T20:24:07Z"
|
||||
},
|
||||
{
|
||||
"path": "golang.org/x/net/context",
|
||||
"revision": "a728288923b47049b2ce791836767ffbe964a5bd",
|
||||
|
|
|
|||
Loading…
Reference in a new issue