self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-07-21 02:14:35 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Enhance config restore to carry previous set content as well (#8483)

Browse source 
This PR brings support for `history` list to
list in the following agreed format

```
~ mc admin config history list -n 2 myminio
RestoreId: df0ebb1e-69b0-4043-b9dd-ab54508f2897
Date: Mon, 04 Nov 2019 17:27:27 GMT

region name="us-east-1" state="on"
region name="us-east-1" state="on"
region name="us-east-1" state="on"
region name="us-east-1" state="on"

RestoreId: ecc6873a-0ed3-41f9-b03e-a2a1bab48b5f
Date: Mon, 04 Nov 2019 17:28:23 GMT

region name=us-east-1 state=off
```

This PR also moves the help templating and coloring to
fully `mc` side instead than `madmin` API.
This commit is contained in:
Harshavardhana 2019-11-05 06:18:26 -08:00 committed by kannappanr
parent 26ed9e81b1
commit 1e457dda7e
8 changed files with 88 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
cmd/admin-handlers-config-kv.go
View file

@ -23,6 +23,7 @@ import (
"encoding/json"
"io"
"net/http"
"strconv"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/config"
@ -236,7 +237,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
return
}
if restoreID == "all" {
chEntries, err := listServerConfigHistory(ctx, objectAPI)
chEntries, err := listServerConfigHistory(ctx, objectAPI, false, -1)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@ -323,7 +324,14 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
return
}
chEntries, err := listServerConfigHistory(ctx, objectAPI)
vars := mux.Vars(r)
count, err := strconv.Atoi(vars["count"])
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
chEntries, err := listServerConfigHistory(ctx, objectAPI, true, count)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@ -335,7 +343,14 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
return
}
writeSuccessResponseJSON(w, data)
password := globalActiveCred.SecretKey
econfigData, err := madmin.EncryptData(password, data)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, econfigData)
}
// HelpConfigKVHandler - GET /minio/admin/v2/help-config-kv?subSys={subSys}&key={key}

2
cmd/admin-router.go
View file

@ -81,7 +81,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler))
adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
}

36
cmd/config.go
View file

@ -34,6 +34,8 @@ import (
const (
minioConfigPrefix = "config"
kvPrefix = ".kv"
// Captures all the previous SetKV operations and allows rollback.
minioConfigHistoryPrefix = minioConfigPrefix + "/history"
@ -44,7 +46,9 @@ const (
minioConfigBackupFile = minioConfigFile + ".backup"
)
func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.ConfigHistoryEntry, error) {
func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) (
[]madmin.ConfigHistoryEntry, error) {
var configHistory []madmin.ConfigHistoryEntry
// List all kvs
@ -55,10 +59,28 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.
return nil, err
}
for _, obj := range res.Objects {
configHistory = append(configHistory, madmin.ConfigHistoryEntry{
RestoreID: path.Base(obj.Name),
cfgEntry := madmin.ConfigHistoryEntry{
RestoreID: strings.TrimSuffix(path.Base(obj.Name), kvPrefix),
CreateTime: obj.ModTime, // ModTime is createTime for config history entries.
})
}
if withData {
data, err := readConfig(ctx, objAPI, obj.Name)
if err != nil {
return nil, err
}
if globalConfigEncrypted {
data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
if err != nil {
return nil, err
}
}
cfgEntry.Data = string(data)
}
configHistory = append(configHistory, cfgEntry)
count--
if count == 0 {
break
}
}
if !res.IsTruncated {
// We are done here
@ -73,12 +95,12 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.
}
func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
return objAPI.DeleteObject(ctx, minioMetaBucket, historyFile)
}
func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
data, err := readConfig(ctx, objAPI, historyFile)
if err != nil {
return nil, err
@ -92,7 +114,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
}
func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
uuidKV := mustGetUUID() + ".kv"
uuidKV := mustGetUUID() + kvPrefix
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
var err error

17
cmd/config/config.go
View file

@ -24,6 +24,7 @@ import (
"github.com/minio/minio-go/pkg/set"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/env"
"github.com/minio/minio/pkg/madmin"
)
// Error config error type
@ -41,7 +42,7 @@ func (e Error) Error() string {
// Default keys
const (
Default = `_`
Default = madmin.Default
State = "state"
Comment = "comment"
@ -133,13 +134,13 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
// Constant separators
const (
SubSystemSeparator = `:`
KvSeparator = `=`
KvSpaceSeparator = ` `
KvComment = `#`
KvNewline = "\n"
KvDoubleQuote = `"`
KvSingleQuote = `'`
SubSystemSeparator = madmin.SubSystemSeparator
KvSeparator = madmin.KvSeparator
KvSpaceSeparator = madmin.KvSpaceSeparator
KvComment = madmin.KvComment
KvNewline = madmin.KvNewline
KvDoubleQuote = madmin.KvDoubleQuote
KvSingleQuote = madmin.KvSingleQuote
// Env prefix used for all envs in MinIO
EnvPrefix = "MINIO_"

46
pkg/madmin/config-help-commands.go
View file

@ -19,42 +19,12 @@ package madmin
import (
"encoding/json"
"io"
"net/http"
"net/url"
"strings"
"text/tabwriter"
"text/template"
"github.com/minio/minio/pkg/color"
)
// Help template used by all sub-systems
const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
{{end}}`
// HelpEnv template used by all sub-systems
const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}}
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
{{end}}`
var funcMap = template.FuncMap{
"colorBlueBold": color.BlueBold,
"colorYellowBold": color.YellowBold,
"colorCyanBold": color.CyanBold,
}
// HelpTemplate - captures config help template
var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
// HelpEnvTemplate - captures config help template
var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv))
// HelpConfigKV - return help for a given sub-system.
func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) {
func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (map[string]string, error) {
v := url.Values{}
v.Set("subSys", subSys)
v.Set("key", key)
@ -84,17 +54,5 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reade
return nil, err
}
var s strings.Builder
w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
if !envOnly {
err = HelpTemplate.Execute(w, help)
} else {
err = HelpEnvTemplate.Execute(w, help)
}
if err != nil {
return nil, err
}
w.Flush()
return strings.NewReader(s.String()), nil
return help, nil
}

28
pkg/madmin/config-history-commands.go
View file

@ -21,6 +21,7 @@ import (
"encoding/json"
"net/http"
"net/url"
"strconv"
"time"
)
@ -80,14 +81,27 @@ func (adm *AdminClient) RestoreConfigHistoryKV(restoreID string) (err error) {
type ConfigHistoryEntry struct {
RestoreID string `json:"restoreId"`
CreateTime time.Time `json:"createTime"`
Data string `json:"data"`
}
// CreateTimeFormatted is used to print formatted time for CreateTime.
func (ch ConfigHistoryEntry) CreateTimeFormatted() string {
return ch.CreateTime.Format(http.TimeFormat)
}
// ListConfigHistoryKV - lists a slice of ConfigHistoryEntries sorted by createTime.
func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) {
func (adm *AdminClient) ListConfigHistoryKV(count int) ([]ConfigHistoryEntry, error) {
if count == 0 {
count = 10
}
v := url.Values{}
v.Set("count", strconv.Itoa(count))
// Execute GET on /minio/admin/v2/list-config-history-kv
resp, err := adm.executeMethod(http.MethodGet,
requestData{
relPath: adminAPIPrefix + "/list-config-history-kv",
relPath: adminAPIPrefix + "/list-config-history-kv",
queryValues: v,
})
defer closeResponse(resp)
if err != nil {
@ -97,11 +111,15 @@ func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) {
return nil, httpRespToErrorResponse(resp)
}
data, err := DecryptData(adm.secretAccessKey, resp.Body)
if err != nil {
return nil, err
}
var chEntries []ConfigHistoryEntry
d := json.NewDecoder(resp.Body)
d.DisallowUnknownFields()
if err = d.Decode(&chEntries); err != nil {
if err = json.Unmarshal(data, &chEntries); err != nil {
return chEntries, err
}
return chEntries, nil
}

3
pkg/madmin/config-kv-commands.go
View file

@ -76,6 +76,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) {
s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment)))
s.WriteString(KvDoubleQuote)
}
s.WriteString(KvNewline)
comment = ""
}
@ -131,5 +132,5 @@ func (adm *AdminClient) GetConfigKV(key string) (Targets, error) {
return nil, err
}
return parseSubSysTarget(data)
return ParseSubSysTarget(data)
}

8
pkg/madmin/parse-kv.go
View file

@ -88,11 +88,11 @@ const (
KvSeparator = `=`
KvSpaceSeparator = ` `
KvComment = `#`
KvNewline = "\n"
KvDoubleQuote = `"`
KvSingleQuote = `'`
KvNewline = "\n"
Default = `_`
Default = `_`
)
// This function is needed, to trim off single or double quotes, creeping into the values.
@ -140,8 +140,8 @@ func convertTargets(s string, targets Targets) error {
return nil
}
// parseSubSysTarget - parse sub-system target
func parseSubSysTarget(buf []byte) (Targets, error) {
// ParseSubSysTarget - parse sub-system target
func ParseSubSysTarget(buf []byte) (Targets, error) {
targets := make(map[string]map[string]KVS)
bio := bufio.NewScanner(bytes.NewReader(buf))
for bio.Scan() {