diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 5455857db..543d482ff 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -785,6 +785,49 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) { keepConnLive(w, respCh) } +func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "HealBackgroundStatus") + + objectAPI := validateAdminReq(ctx, w, r) + if objectAPI == nil { + return + } + + // Check if this setup has an erasure coded backend. + if !globalIsXL { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL) + return + } + + var bgHealStates []madmin.BgHealState + + // Get local heal status first + bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus()) + + if globalIsDistXL { + // Get heal status from other peers + peersHealStates := globalNotificationSys.BackgroundHealStatus() + bgHealStates = append(bgHealStates, peersHealStates...) + } + + // Aggregate healing result + var aggregatedHealStateResult = madmin.BgHealState{} + for _, state := range bgHealStates { + aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount + if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) { + aggregatedHealStateResult.LastHealActivity = state.LastHealActivity + } + + } + + if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + w.(http.Flusher).Flush() +} + // GetConfigHandler - GET /minio/admin/v1/config // Get config.json of this minio setup. func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index 9e869fb1a..4935ea0e3 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -131,6 +131,19 @@ func (ahs *allHealState) periodicHealSeqsClean() { } } +// getHealSequenceByToken - Retrieve a heal sequence by token. The second +// argument returns if a heal sequence actually exists. +func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) { + ahs.Lock() + defer ahs.Unlock() + for _, healSeq := range ahs.healSeqMap { + if healSeq.clientToken == token { + return healSeq, true + } + } + return nil, false +} + // getHealSequence - Retrieve a heal sequence by path. The second // argument returns if a heal sequence actually exists. func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) { @@ -335,6 +348,12 @@ type healSequence struct { // the last result index sent to client lastSentResultIndex int64 + // Number of total items scanned + scannedItemsCount int64 + + // The time of the last scan/heal activity + lastHealActivity time.Time + // Holds the request-info for logging ctx context.Context } @@ -552,17 +571,20 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType) } func (h *healSequence) healItemsFromSourceCh() error { + h.lastHealActivity = UTCNow() + // Start healing the config prefix. if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil { - return err + logger.LogIf(h.ctx, err) } // Start healing the bucket config prefix. if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil { - return err + logger.LogIf(h.ctx, err) } for path := range h.sourceCh { + var itemType madmin.HealItemType switch { case path == "/": @@ -574,8 +596,11 @@ func (h *healSequence) healItemsFromSourceCh() error { } if err := h.queueHealTask(path, itemType); err != nil { - return err + logger.LogIf(h.ctx, err) } + + h.scannedItemsCount++ + h.lastHealActivity = UTCNow() } return nil diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 93468106b..63bbaaaeb 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -61,6 +61,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler)) adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler)) + adminV1Router.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler)) + /// Health operations } diff --git a/cmd/daily-heal-ops.go b/cmd/daily-heal-ops.go index 100084d5a..884bc5bd3 100644 --- a/cmd/daily-heal-ops.go +++ b/cmd/daily-heal-ops.go @@ -60,6 +60,18 @@ func newBgHealSequence(numDisks int) *healSequence { } } +func getLocalBackgroundHealStatus() madmin.BgHealState { + backgroundSequence, ok := globalSweepHealState.getHealSequenceByToken(bgHealingUUID) + if !ok { + return madmin.BgHealState{} + } + + return madmin.BgHealState{ + ScannedItemsCount: backgroundSequence.scannedItemsCount, + LastHealActivity: backgroundSequence.lastHealActivity, + } +} + func initDailyHeal() { go startDailyHeal() } diff --git a/cmd/notification.go b/cmd/notification.go index 6eb16aade..d965e14b2 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -34,6 +34,7 @@ import ( "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/event" + "github.com/minio/minio/pkg/madmin" xnet "github.com/minio/minio/pkg/net" "github.com/minio/minio/pkg/policy" ) @@ -230,6 +231,24 @@ func (sys *NotificationSys) LoadUsers() []NotificationPeerErr { return ng.Wait() } +// BackgroundHealStatus - returns background heal status of all peers +func (sys *NotificationSys) BackgroundHealStatus() []madmin.BgHealState { + states := make([]madmin.BgHealState, len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + st, err := client.BackgroundHealStatus() + if err != nil { + logger.LogIf(context.Background(), err) + } else { + states[idx] = st + } + } + + return states +} + // StartProfiling - start profiling on remote peers, by initiating a remote RPC. func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr { ng := WithNPeers(len(sys.peerClients)) diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index ced7cc4a3..d62fa08a1 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -31,6 +31,7 @@ import ( "github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/rest" "github.com/minio/minio/pkg/event" + "github.com/minio/minio/pkg/madmin" xnet "github.com/minio/minio/pkg/net" "github.com/minio/minio/pkg/policy" ) @@ -422,6 +423,18 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error { return nil } +func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error) { + respBody, err := client.call(peerRESTMethodBackgroundHealStatus, nil, nil, -1) + if err != nil { + return madmin.BgHealState{}, err + } + defer http.DrainBody(respBody) + + state := madmin.BgHealState{} + err = gob.NewDecoder(respBody).Decode(&state) + return state, err +} + // Trace - send http trace request to peer nodes func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) { ch := make(chan []byte) diff --git a/cmd/peer-rest-common.go b/cmd/peer-rest-common.go index 1547af5b9..c44ab17e5 100644 --- a/cmd/peer-rest-common.go +++ b/cmd/peer-rest-common.go @@ -26,6 +26,7 @@ const ( peerRESTMethodDrivePerfInfo = "driveperfinfo" peerRESTMethodDeleteBucket = "deletebucket" peerRESTMethodSignalService = "signalservice" + peerRESTMethodBackgroundHealStatus = "backgroundhealstatus" peerRESTMethodGetLocks = "getlocks" peerRESTMethodBucketPolicyRemove = "removebucketpolicy" peerRESTMethodLoadUser = "loaduser" diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index ca6fcae21..a84be0298 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -709,6 +709,20 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) { } } +func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("invalid request")) + return + } + + ctx := newContext(r, w, "BackgroundHealStatus") + + state := getLocalBackgroundHealStatus() + + defer w.(http.Flusher).Flush() + logger.LogIf(ctx, gob.NewEncoder(w).Encode(state)) +} + func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) { w.WriteHeader(http.StatusForbidden) w.Write([]byte(err.Error())) @@ -755,6 +769,7 @@ func registerPeerRESTHandlers(router *mux.Router) { subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...) subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler) + subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler) router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler)) } diff --git a/pkg/madmin/examples/heal-status.go b/pkg/madmin/examples/heal-status.go new file mode 100644 index 000000000..a25c66673 --- /dev/null +++ b/pkg/madmin/examples/heal-status.go @@ -0,0 +1,45 @@ +// +build ignore + +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "log" + + "github.com/minio/minio/pkg/madmin" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise. + // New returns an MinIO Admin client object. + madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + healStatusResult, err := madmClnt.BackgroundHealStatus() + if err != nil { + log.Fatalln(err) + } + + log.Printf("Heal status result: %+v\n", healStatusResult) +} diff --git a/pkg/madmin/heal-commands.go b/pkg/madmin/heal-commands.go index c40f50351..9c494c1dd 100644 --- a/pkg/madmin/heal-commands.go +++ b/pkg/madmin/heal-commands.go @@ -269,3 +269,37 @@ func (adm *AdminClient) Heal(bucket, prefix string, healOpts HealOpts, } return healStart, healTaskStatus, nil } + +// BgHealState represents the status of the background heal +type BgHealState struct { + ScannedItemsCount int64 + LastHealActivity time.Time +} + +// BackgroundHealStatus returns the background heal status of the +// current server or cluster. +func (adm *AdminClient) BackgroundHealStatus() (BgHealState, error) { + // Execute POST request to background heal status api + resp, err := adm.executeMethod("POST", requestData{relPath: "/v1/background-heal/status"}) + if err != nil { + return BgHealState{}, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return BgHealState{}, httpRespToErrorResponse(resp) + } + + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return BgHealState{}, err + } + + var healState BgHealState + + err = json.Unmarshal(respBytes, &healState) + if err != nil { + return BgHealState{}, err + } + return healState, nil +}