liveness returns "busy" if queued requests > available capacity (#16719)

cmd/api-errors.go

@@ -1520,7 +1520,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBusy: {
-		Code:           "Busy",
+		Code:           "ServerBusy",
 		Description:    "The service is unavailable. Please retry.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},

cmd/handler-api.go

@@ -227,6 +227,13 @@ func (t *apiConfig) getClusterDeadline() time.Duration {
 	return t.clusterDeadline
 }
 
+func (t *apiConfig) getRequestsPoolCapacity() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return cap(t.requestsPool)
+}
+
 func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()

cmd/healthcheck-handler.go

@@ -106,6 +106,17 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
 
+	if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
+		apiErr := getAPIError(ErrBusy)
+		switch r.Method {
+		case http.MethodHead:
+			writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+		case http.MethodGet:
+			writeErrorResponse(r.Context(), w, apiErr, r.URL)
+		}
+		return
+	}
+
 	// Verify if KMS is reachable if its configured
 	if GlobalKMS != nil {
 		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)

cmd/http-stats.go

@@ -255,6 +255,10 @@ type HTTPStats struct {
 	totalS3Canceled         HTTPAPIStats
 }
 
+func (st *HTTPStats) loadRequestsInQueue() int32 {
+	return atomic.LoadInt32(&st.s3RequestsInQueue)
+}
+
 func (st *HTTPStats) addRequestsInQueue(i int32) {
 	atomic.AddInt32(&st.s3RequestsInQueue, i)
 }