From 5246e3be84e36fefd99262af24e43bfc1fd81770 Mon Sep 17 00:00:00 2001 From: Shireesh Anjal <355479+anjalshireesh@users.noreply.github.com> Date: Wed, 16 Nov 2022 03:23:05 +0530 Subject: [PATCH] Send health diagnostics data as part of callhome (#16006) --- cmd/admin-handlers.go | 127 ++++++++++++++++--------------- cmd/callhome.go | 110 +++++++++++++++++++------- internal/config/subnet/subnet.go | 57 +++++++++++--- internal/http/headers.go | 3 + 4 files changed, 195 insertions(+), 102 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 3301720b6..e3a218ea6 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1911,67 +1911,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo { return ki } -// HealthInfoHandler - GET /minio/admin/v3/healthinfo -// ---------- -// Get server health info -func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) { - ctx := newContext(r, w, "HealthInfo") - - defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) - - objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction) - if objectAPI == nil { - return - } - - query := r.Form - healthInfo := madmin.HealthInfo{ - Version: madmin.HealthInfoVersion, - Minio: madmin.MinioHealthInfo{ - Info: madmin.MinioInfo{ - DeploymentID: globalDeploymentID, - }, - }, - } - healthInfoCh := make(chan madmin.HealthInfo) - - enc := json.NewEncoder(w) - - setCommonHeaders(w) - - setEventStreamHeaders(w) - - w.WriteHeader(http.StatusOK) - - errResp := func(err error) { - errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(), - w.Header().Get(xhttp.AmzRequestID), globalDeploymentID) - encodedErrorResponse := encodeResponse(errorResponse) - healthInfo.Error = string(encodedErrorResponse) - logger.LogIf(ctx, enc.Encode(healthInfo)) - } - - deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics. - if dstr := r.Form.Get("deadline"); dstr != "" { - var err error - deadline, err = time.ParseDuration(dstr) - if err != nil { - errResp(err) - return - } - } - - nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress") - lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)) - if err != nil { // returns a locked lock - errResp(err) - return - } - defer nsLock.Unlock(lkctx.Cancel) - - healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline) - defer healthCancel() - +func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) { hostAnonymizer := createHostAnonymizer() // anonAddr - Anonymizes hosts in given input string. anonAddr := func(addr string) string { @@ -2196,7 +2136,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque getAndWriteMinioConfig := func() { if query.Get("minioconfig") == "true" { - config, err := readServerConfig(ctx, objectAPI) + config, err := readServerConfig(healthCtx, objectAPI) if err != nil { healthInfo.Minio.Config = madmin.MinioConfig{ Error: err.Error(), @@ -2244,7 +2184,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque getAndWriteSysConfig() if query.Get("minioinfo") == "true" { - infoMessage := getServerInfo(ctx, r) + infoMessage := getServerInfo(healthCtx, nil) servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers)) for _, server := range infoMessage.Servers { anonEndpoint := anonAddr(server.Endpoint) @@ -2294,6 +2234,67 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque partialWrite(healthInfo) } }() +} + +// HealthInfoHandler - GET /minio/admin/v3/healthinfo +// ---------- +// Get server health info +func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "HealthInfo") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction) + if objectAPI == nil { + return + } + + query := r.Form + healthInfoCh := make(chan madmin.HealthInfo) + enc := json.NewEncoder(w) + + healthInfo := madmin.HealthInfo{ + Version: madmin.HealthInfoVersion, + Minio: madmin.MinioHealthInfo{ + Info: madmin.MinioInfo{ + DeploymentID: globalDeploymentID, + }, + }, + } + + errResp := func(err error) { + errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(), + w.Header().Get(xhttp.AmzRequestID), globalDeploymentID) + encodedErrorResponse := encodeResponse(errorResponse) + healthInfo.Error = string(encodedErrorResponse) + logger.LogIf(ctx, enc.Encode(healthInfo)) + } + + deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics. + if dstr := query.Get("deadline"); dstr != "" { + var err error + deadline, err = time.ParseDuration(dstr) + if err != nil { + errResp(err) + return + } + } + + nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress") + lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)) + if err != nil { // returns a locked lock + errResp(err) + return + } + + defer nsLock.Unlock(lkctx.Cancel) + healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline) + defer healthCancel() + + go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo) + + setCommonHeaders(w) + setEventStreamHeaders(w) + w.WriteHeader(http.StatusOK) ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() diff --git a/cmd/callhome.go b/cmd/callhome.go index 7c20ee093..3ebfa4736 100644 --- a/cmd/callhome.go +++ b/cmd/callhome.go @@ -18,29 +18,20 @@ package cmd import ( + "bytes" + "compress/gzip" "context" + "encoding/json" + "errors" "fmt" "math/rand" + "net/url" "time" "github.com/minio/madmin-go" "github.com/minio/minio/internal/logger" ) -const ( - // callhomeSchemaVersion1 is callhome schema version 1 - callhomeSchemaVersion1 = "1" - - // callhomeSchemaVersion is current callhome schema version. - callhomeSchemaVersion = callhomeSchemaVersion1 -) - -// CallhomeInfo - Contains callhome information -type CallhomeInfo struct { - SchemaVersion string `json:"schema_version"` - AdminInfo madmin.InfoMessage `json:"admin_info"` -} - var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second) // initCallhome will start the callhome task in the background. @@ -118,26 +109,91 @@ func runCallhome(ctx context.Context, objAPI ObjectLayer) bool { } func performCallhome(ctx context.Context) { - err := sendCallhomeInfo( - CallhomeInfo{ - SchemaVersion: callhomeSchemaVersion, - AdminInfo: getServerInfo(ctx, nil), - }) - if err != nil { - logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err)) + deadline := 10 * time.Second // Default deadline is 10secs for callhome + objectAPI := newObjectLayerFn() + if objectAPI == nil { + logger.LogIf(ctx, errors.New("Callhome: object layer not ready")) + return + } + + healthCtx, healthCancel := context.WithTimeout(ctx, deadline) + defer healthCancel() + + healthInfoCh := make(chan madmin.HealthInfo) + + query := url.Values{} + for _, k := range madmin.HealthDataTypesList { + query.Set(string(k), "true") + } + + healthInfo := madmin.HealthInfo{ + Version: madmin.HealthInfoVersion, + Minio: madmin.MinioHealthInfo{ + Info: madmin.MinioInfo{ + DeploymentID: globalDeploymentID, + }, + }, + } + + go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo) + + for { + select { + case hi, hasMore := <-healthInfoCh: + if !hasMore { + // Received all data. Send to SUBNET and return + err := sendHealthInfo(ctx, healthInfo) + if err != nil { + logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err)) + } + return + } + healthInfo = hi + case <-healthCtx.Done(): + return + } } } const ( - callhomeURL = "https://subnet.min.io/api/callhome" - callhomeURLDev = "http://localhost:9000/api/callhome" + healthURL = "https://subnet.min.io/api/health/upload" + healthURLDev = "http://localhost:9000/api/health/upload" ) -func sendCallhomeInfo(ch CallhomeInfo) error { - url := callhomeURL +func sendHealthInfo(ctx context.Context, healthInfo madmin.HealthInfo) error { + url := healthURL if globalIsCICD { - url = callhomeURLDev + url = healthURLDev } - _, err := globalSubnetConfig.Post(url, ch) + + filename := fmt.Sprintf("health_%s.json.gz", UTCNow().Format("20060102150405")) + url += "?filename=" + filename + + _, err := globalSubnetConfig.Upload(url, filename, createHealthJSONGzip(ctx, healthInfo)) return err } + +func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []byte { + var b bytes.Buffer + gzWriter := gzip.NewWriter(&b) + + header := struct { + Version string `json:"version"` + }{Version: healthInfo.Version} + + enc := json.NewEncoder(gzWriter) + if e := enc.Encode(header); e != nil { + logger.LogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e)) + return nil + } + + if e := enc.Encode(healthInfo); e != nil { + logger.LogIf(ctx, fmt.Errorf("Could not encode health info: %w", e)) + return nil + } + + gzWriter.Flush() + gzWriter.Close() + + return b.Bytes() +} diff --git a/internal/config/subnet/subnet.go b/internal/config/subnet/subnet.go index 4ba61e7d2..6905c6fa2 100644 --- a/internal/config/subnet/subnet.go +++ b/internal/config/subnet/subnet.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "mime/multipart" "net/http" "time" @@ -33,25 +34,38 @@ const ( respBodyLimit = 1 << 20 // 1 MiB ) -// Post submit 'payload' to specified URL -func (c Config) Post(reqURL string, payload interface{}) (string, error) { +// Upload given file content (payload) to specified URL +func (c Config) Upload(reqURL string, filename string, payload []byte) (string, error) { if !c.Registered() { return "", errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'") } - body, err := json.Marshal(payload) - if err != nil { - return "", err - } - r, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewReader(body)) - if err != nil { - return "", err + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, e := writer.CreateFormFile("file", filename) + if e != nil { + return "", e } + if _, e = part.Write(payload); e != nil { + return "", e + } + writer.Close() + + r, e := http.NewRequest(http.MethodPost, reqURL, &body) + if e != nil { + return "", e + } + r.Header.Add("Content-Type", writer.FormDataContentType()) + + return c.submitPost(r) +} + +func (c Config) submitPost(r *http.Request) (string, error) { configLock.RLock() - r.Header.Set("Authorization", c.APIKey) + r.Header.Set(xhttp.SubnetAPIKey, c.APIKey) configLock.RUnlock() - - r.Header.Set("Content-Type", "application/json") + r.Header.Set(xhttp.MinioDeploymentID, xhttp.GlobalDeploymentID) client := &http.Client{ Timeout: 10 * time.Second, @@ -76,3 +90,22 @@ func (c Config) Post(reqURL string, payload interface{}) (string, error) { return respStr, fmt.Errorf("SUBNET request failed with code %d and error: %s", resp.StatusCode, respStr) } + +// Post submit 'payload' to specified URL +func (c Config) Post(reqURL string, payload interface{}) (string, error) { + if !c.Registered() { + return "", errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'") + } + body, err := json.Marshal(payload) + if err != nil { + return "", err + } + r, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewReader(body)) + if err != nil { + return "", err + } + + r.Header.Set("Content-Type", "application/json") + + return c.submitPost(r) +} diff --git a/internal/http/headers.go b/internal/http/headers.go index 962df8be5..1f2359a57 100644 --- a/internal/http/headers.go +++ b/internal/http/headers.go @@ -210,6 +210,9 @@ const ( // MinIOCompressed is returned when object is compressed MinIOCompressed = "X-Minio-Compressed" + + // SUBNET related + SubnetAPIKey = "x-subnet-api-key" ) // Common http query params S3 API