logger webhook failure can overrun the queue_size (#14556)

PR introduced in #13819 was incorrect and was not
handling the situation where a buffer is full can
cause incessant amount of logs that would keep the
logger webhook overrun by the requests.

To avoid this only log failures to console logger
instead of all targets as it can cause self reference,
leading to an infinite loop.

internal/logger/logger.go

@@ -360,7 +360,10 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 	// Iterate over all logger targets to send the log entry
 	for _, t := range SystemTargets() {
 		if err := t.Send(entry, entry.LogKind); err != nil {
-			LogAlwaysIf(context.Background(), fmt.Errorf("event(%v) was not sent to Logger target (%v): %v", entry, t, err), entry.LogKind)
+			if consoleTgt != nil {
+				entry.Trace.Message = fmt.Sprintf("event(%#v) was not sent to Logger target (%#v): %#v", entry, t, err)
+				consoleTgt.Send(entry, entry.LogKind)
+			}
 		}
 	}
 }

internal/logger/targets.go

@@ -46,7 +46,11 @@ var (
 	// Must be immutable at all times.
 	// Can be swapped to another while holding swapMu
 	systemTargets = []Target{}
-	nTargets      int32 // atomic count of len(targets)
+
+	// This is always set represent /dev/console target
+	consoleTgt Target
+
+	nTargets int32 // atomic count of len(targets)
 )
 
 // SystemTargets returns active targets.
@@ -90,6 +94,11 @@ func AddSystemTarget(t Target) error {
 		return err
 	}
 	swapMu.Lock()
+	if consoleTgt == nil {
+		if t.Type() == types.TargetConsole {
+			consoleTgt = t
+		}
+	}
 	updated := append(make([]Target, 0, len(systemTargets)+1), systemTargets...)
 	updated = append(updated, t)
 	systemTargets = updated