self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-07-21 10:25:00 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

optimize Listen bucket notification implementation (#9444)

Browse source 
this commit avoids lots of tiny allocations, repeated
channel creates which are performed when filtering
the incoming events, unescaping a key just for matching.

also remove deprecated code which is not needed
anymore, avoids unexpected data structure transformations
from the map to slice.
This commit is contained in:
Harshavardhana 2020-04-27 06:25:05 -07:00 committed by GitHub
parent f216670814
commit f14bf25cb9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 103 additions and 480 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/bucket-notification-handlers.go
View file

@ -22,7 +22,6 @@ import (
"errors"
"io"
"net/http"
"net/url"
"path"
"reflect"
"time"
@ -296,11 +295,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
return false
}
objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
if uerr != nil {
objectName = ev.S3.Object.Key
}
return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
})
for _, peer := range peers {

8
cmd/gateway-main.go
View file

@ -131,6 +131,14 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
logger.FatalIf(err, "Failed to read root CAs (%v)", err)
globalMinioEndpoint = func() string {
host := globalMinioHost
if host == "" {
host = sortIPs(localIP4.ToSlice())[0]
}
return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
}()
// Handle gateway specific env
gatewayHandleEnvVars()

2
cmd/globals.go
View file

@ -138,6 +138,8 @@ var (
globalMinioPort = globalMinioDefaultPort
// Holds the host that was passed using --address
globalMinioHost = ""
// Holds the possible host endpoint.
globalMinioEndpoint = ""
// globalConfigSys server config system.
globalConfigSys *ConfigSys

114
cmd/notification.go
View file

@ -22,7 +22,6 @@ import (
"encoding/xml"
"fmt"
"io"
"net"
"net/url"
"path"
"sort"
@ -50,6 +49,7 @@ import (
type NotificationSys struct {
sync.RWMutex
targetList *event.TargetList
targetResCh chan event.TargetIDResult
bucketRulesMap map[string]event.RulesMap
bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
peerClients []*peerRESTClient
@ -662,19 +662,6 @@ func (sys *NotificationSys) AddRemoteTarget(bucketName string, target event.Targ
return nil
}
// RemoteTargetExist - checks whether given target ID is a HTTP/PeerRPC client target or not.
func (sys *NotificationSys) RemoteTargetExist(bucketName string, targetID event.TargetID) bool {
sys.Lock()
defer sys.Unlock()
targetMap, ok := sys.bucketRemoteTargetRulesMap[bucketName]
if ok {
_, ok = targetMap[targetID]
}
return ok
}
// Loads notification policies for all buckets into NotificationSys.
func (sys *NotificationSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
@ -713,6 +700,17 @@ func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error
}
}
go func() {
for res := range sys.targetResCh {
if res.Err != nil {
reqInfo := &logger.ReqInfo{}
reqInfo.AppendTags("targetID", res.ID.Name)
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogOnceIf(ctx, res.Err, res.ID)
}
}
}()
return sys.load(buckets, objAPI)
}
@ -759,7 +757,9 @@ func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID {
for _, rmap := range sys.bucketRulesMap {
for _, rules := range rmap {
for _, targetSet := range rules {
targetIDs = append(targetIDs, targetSet.ToSlice()...)
for id := range targetSet {
targetIDs = append(targetIDs, id)
}
}
}
}
@ -780,69 +780,41 @@ func (sys *NotificationSys) RemoveNotification(bucketName string) {
delete(sys.bucketRulesMap, bucketName)
targetIDSet := event.NewTargetIDSet()
for targetID := range sys.bucketRemoteTargetRulesMap[bucketName] {
sys.targetList.Remove(targetID)
targetIDSet[targetID] = struct{}{}
delete(sys.bucketRemoteTargetRulesMap[bucketName], targetID)
}
sys.targetList.Remove(targetIDSet)
delete(sys.bucketRemoteTargetRulesMap, bucketName)
}
// RemoveAllRemoteTargets - closes and removes all HTTP/PeerRPC client targets.
// RemoveAllRemoteTargets - closes and removes all notification targets.
func (sys *NotificationSys) RemoveAllRemoteTargets() {
sys.Lock()
defer sys.Unlock()
for _, targetMap := range sys.bucketRemoteTargetRulesMap {
for targetID := range targetMap {
sys.targetList.Remove(targetID)
targetIDSet := event.NewTargetIDSet()
for k := range targetMap {
targetIDSet[k] = struct{}{}
}
sys.targetList.Remove(targetIDSet)
}
}
// RemoveRemoteTarget - closes and removes target by target ID.
func (sys *NotificationSys) RemoveRemoteTarget(bucketName string, targetID event.TargetID) {
for terr := range sys.targetList.Remove(targetID) {
reqInfo := (&logger.ReqInfo{}).AppendTags("targetID", terr.ID.Name)
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogIf(ctx, terr.Err)
}
sys.Lock()
defer sys.Unlock()
if _, ok := sys.bucketRemoteTargetRulesMap[bucketName]; ok {
delete(sys.bucketRemoteTargetRulesMap[bucketName], targetID)
if len(sys.bucketRemoteTargetRulesMap[bucketName]) == 0 {
delete(sys.bucketRemoteTargetRulesMap, bucketName)
}
}
}
func (sys *NotificationSys) send(bucketName string, eventData event.Event, targetIDs ...event.TargetID) (errs []event.TargetIDErr) {
errCh := sys.targetList.Send(eventData, targetIDs...)
for terr := range errCh {
errs = append(errs, terr)
if sys.RemoteTargetExist(bucketName, terr.ID) {
sys.RemoveRemoteTarget(bucketName, terr.ID)
}
}
return errs
}
// Send - sends event data to all matching targets.
func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
func (sys *NotificationSys) Send(args eventArgs) {
sys.RLock()
targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
sys.RUnlock()
if len(targetIDSet) == 0 {
return nil
return
}
targetIDs := targetIDSet.ToSlice()
return sys.send(args.BucketName, args.ToEvent(), targetIDs...)
sys.targetList.Send(args.ToEvent(true), targetIDSet, sys.targetResCh)
}
// PutBucketObjectLockConfig - put bucket object lock configuration to all peers.
@ -1204,6 +1176,7 @@ func NewNotificationSys(endpoints EndpointZones) *NotificationSys {
// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
return &NotificationSys{
targetList: event.NewTargetList(),
targetResCh: make(chan event.TargetIDResult),
bucketRulesMap: make(map[string]event.RulesMap),
bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
peerClients: newPeerRestClients(endpoints),
@ -1221,23 +1194,13 @@ type eventArgs struct {
}
// ToEvent - converts to notification event.
func (args eventArgs) ToEvent() event.Event {
getOriginEndpoint := func() string {
host := globalMinioHost
if host == "" {
// FIXME: Send FQDN or hostname of this machine than sending IP address.
host = sortIPs(localIP4.ToSlice())[0]
}
return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
}
func (args eventArgs) ToEvent(escape bool) event.Event {
eventTime := UTCNow()
uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
respElements := map[string]string{
"x-amz-request-id": args.RespElements["requestId"],
"x-minio-origin-endpoint": getOriginEndpoint(), // MinIO specific custom elements.
"x-minio-origin-endpoint": globalMinioEndpoint, // MinIO specific custom elements.
}
// Add deployment as part of
if globalDeploymentID != "" {
@ -1246,6 +1209,10 @@ func (args eventArgs) ToEvent() event.Event {
if args.RespElements["content-length"] != "" {
respElements["content-length"] = args.RespElements["content-length"]
}
keyName := args.Object.Name
if escape {
keyName = url.QueryEscape(args.Object.Name)
}
newEvent := event.Event{
EventVersion: "2.0",
EventSource: "minio:s3",
@ -1264,7 +1231,7 @@ func (args eventArgs) ToEvent() event.Event {
ARN: policy.ResourceARNPrefix + args.BucketName,
},
Object: event.Object{
Key: url.QueryEscape(args.Object.Name),
Key: keyName,
VersionID: "1",
Sequencer: uniqueID,
},
@ -1308,19 +1275,10 @@ func sendEvent(args eventArgs) {
}
if globalHTTPListen.HasSubscribers() {
globalHTTPListen.Publish(args.ToEvent())
globalHTTPListen.Publish(args.ToEvent(false))
}
notifyCh := globalNotificationSys.Send(args)
go func() {
for _, err := range notifyCh {
reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
reqInfo.AppendTags("EventName", args.EventName.String())
reqInfo.AppendTags("targetID", err.ID.Name)
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogOnceIf(ctx, err.Err, err.ID)
}
}()
globalNotificationSys.Send(args)
}
func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) {

72
cmd/peer-rest-client-target.go
View file

@ -1,72 +0,0 @@
/*
* MinIO Cloud Storage, (C) 2018, 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import "github.com/minio/minio/pkg/event"
// PeerRESTClientTarget - RPCClient is an event.Target which sends event to target of remote peer.
type PeerRESTClientTarget struct {
id event.TargetID
remoteTargetID event.TargetID
restClient *peerRESTClient
bucketName string
}
// ID - returns target ID.
func (target *PeerRESTClientTarget) ID() event.TargetID {
return target.id
}
// IsActive - does nothing and available for interface compatibility.
func (target *PeerRESTClientTarget) IsActive() (bool, error) {
return true, nil
}
// HasQueueStore - No-Op. Added for interface compatibility
func (target PeerRESTClientTarget) HasQueueStore() bool {
return false
}
// Save - Sends event directly without persisting.
func (target *PeerRESTClientTarget) Save(eventData event.Event) error {
return target.send(eventData)
}
// Send - interface compatible method does no-op.
func (target *PeerRESTClientTarget) Send(eventKey string) error {
return nil
}
// sends event to remote peer by making RPC call.
func (target *PeerRESTClientTarget) send(eventData event.Event) error {
return target.restClient.SendEvent(target.bucketName, target.id, target.remoteTargetID, eventData)
}
// Close - does nothing and available for interface compatibility.
func (target *PeerRESTClientTarget) Close() error {
return nil
}
// NewPeerRESTClientTarget - creates RPCClient target with given target ID available in remote peer.
func NewPeerRESTClientTarget(bucketName string, targetID event.TargetID, restClient *peerRESTClient) *PeerRESTClientTarget {
return &PeerRESTClientTarget{
id: event.TargetID{ID: targetID.ID, Name: targetID.Name + "+" + mustGetUUID()},
remoteTargetID: targetID,
bucketName: bucketName,
restClient: restClient,
}
}

72
cmd/peer-rest-client.go
View file

@ -500,78 +500,6 @@ func (client *peerRESTClient) ReloadFormat(dryRun bool) error {
return nil
}
// SendEvent - calls send event RPC.
func (client *peerRESTClient) SendEvent(bucket string, targetID, remoteTargetID event.TargetID, eventData event.Event) error {
numTries := 10
for {
err := client.sendEvent(bucket, targetID, remoteTargetID, eventData)
if err == nil {
return nil
}
if numTries == 0 {
return err
}
numTries--
time.Sleep(5 * time.Second)
}
}
func (client *peerRESTClient) sendEvent(bucket string, targetID, remoteTargetID event.TargetID, eventData event.Event) error {
args := sendEventRequest{
TargetID: remoteTargetID,
Event: eventData,
}
values := make(url.Values)
values.Set(peerRESTBucket, bucket)
var reader bytes.Buffer
err := gob.NewEncoder(&reader).Encode(args)
if err != nil {
return err
}
respBody, err := client.call(peerRESTMethodSendEvent, values, &reader, -1)
if err != nil {
return err
}
var eventResp sendEventResp
defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&eventResp)
if err != nil || !eventResp.Success {
reqInfo := &logger.ReqInfo{BucketName: bucket}
reqInfo.AppendTags("targetID", targetID.Name)
reqInfo.AppendTags("event", eventData.EventName.String())
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogIf(ctx, err)
globalNotificationSys.RemoveRemoteTarget(bucket, targetID)
}
return err
}
// RemoteTargetExist - calls remote target ID exist REST API.
func (client *peerRESTClient) RemoteTargetExist(bucket string, targetID event.TargetID) (bool, error) {
values := make(url.Values)
values.Set(peerRESTBucket, bucket)
var reader bytes.Buffer
err := gob.NewEncoder(&reader).Encode(targetID)
if err != nil {
return false, err
}
respBody, err := client.call(peerRESTMethodTargetExists, values, &reader, -1)
if err != nil {
return false, err
}
defer http.DrainBody(respBody)
var targetExists remoteTargetExistsResp
err = gob.NewDecoder(respBody).Decode(&targetExists)
return targetExists.Exists, err
}
// RemoveBucketPolicy - Remove bucket policy on the peer node.
func (client *peerRESTClient) RemoveBucketPolicy(bucket string) error {
values := make(url.Values)

7
cmd/peer-rest-common.go
View file

@ -17,7 +17,7 @@
package cmd
const (
peerRESTVersion = "v7"
peerRESTVersion = "v8"
peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
peerRESTPrefix = minioReservedBucketPath + "/peer"
peerRESTPath = peerRESTPrefix + peerRESTVersionPrefix
@ -52,17 +52,14 @@ const (
peerRESTMethodDownloadProfilingData = "/downloadprofilingdata"
peerRESTMethodBucketPolicySet = "/setbucketpolicy"
peerRESTMethodBucketNotificationPut = "/putbucketnotification"
peerRESTMethodBucketNotificationListen = "/listenbucketnotification"
peerRESTMethodReloadFormat = "/reloadformat"
peerRESTMethodTargetExists = "/targetexists"
peerRESTMethodSendEvent = "/sendevent"
peerRESTMethodTrace = "/trace"
peerRESTMethodListen = "/listen"
peerRESTMethodLog = "/log"
peerRESTMethodBucketLifecycleSet = "/setbucketlifecycle"
peerRESTMethodBucketLifecycleRemove = "/removebucketlifecycle"
peerRESTMethodBucketEncryptionSet = "/setbucketencryption"
peerRESTMethodBucketEncryptionRemove = "/removebucketencryption"
peerRESTMethodLog = "/log"
peerRESTMethodPutBucketObjectLockConfig = "/putbucketobjectlockconfig"
peerRESTMethodBucketObjectLockConfigRemove = "/removebucketobjectlockconfig"
)

100
cmd/peer-rest-server.go
View file

@ -24,7 +24,6 @@ import (
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
@ -774,97 +773,6 @@ func (s *peerRESTServer) SetBucketSSEConfigHandler(w http.ResponseWriter, r *htt
w.(http.Flusher).Flush()
}
type remoteTargetExistsResp struct {
Exists bool
}
// TargetExistsHandler - Check if Target exists.
func (s *peerRESTServer) TargetExistsHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "TargetExists")
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
return
}
vars := mux.Vars(r)
bucketName := vars[peerRESTBucket]
if bucketName == "" {
s.writeErrorResponse(w, errors.New("Bucket name is missing"))
return
}
var targetID event.TargetID
if r.ContentLength <= 0 {
s.writeErrorResponse(w, errInvalidArgument)
return
}
err := gob.NewDecoder(r.Body).Decode(&targetID)
if err != nil {
s.writeErrorResponse(w, err)
return
}
var targetExists remoteTargetExistsResp
targetExists.Exists = globalNotificationSys.RemoteTargetExist(bucketName, targetID)
defer w.(http.Flusher).Flush()
logger.LogIf(ctx, gob.NewEncoder(w).Encode(&targetExists))
}
type sendEventRequest struct {
Event event.Event
TargetID event.TargetID
}
type sendEventResp struct {
Success bool
}
// SendEventHandler - Send Event.
func (s *peerRESTServer) SendEventHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
return
}
ctx := newContext(r, w, "SendEvent")
vars := mux.Vars(r)
bucketName := vars[peerRESTBucket]
if bucketName == "" {
s.writeErrorResponse(w, errors.New("Bucket name is missing"))
return
}
var eventReq sendEventRequest
if r.ContentLength <= 0 {
s.writeErrorResponse(w, errInvalidArgument)
return
}
err := gob.NewDecoder(r.Body).Decode(&eventReq)
if err != nil {
s.writeErrorResponse(w, err)
return
}
var eventResp sendEventResp
eventResp.Success = true
errs := globalNotificationSys.send(bucketName, eventReq.Event, eventReq.TargetID)
for i := range errs {
reqInfo := (&logger.ReqInfo{}).AppendTags("Event", eventReq.Event.EventName.String())
reqInfo.AppendTags("targetName", eventReq.TargetID.Name)
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogIf(ctx, errs[i].Err)
eventResp.Success = false
s.writeErrorResponse(w, errs[i].Err)
return
}
logger.LogIf(ctx, gob.NewEncoder(w).Encode(&eventResp))
w.(http.Flusher).Flush()
}
// PutBucketNotificationHandler - Set bucket policy.
func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
@ -1077,11 +985,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
return false
}
objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
if uerr != nil {
objectName = ev.S3.Object.Key
}
return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
})
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
@ -1239,8 +1143,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProfilingDataHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTargetExists).HandlerFunc(httpTraceHdrs(server.TargetExistsHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSendEvent).HandlerFunc(httpTraceHdrs(server.SendEventHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBucketNotificationPut).HandlerFunc(httpTraceHdrs(server.PutBucketNotificationHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)

9
cmd/server-main.go
View file

@ -111,7 +111,6 @@ func serverHandleCmdArgs(ctx *cli.Context) {
globalMinioAddr = globalCLIContext.Addr
globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
endpoints := strings.Fields(env.Get(config.EnvEndpoints, ""))
if len(endpoints) > 0 {
globalEndpoints, globalXLSetDriveCount, setupType, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
@ -344,6 +343,14 @@ func serverMain(ctx *cli.Context) {
globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
logger.FatalIf(err, "Failed to read root CAs (%v)", err)
globalMinioEndpoint = func() string {
host := globalMinioHost
if host == "" {
host = sortIPs(localIP4.ToSlice())[0]
}
return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
}()
// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
if globalIsDistXL {
if globalEndpoints.HTTPS() && !globalIsSSL {

2
go.mod
View file

@ -67,7 +67,7 @@ require (
github.com/minio/hdfs/v3 v3.0.1
github.com/minio/highwayhash v1.0.0
github.com/minio/lsync v1.0.1
github.com/minio/minio-go/v6 v6.0.53
github.com/minio/minio-go/v6 v6.0.55-0.20200424204115-7506d2996b22
github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61
github.com/minio/sha256-simd v0.1.1
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37

2
go.sum
View file

@ -276,6 +276,8 @@ github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs=
github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA=
github.com/minio/minio-go/v6 v6.0.53 h1:8jzpwiOzZ5Iz7/goFWqNZRICbyWYShbb5rARjrnSCNI=
github.com/minio/minio-go/v6 v6.0.53/go.mod h1:DIvC/IApeHX8q1BAMVCXSXwpmrmM+I+iBvhvztQorfI=
github.com/minio/minio-go/v6 v6.0.55-0.20200424204115-7506d2996b22 h1:nZEve4vdUhwHBoV18zRvPDgjL6NYyDJE5QJvz3l9bRs=
github.com/minio/minio-go/v6 v6.0.55-0.20200424204115-7506d2996b22/go.mod h1:KQMM+/44DSlSGSQWSfRrAZ12FVMmpWNuX37i2AX0jfI=
github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61 h1:pUSI/WKPdd77gcuoJkSzhJ4wdS8OMDOsOu99MtpXEQA=
github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61/go.mod h1:4trzEJ7N1nBTd5Tt7OCZT5SEin+WiAXpdJ/WgPkESA8=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=

10
pkg/event/rules.go
View file

@ -53,6 +53,16 @@ func (rules Rules) Add(pattern string, targetID TargetID) {
rules[pattern] = NewTargetIDSet(targetID).Union(rules[pattern])
}
// MatchSimple - returns true one of the matching object name in rules.
func (rules Rules) MatchSimple(objectName string) bool {
for pattern := range rules {
if wildcard.MatchSimple(pattern, objectName) {
return true
}
}
return false
}
// Match - returns TargetIDSet matching object name in rules.
func (rules Rules) Match(objectName string) TargetIDSet {
targetIDs := NewTargetIDSet()

5
pkg/event/rulesmap.go
View file

@ -60,6 +60,11 @@ func (rulesMap RulesMap) Remove(rulesMap2 RulesMap) {
}
}
// MatchSimple - returns true if matching object name and event name in rules map.
func (rulesMap RulesMap) MatchSimple(eventName Name, objectName string) bool {
return rulesMap[eventName].MatchSimple(objectName)
}
// Match - returns TargetIDSet matching object name and event name in rules map.
func (rulesMap RulesMap) Match(eventName Name, objectName string) TargetIDSet {
return rulesMap[eventName].Match(objectName)

17
pkg/event/targetidset.go
View file

@ -16,23 +16,12 @@
package event
import "fmt"
// TargetIDSet - Set representation of TargetIDs.
type TargetIDSet map[TargetID]struct{}
// ToSlice - returns TargetID slice from TargetIDSet.
func (set TargetIDSet) ToSlice() []TargetID {
keys := make([]TargetID, 0, len(set))
for k := range set {
keys = append(keys, k)
}
return keys
}
// String - returns string representation.
func (set TargetIDSet) String() string {
return fmt.Sprintf("%v", set.ToSlice())
// IsEmpty returns true if the set is empty.
func (set TargetIDSet) IsEmpty() bool {
return len(set) != 0
}
// Clone - returns copy of this set.

50
pkg/event/targetidset_test.go
View file

@ -21,56 +21,6 @@ import (
"testing"
)
func TestTargetIDSetToSlice(t *testing.T) {
testCases := []struct {
set TargetIDSet
expectedResult []TargetID
}{
{NewTargetIDSet(), []TargetID{}},
{NewTargetIDSet(TargetID{"1", "webhook"}), []TargetID{{"1", "webhook"}}},
{NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), []TargetID{{"1", "webhook"}, {"2", "amqp"}}},
}
for i, testCase := range testCases {
result := testCase.set.ToSlice()
if len(result) != len(testCase.expectedResult) {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
for _, targetID1 := range result {
var found bool
for _, targetID2 := range testCase.expectedResult {
if reflect.DeepEqual(targetID1, targetID2) {
found = true
break
}
}
if !found {
t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
}
func TestTargetIDSetString(t *testing.T) {
testCases := []struct {
set TargetIDSet
expectedResult string
}{
{NewTargetIDSet(), "[]"},
{NewTargetIDSet(TargetID{"1", "webhook"}), "[1:webhook]"},
}
for i, testCase := range testCases {
result := testCase.set.String()
if result != testCase.expectedResult {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
func TestTargetIDSetClone(t *testing.T) {
testCases := []struct {
set TargetIDSet

63
pkg/event/targetlist.go
View file

@ -61,8 +61,9 @@ func (list *TargetList) Exists(id TargetID) bool {
return found
}
// TargetIDErr returns error associated for a targetID
type TargetIDErr struct {
// TargetIDResult returns result of Remove/Send operation, sets err if
// any for the associated TargetID
type TargetIDResult struct {
// ID where the remove or send were initiated.
ID TargetID
// Stores any error while removing a target or while sending an event.
@ -70,40 +71,17 @@ type TargetIDErr struct {
}
// Remove - closes and removes targets by given target IDs.
func (list *TargetList) Remove(targetids ...TargetID) <-chan TargetIDErr {
errCh := make(chan TargetIDErr)
func (list *TargetList) Remove(targetIDSet TargetIDSet) {
list.Lock()
defer list.Unlock()
go func() {
defer close(errCh)
var wg sync.WaitGroup
for _, id := range targetids {
list.RLock()
target, ok := list.targets[id]
list.RUnlock()
if ok {
wg.Add(1)
go func(id TargetID, target Target) {
defer wg.Done()
if err := target.Close(); err != nil {
errCh <- TargetIDErr{
ID: id,
Err: err,
}
}
}(id, target)
}
}
wg.Wait()
list.Lock()
for _, id := range targetids {
for id := range targetIDSet {
target, ok := list.targets[id]
if ok {
target.Close()
delete(list.targets, id)
}
list.Unlock()
}()
return errCh
}
}
// Targets - list all targets
@ -140,14 +118,10 @@ func (list *TargetList) TargetMap() map[TargetID]Target {
}
// Send - sends events to targets identified by target IDs.
func (list *TargetList) Send(event Event, targetIDs ...TargetID) <-chan TargetIDErr {
errCh := make(chan TargetIDErr)
func (list *TargetList) Send(event Event, targetIDset TargetIDSet, resCh chan<- TargetIDResult) {
go func() {
defer close(errCh)
var wg sync.WaitGroup
for _, id := range targetIDs {
for id := range targetIDset {
list.RLock()
target, ok := list.targets[id]
list.RUnlock()
@ -155,19 +129,18 @@ func (list *TargetList) Send(event Event, targetIDs ...TargetID) <-chan TargetID
wg.Add(1)
go func(id TargetID, target Target) {
defer wg.Done()
tgtRes := TargetIDResult{ID: id}
if err := target.Save(event); err != nil {
errCh <- TargetIDErr{
ID: id,
Err: err,
}
tgtRes.Err = err
}
resCh <- tgtRes
}(id, target)
} else {
resCh <- TargetIDResult{ID: id}
}
}
wg.Wait()
}()
return errCh
}
// NewTargetList - creates TargetList.

43
pkg/event/targetlist_test.go
View file

@ -163,40 +163,6 @@ func TestTargetListExists(t *testing.T) {
}
}
func TestTargetListRemove(t *testing.T) {
targetListCase1 := NewTargetList()
targetListCase2 := NewTargetList()
if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
panic(err)
}
targetListCase3 := NewTargetList()
if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, true}); err != nil {
panic(err)
}
testCases := []struct {
targetList *TargetList
targetID TargetID
expectErr bool
}{
{targetListCase1, TargetID{"1", "webhook"}, false},
{targetListCase2, TargetID{"1", "webhook"}, false},
{targetListCase3, TargetID{"3", "testcase"}, true},
}
for i, testCase := range testCases {
errCh := testCase.targetList.Remove(testCase.targetID)
err := <-errCh
expectErr := (err.Err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
}
}
}
func TestTargetListList(t *testing.T) {
targetListCase1 := NewTargetList()
@ -273,10 +239,13 @@ func TestTargetListSend(t *testing.T) {
{targetListCase4, TargetID{"4", "testcase"}, true},
}
resCh := make(chan TargetIDResult)
for i, testCase := range testCases {
errCh := testCase.targetList.Send(Event{}, testCase.targetID)
err := <-errCh
expectErr := (err.Err != nil)
testCase.targetList.Send(Event{}, map[TargetID]struct{}{
testCase.targetID: {},
}, resCh)
res := <-resCh
expectErr := (res.Err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)