Create & Init Project...

app/service/ops/log-agent/processor/sample/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "sample.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/sample",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/sample/conf.go

@@ -0,0 +1,79 @@
+package sample
+
+import (
+	"fmt"
+	"errors"
+	"time"
+
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	logSample = "sample.toml"
+)
+
+type Config struct {
+	Local        bool             `toml:"local"`
+	SampleConfig map[string]int64 `toml:"sampleConfig"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	if c.SampleConfig == nil {
+		c.SampleConfig = make(map[string]int64)
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	config := new(Config)
+	if err = md.PrimitiveDecode(primValue, config); err != nil {
+		return nil, err
+	}
+
+	// read config from config center
+	if !config.Local {
+		if err = config.readConfig(); err != nil {
+			return nil, err
+		}
+		// watch update and reload config
+		go func() {
+			currentVersion := configcenter.Version
+			for {
+				if currentVersion != configcenter.Version {
+					log.Info("sample config reload")
+					if err := config.readConfig(); err != nil {
+						log.Error("sample config reload error (%v", err)
+					}
+					currentVersion = configcenter.Version
+				}
+				time.Sleep(time.Second)
+			}
+		}()
+	}
+
+	return config, nil
+}
+
+func (c *Config) readConfig() (err error) {
+	var (
+		ok        bool
+		value     string
+		tmpSample map[string]int64
+	)
+
+	// sample config
+	if value, ok = configcenter.Client.Value(logSample); !ok {
+		return errors.New("failed to get sample.toml")
+	}
+	if _, err = toml.Decode(value, &tmpSample); err != nil {
+		return err
+	}
+	c.SampleConfig = tmpSample
+	return nil
+}

app/service/ops/log-agent/processor/sample/sample.go

@@ -0,0 +1,77 @@
+package sample
+
+import (
+	"math/rand"
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+)
+
+type Sample struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("sample", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	sample := new(Sample)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for sample Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		sample.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// only do sample for ops-log
+				if e.Destination != "lancer-ops-log" {
+					output <- e
+					continue
+				}
+				if !sample.sample(e) {
+					output <- e
+				} else {
+					flowmonitor.Fm.AddEvent(e, "log-agent.processor.sample", "WARN", "sampled")
+					event.PutEvent(e)
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}
+
+//sample log, if return ture, the log should be discard
+func (s *Sample) sample(e *event.ProcessorEvent) bool {
+	if common.CriticalLog(e.Level) {
+		return false // keep log if level isn't INFO or DEBUG
+	}
+
+	if e.Priority == "high" {
+		return false
+	}
+
+	if val, ok := s.c.SampleConfig[string(e.AppId)]; ok {
+
+		if rand.Intn(100) < 100-int(val) {
+			return true // discard
+		}
+	}
+	return false
+}