Create & Init Project...

app/service/ops/log-agent/processor/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["processor.go"],
+    importpath = "go-common/app/service/ops/log-agent/processor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/output:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/processor/classify:all-srcs",
+        "//app/service/ops/log-agent/processor/fileLog:all-srcs",
+        "//app/service/ops/log-agent/processor/grok:all-srcs",
+        "//app/service/ops/log-agent/processor/httpstream:all-srcs",
+        "//app/service/ops/log-agent/processor/jsonLog:all-srcs",
+        "//app/service/ops/log-agent/processor/lengthCheck:all-srcs",
+        "//app/service/ops/log-agent/processor/sample:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/classify/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "classify.go",
+        "config.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/classify",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/classify/classify.go

@@ -0,0 +1,80 @@
+package classify
+
+import (
+	"strings"
+	"context"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+)
+
+type Classify struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("classify", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	classify := new(Classify)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for Classify Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		classify.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// only do classify for ops-log
+				if e.Destination == "lancer-ops-log" {
+					if common.CriticalLog(e.Level) || e.Priority == "high" {
+						e.LogId = classify.getLogIdByLevel("important")
+					} else {
+						e.LogId = classify.getLogIdByAppId(e.AppId)
+					}
+				}
+				output <- e
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}
+
+// getLogLevel get logId level by appId
+func (c *Classify) getLogIdByAppId(appId []byte) (logId string) {
+	// get logId by setting
+	if logLevel, ok := c.c.LogLevelMapConfig[string(appId)]; ok {
+		return c.getLogIdByLevel(logLevel)
+	}
+
+	// appId format error, logId 1
+	if len(strings.Split(string(appId), ".")) < 3 {
+		return c.getLogIdByLevel("low") // low level
+	}
+
+	// set logLevel to 2 by default
+	return c.getLogIdByLevel("normal") // normal level
+}
+
+// getLogIdByLevel return logid by level
+func (c *Classify) getLogIdByLevel(level string) (logId string) {
+	if logId, ok := c.c.LogIdMapConfig[level]; ok {
+		return logId
+	} else {
+		// return 000161 by default
+		return "000161"
+	}
+}

app/service/ops/log-agent/processor/classify/config.go

@@ -0,0 +1,101 @@
+package classify
+
+import (
+	"fmt"
+	"errors"
+	"time"
+
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	logLevelMap = "logLevelMap.toml"
+	logIdMap    = "logIdMap.toml"
+)
+
+type Config struct {
+	Local             bool              `toml:"local"`
+	LogLevelMapConfig map[string]string `toml:"logLevelMapConfig"`
+	LogIdMapConfig    map[string]string `toml:"logIdMapConfig"`
+	PriorityBlackList map[string]string `toml:"priorityBlackList"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	if c.LogLevelMapConfig == nil {
+		c.LogLevelMapConfig = make(map[string]string)
+	}
+
+	if c.LogIdMapConfig == nil {
+		return fmt.Errorf("LogIdMapConfig of classify can't be nil")
+	}
+
+	if c.PriorityBlackList == nil {
+		c.PriorityBlackList = make(map[string]string)
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	config := new(Config)
+	if err = md.PrimitiveDecode(primValue, config); err != nil {
+		return nil, err
+	}
+
+	// read config from config center
+	if !config.Local {
+		if err = config.readConfig(); err != nil {
+			return nil, err
+		}
+		// watch update and reload config
+		go func() {
+			currentVersion := configcenter.Version
+			for {
+				if currentVersion != configcenter.Version {
+					log.Info("classify config reload")
+					if err := config.readConfig(); err != nil {
+						log.Error("classify config reload error (%v)", err)
+					}
+					currentVersion = configcenter.Version
+				}
+				time.Sleep(time.Second)
+			}
+		}()
+	}
+
+	return config, nil
+}
+
+func (c *Config) readConfig() (err error) {
+	var (
+		ok             bool
+		value          string
+		tmplogLevelMap map[string]string
+		tmplogIdMap    map[string]string
+	)
+
+	// logLevel config
+	if value, ok = configcenter.Client.Value(logLevelMap); !ok {
+		return errors.New("failed to get logLevelMap.toml")
+	}
+	if _, err = toml.Decode(value, &tmplogLevelMap); err != nil {
+		return err
+	}
+	c.LogLevelMapConfig = tmplogLevelMap
+
+	// logIdMap config
+	if value, ok = configcenter.Client.Value(logIdMap); !ok {
+		return errors.New("failed to get logIdMap.toml")
+	}
+	if _, err = toml.Decode(value, &tmplogIdMap); err != nil {
+		return err
+	}
+	c.LogIdMapConfig = tmplogIdMap
+
+	return nil
+}

app/service/ops/log-agent/processor/fileLog/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "fileLog.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/fileLog",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/fileLog/config.go

@@ -0,0 +1,24 @@
+package fileLog
+
+import (
+	"fmt"
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/processor/fileLog/fileLog.go

@@ -0,0 +1,94 @@
+package fileLog
+
+import (
+	"time"
+	"context"
+	"encoding/json"
+	"os"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+)
+
+const (
+	_logIdLen           = 6
+	_logLancerHeaderLen = 19
+	_appIdKey           = `"app_id":`
+	_levelKey           = `"level":`
+	_logTime            = `"time":`
+)
+
+var (
+	local, _    = time.LoadLocation("Local")
+	hostname, _ = os.Hostname()
+)
+
+type FileLog struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("fileLog", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	fileLog := new(FileLog)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for jsonLog Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		fileLog.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// only do jsonLog for ops-log
+				if e.Destination != "lancer-ops-log" {
+					output <- e
+					continue
+				}
+
+				// format message
+				message := make(map[string]interface{})
+				if len(e.ParsedFields) != 0 {
+					for k, v := range e.ParsedFields {
+						message[k] = v
+					}
+				}
+
+				message["log"] = e.String()
+
+				e.Fields["hostname"] = hostname
+
+				if len(e.Tags) != 0 {
+					e.Fields["tag"] = e.Tags
+				}
+
+				if len(e.Fields) != 0 {
+					message["fields"] = e.Fields
+				}
+
+				message["app_id"] = string(e.AppId)
+				message["time"] = e.Time.UTC().Format(time.RFC3339Nano)
+
+				if body, err := json.Marshal(message); err == nil {
+					e.Write(body)
+					output <- e
+				}
+
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}

app/service/ops/log-agent/processor/grok/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "grok.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/grok",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/vjeantet/grok:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/grok/conf.go

@@ -0,0 +1,30 @@
+package grok
+
+import (
+	"fmt"
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	Pattern string `toml:"pattern"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("config of grok can't be nil")
+	}
+
+	if c.Pattern == "" {
+		return fmt.Errorf("pattern of grok can't be nil")
+	}
+
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/processor/grok/grok.go

@@ -0,0 +1,62 @@
+package grok
+
+import (
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+
+	"github.com/vjeantet/grok"
+)
+
+type Grok struct {
+	c *Config
+	g *grok.Grok
+}
+
+func init() {
+	err := processor.Register("grok", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	g := new(Grok)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for Grok Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		g.c = c
+	}
+
+	if g.g, err = grok.New(); err != nil {
+		return nil, err
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				values, err := g.g.Parse(g.c.Pattern, e.String())
+				if err != nil || len(values) == 0 {
+					flowmonitor.Fm.AddEvent(e, "log-agent.processor.grok", "WARN", "grok error")
+					e.Tags = append(e.Tags, "grok_error")
+					output <- e
+					continue
+				}
+				e.ParsedFields = values
+				output <- e
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return output, nil
+}

app/service/ops/log-agent/processor/httpstream/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "httpstream.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/httpstream",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/httpstream:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/httpstream/conf.go

@@ -0,0 +1,24 @@
+package httpstream
+
+import (
+	"fmt"
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/processor/httpstream/httpstream.go

@@ -0,0 +1,51 @@
+package httpstream
+
+import (
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	phttpstream "go-common/app/service/ops/log-agent/pkg/httpstream"
+)
+
+type HttpStream struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("httpStream", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	h := new(HttpStream)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for lengthCheck Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		h.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				select {
+				case phttpstream.LogSourceChan <- e:
+				default:
+				}
+				output <- e
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return output, nil
+}

app/service/ops/log-agent/processor/jsonLog/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "jsonLog.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/jsonLog",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/jsonLog/conf.go

@@ -0,0 +1,24 @@
+package jsonLog
+
+import (
+	"fmt"
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/processor/jsonLog/jsonLog.go

@@ -0,0 +1,105 @@
+package jsonLog
+
+import (
+	"time"
+	"strconv"
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+)
+
+const (
+	_appIdKey = `"app_id":`
+	_levelKey = `"level":`
+	_logTime  = `"time":`
+)
+
+var (
+	local, _ = time.LoadLocation("Local")
+)
+
+type JsonLog struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("jsonLog", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	jsonLog := new(JsonLog)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for jsonLog Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		jsonLog.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	var (
+		t time.Time
+	)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// only do jsonLog for ops-log
+				if e.Destination != "lancer-ops-log" {
+					output <- e
+					continue
+				}
+				if e.Length == 0 {
+					event.PutEvent(e)
+					continue
+				}
+
+				// seek app_id
+				if appId, err := common.SeekValue([]byte(_appIdKey), e.Bytes()); err == nil {
+					e.AppId = appId
+				}
+
+				// priority
+				if priority, err := common.GetPriority(e.Bytes()); err == nil {
+					e.Priority = string(priority)
+				}
+
+				// seek time
+				if timeValue, err := common.SeekValue([]byte(_logTime), e.Bytes()); err == nil {
+					if len(timeValue) >= 19 {
+						// parse time
+						if t, err = time.Parse(time.RFC3339Nano, string(timeValue)); err != nil {
+							if t, err = time.ParseInLocation("2006-01-02T15:04:05", string(timeValue), local); err != nil {
+								if t, err = time.ParseInLocation("2006-01-02T15:04:05", string(timeValue[0:19]), local); err != nil {
+								}
+							}
+						}
+						e.Time = t
+					}
+				}
+
+				// TimeRangeKey for flow monitor
+				if !e.Time.IsZero() {
+					e.TimeRangeKey = strconv.FormatInt(e.Time.Unix()/100*100, 10)
+				}
+				// seek level
+				if level, err := common.SeekValue([]byte(_levelKey), e.Bytes()); err == nil {
+					e.Level = level
+				}
+				flowmonitor.Fm.AddEvent(e, "log-agent.processor.jsonLog", "OK", "received")
+				output <- e
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}

app/service/ops/log-agent/processor/lengthCheck/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "lengthCheck.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/lengthCheck",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/lengthCheck/config.go

@@ -0,0 +1,30 @@
+package lengthCheck
+
+import (
+	"fmt"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	MaxLength int `toml:"maxLength"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	if c.MaxLength == 0 || c.MaxLength > 1024*32 {
+		c.MaxLength = 1024 * 32 //32K by default
+	}
+
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/processor/lengthCheck/lengthCheck.go

@@ -0,0 +1,52 @@
+package lengthCheck
+
+import (
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+)
+
+type LengthCheck struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("lengthCheck", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	lcheck := new(LengthCheck)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for lengthCheck Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		lcheck.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// log length check
+				if e.Length > lcheck.c.MaxLength {
+					flowmonitor.Fm.AddEvent(e, "log-agent.processor.lengthCheck", "ERROR", "too long")
+					event.PutEvent(e)
+					continue
+				}
+				output <- e
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}

app/service/ops/log-agent/processor/processor.go

@@ -0,0 +1,62 @@
+package processor
+
+import (
+	"fmt"
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/output"
+)
+
+// Factory is used to register functions creating new output instances.
+type Factory = func(cxt context.Context, config interface{}, input <-chan *event.ProcessorEvent) (chan *event.ProcessorEvent, error)
+
+var registry = make(map[string]Factory)
+
+func Register(name string, factory Factory) error {
+	log.Info("Registering  processor factory")
+	if name == "" {
+		return fmt.Errorf("Error registering processor: name cannot be empty")
+	}
+	if factory == nil {
+		return fmt.Errorf("Error registering processor '%v': factory cannot be empty", name)
+	}
+	if _, exists := registry[name]; exists {
+		return fmt.Errorf("Error registering processor '%v': already registered", name)
+	}
+
+	registry[name] = factory
+	log.Info("Successfully registered processor: '%v'", name)
+
+	return nil
+}
+
+func GetFactory(name string) (Factory, error) {
+	if _, exists := registry[name]; !exists {
+		return nil, fmt.Errorf("Error creating processor. No such processor type exist: '%v'", name)
+	}
+	return registry[name], nil
+}
+
+func WriteToOutput(ctx context.Context, dest string, input <-chan *event.ProcessorEvent) (err error) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case e := <-input:
+				if dest != "" {
+					e.Destination = dest
+				}
+				outputChan, err := output.GetOutputChan(e.Destination)
+				if err != nil {
+					log.Error("failed to get output chan:%s; discard log", err)
+					continue
+				}
+				outputChan <- e
+			}
+		}
+	}()
+	return nil
+}

app/service/ops/log-agent/processor/sample/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "sample.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/sample",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/sample/conf.go

@@ -0,0 +1,79 @@
+package sample
+
+import (
+	"fmt"
+	"errors"
+	"time"
+
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	logSample = "sample.toml"
+)
+
+type Config struct {
+	Local        bool             `toml:"local"`
+	SampleConfig map[string]int64 `toml:"sampleConfig"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	if c.SampleConfig == nil {
+		c.SampleConfig = make(map[string]int64)
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	config := new(Config)
+	if err = md.PrimitiveDecode(primValue, config); err != nil {
+		return nil, err
+	}
+
+	// read config from config center
+	if !config.Local {
+		if err = config.readConfig(); err != nil {
+			return nil, err
+		}
+		// watch update and reload config
+		go func() {
+			currentVersion := configcenter.Version
+			for {
+				if currentVersion != configcenter.Version {
+					log.Info("sample config reload")
+					if err := config.readConfig(); err != nil {
+						log.Error("sample config reload error (%v", err)
+					}
+					currentVersion = configcenter.Version
+				}
+				time.Sleep(time.Second)
+			}
+		}()
+	}
+
+	return config, nil
+}
+
+func (c *Config) readConfig() (err error) {
+	var (
+		ok        bool
+		value     string
+		tmpSample map[string]int64
+	)
+
+	// sample config
+	if value, ok = configcenter.Client.Value(logSample); !ok {
+		return errors.New("failed to get sample.toml")
+	}
+	if _, err = toml.Decode(value, &tmpSample); err != nil {
+		return err
+	}
+	c.SampleConfig = tmpSample
+	return nil
+}

app/service/ops/log-agent/processor/sample/sample.go

@@ -0,0 +1,77 @@
+package sample
+
+import (
+	"math/rand"
+	"context"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+)
+
+type Sample struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("sample", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	sample := new(Sample)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for sample Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		sample.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// only do sample for ops-log
+				if e.Destination != "lancer-ops-log" {
+					output <- e
+					continue
+				}
+				if !sample.sample(e) {
+					output <- e
+				} else {
+					flowmonitor.Fm.AddEvent(e, "log-agent.processor.sample", "WARN", "sampled")
+					event.PutEvent(e)
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}
+
+//sample log, if return ture, the log should be discard
+func (s *Sample) sample(e *event.ProcessorEvent) bool {
+	if common.CriticalLog(e.Level) {
+		return false // keep log if level isn't INFO or DEBUG
+	}
+
+	if e.Priority == "high" {
+		return false
+	}
+
+	if val, ok := s.c.SampleConfig[string(e.AppId)]; ok {
+
+		if rand.Intn(100) < 100-int(val) {
+			return true // discard
+		}
+	}
+	return false
+}