Create & Init Project...

app/interface/openplatform/monitor-end/service/BUILD

@@ -0,0 +1,77 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "alert_test.go",
+        "consumer_test.go",
+        "monitor_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/model:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/kafka:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/monitor:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/container/pool:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "alert.go",
+        "consumer.go",
+        "infoc.go",
+        "mail.go",
+        "monitor.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/interface/openplatform/monitor-end/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/dao:go_default_library",
+        "//app/interface/openplatform/monitor-end/model:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/kafka:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/monitor:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/prom:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/bsm/sarama-cluster:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+        "//vendor/github.com/scorredoira/email:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/service/alert.go

@@ -0,0 +1,412 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/ecode"
+)
+
+var (
+	_duplicateNameErr    = errors.New("组名已经存在！")
+	_duplicateTargetErr  = errors.New("已经存在该告警sub_event!")
+	_invalidTargetIDErr  = errors.New("无效的id！")
+	_duplicateProductErr = errors.New("重复的product name")
+)
+
+func targetKey(t *model.Target) string {
+	if t.Source == "" || t.Product == "" || t.Event == "" || t.SubEvent == "" {
+		return ""
+	}
+	return fmt.Sprintf("%s_%s_%s_%s", t.Source, t.Product, t.Event, t.SubEvent)
+
+}
+
+func productKey(p string) string {
+	if p == "" {
+		return ""
+	}
+	return fmt.Sprintf("%s", p)
+}
+
+func (s *Service) loadalertsettings() {
+	var (
+		c   = context.Background()
+		gm  = make(map[int64]*model.Group)
+		tm  = make(map[int64]*model.Target)
+		tmk = make(map[string]*model.Target)
+		tmn = make(map[string]*model.Target)
+		pm  = make(map[int64]*model.Product)
+		pmk = make(map[string]*model.Product)
+	)
+	if gs, err := s.dao.AllGroups(c); err == nil {
+		for _, g := range gs {
+			gm[g.ID] = g
+		}
+	}
+
+	if ts, err := s.dao.AllTargets(c, 1); err == nil {
+		for _, t := range ts {
+			tm[t.ID] = t
+			if key := targetKey(t); key != "" {
+				tmk[key] = t
+			}
+		}
+	}
+	if ts, err := s.dao.AllTargets(c, 0); err == nil {
+		for _, t := range ts {
+			if key := targetKey(t); key != "" {
+				tmn[key] = t
+			}
+		}
+	}
+	if ps, err := s.dao.AllProducts(c); err == nil {
+		for _, p := range ps {
+			pm[p.ID] = p
+			if key := productKey(p.Name); err == nil {
+				pmk[key] = p
+			}
+		}
+	}
+	s.mapMutex.Lock()
+	s.groups = gm
+	s.targets = tm
+	s.products = pm
+	s.targetKeys = tmk
+	s.productKeys = pmk
+	s.newTargets = tmn
+	s.mapMutex.Unlock()
+}
+
+// AddGroup add a new group.
+func (s *Service) AddGroup(c context.Context, group *model.Group) (id int64, err error) {
+	var g *model.Group
+	if group.Name == "" || group.Receivers == "" || group.Interval < 0 {
+		err = ecode.RequestErr
+		return
+	}
+	if g, err = s.GroupByName(c, group.Name); err != nil {
+		return
+	}
+	if g.ID > 0 {
+		err = _duplicateNameErr
+		return
+	}
+	if group.Interval == 0 {
+		group.Interval = 30
+	}
+	return s.dao.AddGroup(c, group)
+}
+
+// UpdateGroup update group.
+func (s *Service) UpdateGroup(c context.Context, group *model.Group) (err error) {
+	var g *model.Group
+	if group.ID == 0 || group.Name == "" || group.Receivers == "" || group.Interval < 0 {
+		err = ecode.RequestErr
+		return
+	}
+	if g, err = s.GroupByName(c, group.Name); err != nil {
+		return
+	}
+	if g.ID != 0 && g.ID != group.ID {
+		err = _duplicateNameErr
+		return
+	}
+	if group.Interval == 0 {
+		group.Interval = 30
+	}
+	_, err = s.dao.UpdateGroup(c, group)
+	return
+}
+
+// DeleteGroup delete group.
+func (s *Service) DeleteGroup(c context.Context, id int64) (err error) {
+	if id == 0 {
+		err = ecode.RequestErr
+		return
+	}
+	_, err = s.dao.DeleteGroup(c, id)
+	return
+}
+
+// GroupList return all groups.
+func (s *Service) GroupList(c context.Context, params *model.GroupListParams) (res *model.Groups, err error) {
+	res = &model.Groups{}
+	if res.Groups, err = s.dao.AllGroups(c); err != nil {
+		return
+	}
+	res.Total = len(res.Groups)
+	return
+}
+
+// GroupByName get group by name.
+func (s *Service) GroupByName(c context.Context, name string) (res *model.Group, err error) {
+	return s.dao.GroupByName(c, name)
+}
+
+// Target get target by id.
+func (s *Service) Target(c context.Context, id int64) (res *model.Target, err error) {
+	return s.dao.Target(c, id)
+}
+
+// AddTarget add a new target.
+func (s *Service) AddTarget(c context.Context, t *model.Target) (id int64, err error) {
+	if t.SubEvent == "" || t.Event == "" || t.Product == "" || t.Source == "" {
+		err = ecode.RequestErr
+		return
+	}
+	if err = s.checkTarget(c, t, true); err != nil {
+		return
+	}
+	return s.dao.AddTarget(c, t)
+}
+
+// UpdateTarget update target.
+func (s *Service) UpdateTarget(c context.Context, t *model.Target) (err error) {
+	var (
+		oldTarget *model.Target
+	)
+	if oldTarget, err = s.Target(c, t.ID); err != nil {
+		return
+	}
+	if oldTarget == nil {
+		err = _invalidTargetIDErr
+		return
+	}
+	mergeTarget(t, oldTarget)
+	if err = s.checkTarget(c, t, false); err != nil {
+		return
+	}
+	_, err = s.dao.UpdateTarget(c, t)
+	return
+}
+
+// TargetList .
+func (s *Service) TargetList(c context.Context, t *model.Target, pn int, ps int, sort string) (res *model.Targets, err error) {
+	// query := "SELECT id, sub_event, event, product, source, group_id, threshold, duration, state FROM target"
+	var (
+		where      = ""
+		order      = ""
+		empty      struct{}
+		sortFields = map[string]struct{}{
+			"sub_event": empty,
+			"mtime":     empty,
+			"ctime":     empty,
+			"state":     empty,
+		}
+		sortOrder = map[string]string{
+			"0": "DESC",
+			"1": "ASC",
+		}
+	)
+	if t.SubEvent != "" {
+		where += " sub_event LIKE '%" + t.SubEvent + "%'"
+	}
+	if t.Event != "" {
+		where += " event = '" + t.Event + "'"
+	}
+	if t.Product != "" {
+		where += " product = '" + t.Product + "'"
+	}
+	if t.Source != "" {
+		where += " source = '" + t.Source + "'"
+	}
+	if t.States != "" {
+		where += " state in (" + t.States + ")"
+	}
+	if where == "" {
+		where = " WHERE" + where + " deleted_time = 0"
+	} else {
+		where = " WHERE" + where + " AND deleted_time = 0"
+	}
+	countWhere := where
+	pn = (pn - 1) * ps
+	sorts := strings.Split(sort, ",")
+	if len(sorts) == 2 {
+		_, ok1 := sortFields[sorts[0]]
+		d, ok2 := sortOrder[sorts[1]]
+		if ok1 && ok2 {
+			order = " ORDER BY " + sorts[0] + " " + d
+		}
+	}
+	where += order
+	where += fmt.Sprintf(" LIMIT %d, %d", pn, ps)
+	res = &model.Targets{}
+	if res.Targets, err = s.dao.TargetsByQuery(c, where); err != nil {
+		return
+	}
+	if res.Total, err = s.dao.CountTargets(c, countWhere); err != nil {
+		return
+	}
+	res.Page = pn
+	res.PageSize = ps
+	return
+}
+
+// TargetSync sync target state.
+func (s *Service) TargetSync(c context.Context, id int64, state int) (err error) {
+	return s.dao.TargetSync(c, id, state)
+}
+
+// DeleteTarget delete target by id.
+func (s *Service) DeleteTarget(c context.Context, id int64) (err error) {
+	_, err = s.dao.DeleteTarget(c, id)
+	return
+}
+
+func (s *Service) checkTarget(c context.Context, t *model.Target, isNew bool) (err error) {
+	var id int64
+	t.SubEvent = induceSubEvent(t.SubEvent)
+	if id, err = s.dao.IsExisted(c, t); err != nil {
+		return
+	}
+	if isNew && id != 0 {
+		fmt.Println("id", id)
+		err = _duplicateTargetErr
+	}
+	if !isNew && id != 0 && t.ID != id {
+		err = _duplicateTargetErr
+	}
+	return
+}
+
+func mergeTarget(t *model.Target, o *model.Target) {
+	te := reflect.ValueOf(t).Elem()
+	oe := reflect.ValueOf(o).Elem()
+	for i := 0; i < te.NumField()-2; i++ {
+		switch v := te.Field(i).Interface().(type) {
+		case int, int64:
+			if v == 0 {
+				te.Field(i).Set(oe.Field(i))
+			}
+		case string:
+			if v == "" {
+				te.Field(i).Set(oe.Field(i))
+			}
+		}
+	}
+}
+
+// AddProduct add a new group.
+func (s *Service) AddProduct(c context.Context, p *model.Product) (id int64, err error) {
+	if p.Name == "" || p.GroupIDs == "" {
+		err = ecode.RequestErr
+		return
+	}
+	var a *model.Product
+	if a, err = s.dao.ProductByName(c, p.Name); err != nil {
+		return
+	}
+	if a != nil {
+		err = _duplicateProductErr
+		return
+	}
+	id, err = s.dao.AddProduct(c, p)
+	return
+}
+
+// UpdateProduct update product.
+func (s *Service) UpdateProduct(c context.Context, p *model.Product) (err error) {
+	if p.ID == 0 || p.Name == "" || p.GroupIDs == "" {
+		err = ecode.RequestErr
+		return
+	}
+	var a *model.Product
+	if a, err = s.dao.ProductByName(c, p.Name); err != nil {
+		return
+	}
+	if a != nil && a.ID != p.ID {
+		err = _duplicateProductErr
+		return
+	}
+	_, err = s.dao.UpdateProduct(c, p)
+	return
+}
+
+// DeleteProduct delete product.
+func (s *Service) DeleteProduct(c context.Context, id int64) (err error) {
+	_, err = s.dao.DeleteProduct(c, id)
+	return
+}
+
+// AllProducts return all products.
+func (s *Service) AllProducts(c context.Context) (res *model.Products, err error) {
+	res = &model.Products{}
+	if res.Products, err = s.dao.AllProducts(c); err != nil {
+		return
+	}
+	res.Total = len(res.Products)
+	return
+}
+
+// Collect collect.
+func (s *Service) Collect(c context.Context, p *monitor.Log) {
+	var (
+		curr int
+		key  string
+		t    *model.Target
+	)
+	target := &model.Target{
+		Source:   sourceFromLog(p),
+		Product:  p.Product,
+		Event:    p.Event,
+		SubEvent: induceSubEvent(p.SubEvent),
+		State:    0,
+	}
+	s.infoCh <- p
+	//TODO 获取buvid, ip等信息，过滤重复请求
+	if key = targetKey(target); key == "" {
+		return
+	}
+
+	if t = s.targetKeys[key]; t == nil {
+		if s.newTargets[key] == nil {
+			// 添加新的target
+			s.AddTarget(c, target)
+		}
+		s.mapMutex.Lock()
+		s.newTargets[key] = t
+		s.mapMutex.Unlock()
+		return
+	}
+	if t.Threshold == 0 {
+		return
+	}
+	p.CalCode()
+	code := codeFromLog(p)
+	curr = s.dao.TargetIncr(c, t, code)
+	if curr > t.Threshold {
+		go s.mail(c, p, t, curr, code)
+	}
+}
+
+func sourceFromLog(l *monitor.Log) string {
+	if l.Type == "web/h5" {
+		return l.Type
+	}
+	if l.RequestURI == "" {
+		return "app"
+	}
+	if res := strings.Split(l.RequestURI, "?"); len(res) > 1 {
+		return res[1]
+	}
+	return l.RequestURI
+}
+
+func codeFromLog(l *monitor.Log) string {
+	if l.Codes == "" {
+		return "999"
+	}
+	if l.HTTPCode != "" && l.HTTPCode != "200" {
+		return l.HTTPCode
+	}
+	if l.BusinessCode != "" && l.BusinessCode != "0" {
+		return l.BusinessCode
+	}
+	return "-999"
+}

app/interface/openplatform/monitor-end/service/alert_test.go

@@ -0,0 +1,237 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testGroup = &model.Group{
+		Name:      "test",
+		Receivers: "lalalal",
+		Interval:  30,
+	}
+	tmpGroup, tmpGroup2 *model.Group
+)
+
+// TestAddGroup .
+func TestAddGroup(t *testing.T) {
+	c := context.Background()
+	Convey("add group ", t, func() {
+		id, err := svr.AddGroup(c, testGroup)
+		So(err, ShouldBeNil)
+		tmpGroup, err = svr.dao.Group(c, id)
+		So(err, ShouldBeNil)
+		testGroup.Name = "hahaha"
+		id, err = svr.AddGroup(c, testGroup)
+		So(err, ShouldBeNil)
+		tmpGroup2, err = svr.dao.Group(c, id)
+		So(err, ShouldBeNil)
+	})
+	Convey("add group again with duplicate name", t, func() {
+		_, err := svr.AddGroup(c, testGroup)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestUpdateGroup .
+func TestUpdateGroup(t *testing.T) {
+	c := context.Background()
+	Convey("update group ", t, func() {
+		tmpGroup.Name = "huhuhu"
+		err := svr.UpdateGroup(c, tmpGroup)
+		So(err, ShouldBeNil)
+	})
+	Convey("update group again with duplicate name", t, func() {
+		tmpGroup2.Name = "huhuhu"
+		err := svr.UpdateGroup(c, tmpGroup2)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestGroupList .
+func TestGroupList(t *testing.T) {
+	c := context.Background()
+	Convey("group list", t, func() {
+		_, err := svr.GroupList(c, nil)
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestDeleteGroup .
+func TestDeleteGroup(t *testing.T) {
+	c := context.Background()
+	Convey("delete group", t, func() {
+		err := svr.DeleteGroup(c, tmpGroup.ID)
+		So(err, ShouldBeNil)
+		err = svr.DeleteGroup(c, tmpGroup2.ID)
+		So(err, ShouldBeNil)
+	})
+}
+
+var (
+	testTarget = &model.Target{
+		SubEvent:  "mall.bilibili.com/home",
+		Event:     "test",
+		Product:   "test",
+		Source:    "test",
+		GroupIDs:  "1,2,3",
+		Threshold: 4,
+		Duration:  5,
+	}
+	tmpTarget, tmpTarget2 *model.Target
+)
+
+// TestAddTarget .
+func TestAddTarget(t *testing.T) {
+	c := context.Background()
+	Convey("add target ", t, func() {
+		id, err := svr.AddTarget(c, testTarget)
+		So(err, ShouldBeNil)
+		tmpTarget, err = svr.dao.Target(c, id)
+		So(err, ShouldBeNil)
+		testTarget.Product = "tttt"
+		id, err = svr.AddTarget(c, testTarget)
+		So(err, ShouldBeNil)
+		tmpTarget2, err = svr.dao.Target(c, id)
+		So(err, ShouldBeNil)
+	})
+	Convey("add Target again with duplicate name", t, func() {
+		_, err := svr.AddTarget(c, testTarget)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestUpdateTarget .
+func TestUpdateTarget(t *testing.T) {
+	c := context.Background()
+	Convey("update Target ", t, func() {
+		tmpTarget.Product = "xxxx"
+		err := svr.UpdateTarget(c, tmpTarget)
+		So(err, ShouldBeNil)
+	})
+	Convey("update Target again with duplicate name", t, func() {
+		tmpTarget2.Product = "xxxx"
+		err := svr.UpdateTarget(c, tmpTarget2)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestTargetList .
+func TestTargetList(t *testing.T) {
+	c := context.Background()
+	Convey("Target list", t, func() {
+		t := &model.Target{}
+		_, err := svr.TargetList(c, t, 1, 2, "mtime,0")
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestTargetSync .
+func TestTargetSync(t *testing.T) {
+	c := context.Background()
+	Convey("sync Target", t, func() {
+		err := svr.TargetSync(c, tmpTarget.ID, 1)
+		So(err, ShouldBeNil)
+		err = svr.TargetSync(c, tmpTarget2.ID, 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestDeleteTarget .
+func TestDeleteTarget(t *testing.T) {
+	c := context.Background()
+	Convey("delete Target", t, func() {
+		err := svr.DeleteTarget(c, tmpTarget.ID)
+		So(err, ShouldBeNil)
+		err = svr.DeleteTarget(c, tmpTarget2.ID)
+		So(err, ShouldBeNil)
+	})
+}
+
+var (
+	testProduct = &model.Product{
+		Name:     "test",
+		GroupIDs: "1,2",
+		State:    1,
+	}
+	tmpProduct, tmpProduct2 *model.Product
+)
+
+// TestAddProduct .
+func TestAddProduct(t *testing.T) {
+	c := context.Background()
+	Convey("add product ", t, func() {
+		id, err := svr.AddProduct(c, testProduct)
+		So(err, ShouldBeNil)
+		tmpProduct, err = svr.dao.Product(c, id)
+		So(err, ShouldBeNil)
+		testProduct.Name = "hahaha"
+		id, err = svr.AddProduct(c, testProduct)
+		So(err, ShouldBeNil)
+		tmpProduct2, err = svr.dao.Product(c, id)
+		So(err, ShouldBeNil)
+	})
+	Convey("add product again with duplicate name", t, func() {
+		_, err := svr.AddProduct(c, testProduct)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestUpdateProduct .
+func TestUpdateProduct(t *testing.T) {
+	c := context.Background()
+	Convey("update product ", t, func() {
+		tmpProduct.Name = "huhuhu"
+		err := svr.UpdateProduct(c, tmpProduct)
+		So(err, ShouldBeNil)
+	})
+	Convey("update Product again with duplicate name", t, func() {
+		tmpProduct2.Name = "huhuhu"
+		err := svr.UpdateProduct(c, tmpProduct2)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestAllProducts .
+func TestAllProducts(t *testing.T) {
+	c := context.Background()
+	Convey("all products", t, func() {
+		_, err := svr.AllProducts(c)
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestDeleteProduct .
+func TestDeleteProduct(t *testing.T) {
+	c := context.Background()
+	Convey("delete product", t, func() {
+		err := svr.DeleteProduct(c, tmpProduct.ID)
+		So(err, ShouldBeNil)
+		err = svr.DeleteProduct(c, tmpProduct2.ID)
+		So(err, ShouldBeNil)
+	})
+}
+
+var (
+	testCollect = &monitor.Log{
+		SubEvent: "aaa",
+		Event:    "bbb",
+		Product:  "bbb",
+		ExtJSON:  "xxx",
+		HTTPCode: "404",
+	}
+)
+
+// TestCollect .
+func TestCollect(t *testing.T) {
+	c := context.Background()
+	Convey("test collect", t, func() {
+		svr.Collect(c, testCollect)
+	})
+}

app/interface/openplatform/monitor-end/service/consumer.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/model/kafka"
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	cluster "github.com/bsm/sarama-cluster"
+)
+
+const _group = "open-monitor-end"
+
+// Consumer .
+type Consumer struct {
+	// c      sarama.Consumer
+	c        *cluster.Consumer
+	Config   *kafka.Config
+	closed   bool
+	paused   bool
+	duration time.Duration
+	messages chan *sarama.ConsumerMessage
+}
+
+var (
+	errClosedMsgChannel    = errors.New("message channel is closed")
+	errClosedNotifyChannel = errors.New("notification channel is closed")
+	errConsumerOver        = errors.New("too many consumers")
+)
+
+// NewConsumer .
+func NewConsumer(c *conf.Config) (con *Consumer, err error) {
+	con = &Consumer{
+		Config:   c.Kafka,
+		messages: make(chan *sarama.ConsumerMessage, 1024),
+	}
+	cfg := cluster.NewConfig()
+	cfg.Version = sarama.V0_8_2_0
+	cfg.ClientID = fmt.Sprintf("%s-%s", _group, c.Kafka.Topic)
+	cfg.Net.KeepAlive = 30 * time.Second
+	// NOTE cluster auto commit offset interval
+	cfg.Consumer.Offsets.CommitInterval = time.Second * 1
+	// NOTE set fetch.wait.max.ms
+	cfg.Consumer.MaxWaitTime = time.Millisecond * 250
+	cfg.Consumer.MaxProcessingTime = 50 * time.Millisecond
+	// NOTE errors that occur during offset management,if enabled, c.Errors channel must be read
+	cfg.Consumer.Return.Errors = true
+	// NOTE notifications that occur during consumer, if enabled, c.Notifications channel must be read
+	cfg.Group.Return.Notifications = true
+	// con.c = sarama.NewConsumer(c.Kafka.Addr, nil)
+	// consumer.Partitions = consumer.c.Partitions(consumer.Config.Topic)
+	cfg.Consumer.Offsets.Initial = sarama.OffsetNewest
+	if con.c, err = cluster.NewConsumer(c.Kafka.Addr, _group, []string{c.Kafka.Topic}, cfg); err != nil {
+		log.Error("s.NewConsumer group(%s) topic(%s) addr(%s) cluster.NewConsumer() error(%v)", _group, c.Kafka.Topic, strings.Join(c.Kafka.Addr, ","), err)
+	} else {
+		log.Info("s.NewConsumer group(%s) topic(%s) addr(%s) cluster.NewConsumer() ok", _group, c.Kafka.Topic, strings.Join(c.Kafka.Addr, ","))
+	}
+	return
+}
+
+func (s *Service) consume() {
+	for {
+		if s.consumer.closed {
+			return
+		}
+		if err := s.consumer.message(); err != nil {
+			time.Sleep(time.Minute)
+		}
+	}
+}
+
+func (s *Service) handleMsg() {
+	for {
+		select {
+		case msg := <-s.consumer.messages:
+			s.HandleMsg(msg.Value)
+		}
+		if s.consumer.closed {
+			return
+		}
+	}
+}
+
+func (s *Consumer) message() (err error) {
+	var (
+		msg    *sarama.ConsumerMessage
+		notify *cluster.Notification
+		ok     bool
+	)
+	for {
+		if s.closed {
+			s.c.Close()
+			err = nil
+			return
+		}
+		if s.paused {
+			s.paused = false
+			time.Sleep(s.duration)
+		}
+		select {
+		case err = <-s.c.Errors():
+			log.Error("group(%s) topic(%s) addr(%s) catch error(%v)", _group, s.Config.Topic, s.Config.Addr, err)
+			return
+		case notify, ok = <-s.c.Notifications():
+			if !ok {
+				log.Info("notification notOk group(%s) topic(%s) addr(%v) catch error(%v)", _group, s.Config.Topic, s.Config.Addr, err)
+				err = errClosedNotifyChannel
+				return
+			}
+			switch notify.Type {
+			case cluster.UnknownNotification, cluster.RebalanceError:
+				log.Error("notification(%s) group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+				err = errClosedNotifyChannel
+				return
+			case cluster.RebalanceStart:
+				log.Info("notification(%s) group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+				continue
+			case cluster.RebalanceOK:
+				log.Info("notification(%s) group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+			}
+			if len(notify.Current[s.Config.Topic]) == 0 {
+				log.Warn("notification(%s) no topic group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+				err = errConsumerOver
+				return
+			}
+		case msg, ok = <-s.c.Messages():
+			if !ok {
+				log.Error("group(%s) topic(%s) addr(%v) message channel closed", _group, s.Config.Topic, s.Config.Addr)
+				err = errClosedMsgChannel
+				return
+			}
+			s.messages <- msg
+		}
+	}
+}

app/interface/openplatform/monitor-end/service/consumer_test.go

@@ -0,0 +1,39 @@
+package service
+
+import (
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestStartConsume .
+func TestStartConsume(t *testing.T) {
+	Convey("start consume", t, func() {
+		err := svr.StartConsume()
+		So(err, ShouldNotBeNil)
+	})
+}
+
+func TestStartHandle(t *testing.T) {
+	go svr.handleMsg()
+}
+
+// TestHandle .
+func TestHandle(t *testing.T) {
+	Convey("handle msg", t, func() {
+		var l = `a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z|1|2|3|4|5|6|7|8`
+		msg := &sarama.ConsumerMessage{
+			Value: []byte(l),
+		}
+		svr.consumer.messages <- msg
+		time.Sleep(time.Second)
+		So(len(svr.consumer.messages), ShouldEqual, 0)
+	})
+}
+
+// TestClose .
+func TestClose(t *testing.T) {
+	svr.Close()
+}

app/interface/openplatform/monitor-end/service/infoc.go

@@ -0,0 +1,25 @@
+package service
+
+import (
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+)
+
+// writeInfoc
+func (s *Service) infocproc() {
+	var (
+		collectInfoc = infoc.New(s.c.CollectInfoc)
+	)
+	for {
+		i, ok := <-s.infoCh
+		if !ok {
+			log.Warn("infoc proc exit")
+			return
+		}
+		switch l := i.(type) {
+		case model.CollectParams:
+			collectInfoc.Info(l.Source, l.Product, l.Event, l.SubEvent, l.Code, l.ExtJSON, l.Mid, l.IP, l.Buvid, l.UserAgent)
+		}
+	}
+}

app/interface/openplatform/monitor-end/service/mail.go

@@ -0,0 +1,87 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/mail"
+	"net/smtp"
+	"strings"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/log"
+
+	"github.com/scorredoira/email"
+)
+
+var text = `消息时间:%s
+最近%d秒内，%s端%s服务%s的%s出现异常（%s)，异常数量超过告警阀值 %d
+`
+
+func (s *Service) mail(c context.Context, p *monitor.Log, t *model.Target, curr int, code string) {
+	if t == nil {
+		return
+	}
+	var groups = t.Groups
+	if groups == nil || len(t.Groups) == 0 {
+		product := s.productKeys[productKey(t.Product)]
+		if product == nil || len(product.Groups) == 0 {
+			return
+		}
+		groups = product.Groups
+	}
+	for _, g := range groups {
+		if g.Name == "" || g.Interval == 0 {
+			continue
+		}
+		if ok, err := s.dao.GetMailLock(c, g.Name, g.Interval, t, code); err != nil || !ok {
+			continue
+		}
+		go s.mailByGroup(c, g.Receivers, p, curr, t.Threshold, t.Duration, code)
+	}
+	return
+}
+
+func (s *Service) mailByGroup(c context.Context, receivers string, p *monitor.Log, curr int, threshold int, duration int, code string) {
+	tos := strings.Split(receivers, ",")
+	if len(tos) == 0 {
+		return
+	}
+	for i, t := range tos {
+		tos[i] = t + "@bilibili.com"
+	}
+	source := sourceFromLog(p)
+	now := time.Now().Format("2006-01-02 15:03:04")
+	title := fmt.Sprintf("【端监控告警】%s端%s出现异常", source, p.Product)
+	body := fmt.Sprintf(text, now, duration, source, p.Product, p.Event, p.SubEvent, code, threshold)
+	if err := send(tos, title, body, 0); err != nil {
+		log.Error("s.mailByGroup.send error(%+v), mail to(%s), title(%s), body(%s)", err, receivers, title, body)
+	} else {
+		log.Info("s.mailByGroup.send successed, mail to(%s), title(%s), body(%s)", receivers, title, body)
+	}
+}
+
+/*
+    * const HOST = 'smtp.exmail.qq.com';
+    * const USER = 'show@bilibili.com';
+    * const PASS = 'Kfpt2017';
+	* const NAME = 'bilibili演出票务';
+*/
+func send(to []string, title string, body string, mode int) error {
+	var (
+		m    *email.Message
+		host = "smtp.exmail.qq.com:25"
+	)
+	if mode == 0 {
+		m = email.NewMessage(title, body)
+	} else {
+		m = email.NewHTMLMessage(title, body)
+	}
+	m.From = mail.Address{
+		Name:    "kfc监控告警",
+		Address: "show@bilibili.com",
+	}
+	m.To = to
+	return email.Send(host, smtp.PlainAuth("", "show@bilibili.com", "Kfpt2017", "smtp.exmail.qq.com"), m)
+}

app/interface/openplatform/monitor-end/service/monitor.go

@@ -0,0 +1,431 @@
+package service
+
+import (
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/app/interface/openplatform/monitor-end/model/prom"
+	"go-common/library/log"
+
+	"github.com/json-iterator/go"
+)
+
+const _regex = `.*\d{2,}`
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+const (
+	_typeAPP       = "app"
+	_typeWeb       = "web/h5"
+	_typeAPPH      = "app_h"
+	_eventPage     = "page"
+	_eventAPI      = "api"
+	_eventResource = "resource"
+	_levelInfo     = "info"
+	_levelWarning  = "warning"
+	_levelError    = "error"
+	_logTypeOne    = "1"
+	_logTypeTwo    = "2"
+)
+
+// Report .
+func (s *Service) Report(c context.Context, params *model.LogParams, mid int64, ip string, buvid string, userAgent string) (err error) {
+	var l *monitor.Log
+	log.Info("get log report(%+v)", params)
+	if params.IsAPP == 1 {
+		_, err = s.nativeLog(c, params.Log)
+	} else {
+		l, err = s.frontendLog(c, params.Source, params.Log, mid, ip, buvid, userAgent)
+		go s.promsFE(context.TODO(), l)
+	}
+	return
+}
+func induceSubEvent(s string) string {
+	if strings.Contains(s, "?") {
+		s = s[:strings.Index(s, "?")]
+	}
+	if strings.Contains(s, "://") {
+		s = s[(strings.Index(s, "://") + 3):]
+	}
+	if len(s) > 512 {
+		s = s[:512]
+	}
+	var (
+		ok  bool
+		err error
+	)
+	for {
+		if ok, err = regexp.Match(_regex, []byte(s)); err != nil {
+			log.Error("s.Report.regexp error(%+v), data(%s)", err, s)
+			return s
+		}
+		if !ok {
+			break
+		}
+		if !strings.Contains(s, "/") {
+			return s
+		}
+		s = s[:strings.LastIndex(s, "/")]
+	}
+	return s
+}
+
+func (s *Service) promsFE(c context.Context, l *monitor.Log) {
+	if !s.c.Prom.Promed || l == nil {
+		return
+	}
+	var (
+		ok  bool
+		err error
+	)
+	l.SubEvent = induceSubEvent(l.SubEvent)
+	if l.Event == _eventResource {
+		regex := `.*i\d{1}.hdslb.com`
+		if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
+			log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
+			return
+		}
+		if ok {
+			l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
+		}
+		prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
+		return
+	}
+	prom.AddHTTPCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
+	prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
+	if l.Details == nil {
+		var cost int64
+		if l.Duration == "" {
+			return
+		}
+		if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
+			log.Warn("s.Info.ParseInt can not convert duration(%s) to int64", l.Duration)
+			return
+		}
+		prom.AddCommonLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
+	} else {
+		prom.AddDetailedLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.Details)
+	}
+}
+
+func (s *Service) nativeLog(c context.Context, data string) (l *monitor.Log, err error) {
+	l = &monitor.Log{}
+	if err = json.Unmarshal([]byte(data), l); err != nil {
+		log.Error("s.nativeLog.unmarshal error(%+v), data(%s)", err, data)
+		return
+	}
+	l.Type = _typeAPPH
+	s.handleLog(c, l)
+	return
+}
+
+// HandleMsg .
+func (s *Service) HandleMsg(msg []byte) {
+	l := monitor.LogFromBytes(msg)
+	l.Type = _typeAPP
+	s.handleLog(context.TODO(), l)
+}
+
+func (s *Service) handleLog(c context.Context, l *monitor.Log) {
+	var (
+		appID   string
+		kv      []log.D
+		err     error
+		logtype int
+		isInt   bool
+		cost    int64
+		ok      bool
+		app     = "android"
+	)
+	l.TraceidSvr = l.Traceid
+	l.Traceid = ""
+	l.CalCode()
+	if appID, _, kv, err = l.LogData(); err != nil {
+		log.Error("s.nativeLog.LogData error(%+v), data(%+v)", err, l)
+		return
+	}
+	if l.Result == "1" || l.Result == "" {
+		s.mh.Info(c, appID, kv...)
+	} else {
+		s.mh.Error(c, appID, kv...)
+	}
+	if l.SubProduct == "" {
+		l.SubProduct = l.Product
+	}
+	if strings.Contains(l.RequestURI, "ios") {
+		app = "ios"
+		isInt = true
+	}
+	if logtype, err = bStrToInt(l.LogType, isInt); err != nil {
+		return
+	}
+	if s.c.Prom.IgnoreNA && !s.checkProduct(l) {
+		return
+	}
+	//	丢弃老版本的network日志
+	if l.Event == "network" && l.Codes == "" {
+		return
+	}
+	l.SubEvent = induceSubEvent(l.SubEvent)
+	regex := `.*i\d{1}.hdslb.com`
+	if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
+		log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
+		return
+	}
+	if ok {
+		l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
+	}
+	if (logtype & 1) == 1 {
+		// 性能日志
+		if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
+			log.Warn("s.handleLog.ParseInt can not convert duration(%s) to int64", l.Duration)
+			return
+		}
+		prom.AddCommonLog(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
+	}
+	if (logtype >> 2 & 1) == 1 {
+		// 成功/失败日志
+		if l.Event == "network" {
+			// 网络类型 上报业务码和http状态
+			prom.AddHTTPCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
+			prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
+		} else {
+			// 其他类型 只上报失败成功 result=1 表示成功
+			res := "999"
+			if l.Result == "1" {
+				res = "0"
+			}
+			prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, res)
+		}
+	}
+}
+
+func (s *Service) frontendLog(c context.Context, source string, data string, mid int64, ip string, buvid string, userAgent string) (l *monitor.Log, err error) {
+	var (
+		ms                []interface{}
+		kv                []log.D
+		logtype, loglevel string
+		url, query        string
+	)
+	if err = json.Unmarshal([]byte(data), &ms); err != nil {
+		log.Error("s.frontendLog.unmarshal error(%+v), data(%s)", err, data)
+		return
+	}
+	for _, m := range ms {
+		switch m := m.(type) {
+		case map[string]interface{}:
+			logtype = stringValueByKey(m, "1", "logtype")
+			loglevel = stringValueByKey(m, "info", "level")
+			tmp := stringValueByKey(m, "", "url")
+			if strings.Contains(tmp, "?") {
+				url = strings.Split(tmp, "?")[0]
+				query = strings.Split(tmp, "?")[1]
+			} else {
+				url = tmp
+			}
+			switch logtype {
+			case _logTypeOne:
+				l = &monitor.Log{
+					LogType:      _logTypeOne,
+					Type:         _typeWeb,
+					Product:      source,
+					IP:           ip,
+					Buvid:        buvid,
+					UserAgent:    userAgent,
+					Event:        _eventAPI,
+					Mid:          strconv.FormatInt(mid, 10),
+					SubEvent:     url,
+					Query:        query,
+					Duration:     stringValueByKey(m, "", "cost"),
+					TraceidSvr:   stringValueByKey(m, "", "traceid_svr"),
+					TraceidEnd:   stringValueByKey(m, "", "traceid_end"),
+					HTTPCode:     stringValueByKey(m, "200", "status"),
+					BusinessCode: stringValueByKey(m, "0", "errno", "code"),
+					SubProduct:   stringValueByKey(m, source, "sub_product"),
+					Result:       "0",
+				}
+				srcURL := stringValueByKey(m, "", "srcUrl")
+				if srcURL != "" {
+					l.Event = _eventResource
+					l.SubEvent = srcURL
+					l.BusinessCode = "-999"
+				}
+				if loglevel == _levelInfo {
+					l.Result = "1"
+				}
+				if _, _, kv, err = l.LogData(); err != nil {
+					return
+				}
+				kv = append(kv, extKV(m)...)
+			case _logTypeTwo:
+				var msg []byte
+				l = &monitor.Log{
+					LogType:      _logTypeTwo,
+					Type:         _typeWeb,
+					Product:      source,
+					Event:        _eventPage,
+					IP:           ip,
+					Buvid:        buvid,
+					UserAgent:    userAgent,
+					Mid:          strconv.FormatInt(mid, 10),
+					SubEvent:     url,
+					Query:        query,
+					HTTPCode:     "200",
+					Details:      details(m),
+					Result:       "1",
+					BusinessCode: "0",
+					SubProduct:   stringValueByKey(m, source, "sub_product"),
+				}
+				if msg, err = json.Marshal(m); err != nil {
+					log.Error("s.frontendLog.Marshal error(%v), data(%v)", err, m)
+					continue
+				}
+				l.Message = string(msg)
+				if _, _, kv, err = l.LogData(); err != nil {
+					return
+				}
+			default:
+				log.Error("s.frontendLog error logype(%v)", logtype)
+				return
+			}
+			switch loglevel {
+			case _levelInfo:
+				s.mh.Info(c, source, kv...)
+			case _levelWarning:
+				s.mh.Warn(c, source, kv...)
+			case _levelError:
+				s.collectFE(c, l)
+				s.mh.Error(c, source, kv...)
+			default:
+				s.mh.Info(c, source, kv...)
+			}
+		default:
+			log.Error("s.frontendLog log data is not json type: " + data)
+		}
+	}
+	return
+}
+
+func (s *Service) collectFE(c context.Context, l *monitor.Log) {
+	if s.c.CollectFE {
+		regex := `.*i\d{1}.hdslb.com`
+		if ok, err := regexp.Match(regex, []byte(l.SubEvent)); err != nil {
+			log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
+			return
+		} else if ok {
+			l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
+		}
+		go s.Collect(context.TODO(), l)
+	}
+}
+
+func stringValueByKey(m map[string]interface{}, defValue string, keys ...string) (r string) {
+	r = defValue
+	if m == nil {
+		return
+	}
+	var (
+		res interface{}
+		ok  bool
+	)
+	for _, key := range keys {
+		if res, ok = m[key]; ok {
+			break
+		}
+		return
+	}
+	if res == nil {
+		return
+	}
+	switch d := res.(type) {
+	case int:
+		r = strconv.Itoa(d)
+	case float64:
+		r = strconv.FormatFloat(d, 'f', -1, 64)
+	case int64:
+		r = strconv.FormatInt(d, 10)
+	case string:
+		r = d
+	default:
+		log.Warn("s.stringValueByKey unexcept type of value(%v)", d)
+	}
+	return
+}
+
+func extKV(m map[string]interface{}) (kv []log.D) {
+	if m == nil {
+		return
+	}
+	var keys = []string{"level", "logtype", "url", "cost", "traceid_svr", "traceid_end", "status", "code", "errno", "sub_product"}
+	for _, key := range keys {
+		delete(m, key)
+	}
+	for k, v := range m {
+		kv = append(kv, log.KV(k, v))
+	}
+	return
+}
+
+func details(m map[string]interface{}) (res map[string]int64) {
+	res = make(map[string]int64)
+	if m == nil {
+		return
+	}
+	delete(m, "level")
+	delete(m, "logtype")
+	delete(m, "url")
+	for k, v := range m {
+		var d int64
+		if v == nil {
+			res[k] = 0
+			continue
+		}
+		switch v := v.(type) {
+		case int64:
+			d = v
+		case float64:
+			d = int64(v)
+		case int:
+			d = int64(v)
+		case int32:
+			d = int64(v)
+		case string:
+			d, _ = strconv.ParseInt(v, 10, 64)
+		default:
+			log.Warn("s.details unexcept type of value(%v)", v)
+		}
+		res[k] = d
+	}
+	return
+}
+
+func bStrToInt(str string, isInt bool) (r int, err error) {
+	if isInt {
+		if r, err = strconv.Atoi(str); err != nil {
+			log.Warn("s.bStrToInt error(%+v), string(%s)", err, str)
+		}
+		return
+	}
+	for _, i := range str {
+		if i == '1' {
+			r = r*2 + 1
+		} else {
+			r *= 2
+		}
+	}
+	return
+}
+
+func (s *Service) checkProduct(l *monitor.Log) bool {
+	if s.naProducts == nil {
+		return true
+	}
+	if s.naProducts[l.Product] {
+		return true
+	}
+	return false
+}

app/interface/openplatform/monitor-end/service/monitor_test.go

@@ -0,0 +1,100 @@
+package service
+
+import (
+	"testing"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestFELog .
+func TestFELog(t *testing.T) {
+	var (
+		tcs = []TestCase{
+			TestCase{
+				tag:      "logtype1 normal",
+				testData: `[{"level":"info","logtype":1,"url":"https://show.bilibili.com/api/ticket/district/geocoder","status":200,"cost":323,"traceid_end":"2565683971","traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype1 empty data",
+				testData: "",
+				expected: 1,
+			},
+			TestCase{
+				tag:      "logtype1 valide data2",
+				testData: `[{"level":"info","logtype":1,"url":"https://show.bilibili.com/api/ticket/district/geocoder","status":"200","cost":"323","traceid_end":2565683971,"traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype1 valid data2",
+				testData: `[{"level":"info","logtype":1,"url":111,"status":"200","cost":"323","traceid_end":2565683971,"traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype1 valid data3",
+				testData: `[{"level":"error","logtype":1,"url":111,"status":"200","cost":"323","traceid_end":2565683971,"traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype2 data",
+				testData: `[{"level":"info","logtype":2,"url":"https://show.bilibili.com/platform/home.html","navigationStart":0,"redirectStart":0,"redirectEnd":0,"fetchStart":2,"domainLookupStart":2,"domainLookupEnd":2,"connectStart":2,"secureConnectionStart":0,"connectEnd":2,"requestStart":71,"responseStart":73,"responseEnd":89,"domLoading":107,"domInteractive":442,"domContentLoadedEventStart":442,"domContentLoadedEventEnd":471,"domComplete":873,"loadEventStart":873,"loadEventEnd":873,"firstPaint":301,"firstContentfulPaint":515}]`,
+				expected: 0,
+			},
+		}
+	)
+	for _, tc := range tcs {
+		Convey(tc.tag, t, func() {
+			err := svr.Report(ctx, &model.LogParams{Source: "aaa", Log: tc.testData}, 111, "1.1.1.1", "", "")
+			if tc.expected == 0 {
+				So(err, ShouldBeNil)
+			} else {
+				So(err, ShouldNotBeNil)
+			}
+		})
+	}
+}
+
+// TestAPPLog .
+func TestAPPLog(t *testing.T) {
+	var (
+		tcs = []TestCase{
+			TestCase{
+				tag:      "logtype1 normal",
+				testData: `{"request_uri":"\/log\/mobile?ios","time_iso":"1539331278223","ip":"163.142.141.67","version":"2","buvid":"6baad8d5278a2982d204cafd403f089e","fts":"1525072552","proid":"1","chid":"AppStore","pid":"11","brand":"Apple","deviceid":"6baad8d5278a2982d204cafd403f089e","model":"iPhone 7","osver":"11.4.1","ctime":"20181012160107","mid":"18260473","ver":"5.32(8170)","net":"1","oid":"","product":"payment","createtime":"20181012-16:01:07.029GMT+08:00","event":"payment_iap","sub_event":"fetch_products","log_type":"16","duration":"0","message":"\u4eceApple\u83b7\u53d6\u6240\u6709products","result":"0","ext_json":"{\"productIds\":\"tv.danmaku.bilianimex68Bcoin,tv.danmaku.bilianimexnewpanel4998Bcoin,tv.danmaku.bilianimex3BigBcoin,tv.danmaku.bilianimex3VIPbf1,tv.danmaku.bilianimexnewpanel158Bcoin,tv.danmaku.bilianimexnewpanel1598Bcoin,tv.danmaku.bilianimex998Bcoin,tv.danmaku.bilianimexnewpanel648Bcoin,tv.danmaku.bilianimexnewpanel68Bcoin,tv.danmaku.bilianimex12VIPbf1,tv.danmaku.bilianimex3VIP,tv.danmaku.bilianimex18Bcoin,tv.danmaku.bilianimex12VIP,tv.danmaku.bilianimexnewpanel388Bcoinbf","traceid":"","desc":"","network":"1"}`,
+				expected: 0,
+			},
+			TestCase{
+				tag: "logtype1 normal",
+				testData: `	{"request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"1","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 0,
+			},
+			TestCase{
+				tag: "logtype1 error duration",
+				testData: `	{"request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277a","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"1","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 0,
+			},
+			TestCase{
+				tag: "logtype1 no json",
+				testData: `	{request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"1","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 1,
+			},
+			TestCase{
+				tag: "logtype1 normal",
+				testData: `	{"request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"0","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 0,
+			},
+		}
+	)
+	for _, tc := range tcs {
+		Convey(tc.tag, t, func() {
+			err := svr.Report(ctx, &model.LogParams{Source: "test", Log: tc.testData, IsAPP: 1}, 111, "1.1.1.1", "", "")
+			if tc.expected == 0 {
+				So(err, ShouldBeNil)
+			} else {
+				So(err, ShouldNotBeNil)
+			}
+		})
+	}
+}

app/interface/openplatform/monitor-end/service/service.go

@@ -0,0 +1,154 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/dao"
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/app/interface/openplatform/monitor-end/model/prom"
+	"go-common/library/log"
+)
+
+var (
+	_notConsumedErr = errors.New("未启动消费")
+	_pausedNowErr   = errors.New("暂停消费中")
+	_closedNowErr   = errors.New("已停止消费")
+	_consumedNowErr = errors.New("已启动消费")
+)
+
+// Service struct
+type Service struct {
+	c        *conf.Config
+	dao      *dao.Dao
+	mh       *monitor.MonitorHandler
+	consumer *Consumer
+	consumed bool
+	// settings
+	groups      map[int64]*model.Group
+	targets     map[int64]*model.Target
+	targetKeys  map[string]*model.Target
+	products    map[int64]*model.Product
+	productKeys map[string]*model.Product
+	newTargets  map[string]*model.Target
+	naProducts  map[string]bool
+	mapMutex    sync.Mutex
+	// infoc
+	infoCh chan interface{}
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:           c,
+		dao:         dao.New(c),
+		mh:          monitor.NewMonitor(c.Monitor),
+		groups:      make(map[int64]*model.Group),
+		targets:     make(map[int64]*model.Target),
+		targetKeys:  make(map[string]*model.Target),
+		newTargets:  make(map[string]*model.Target),
+		products:    make(map[int64]*model.Product),
+		productKeys: make(map[string]*model.Product),
+		naProducts:  make(map[string]bool),
+		infoCh:      make(chan interface{}, 1024),
+	}
+	for c.NeedConsume && !s.consumed {
+		var err error
+		if s.consumer, err = NewConsumer(c); err != nil {
+			log.Error("s.New.NewConsumer error(%+v)", err)
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		go s.consume()
+		go s.handleMsg()
+		s.consumed = true
+		break
+	}
+	prom.Init(c)
+	go s.loadalertsettingproc()
+	go s.infocproc()
+	go s.loadNAProducts()
+	// go s.alertproc()
+	return
+}
+
+// Ping .
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// StartConsume .
+func (s *Service) StartConsume() (err error) {
+	if s.consumed {
+		return _consumedNowErr
+	}
+	if s.consumer, err = NewConsumer(s.c); err != nil {
+		log.Error("s.New.NewConsumer error(%+v)", err)
+		return
+	}
+	go s.consume()
+	go s.handleMsg()
+	s.consumed = true
+	return
+}
+
+// Close .
+func (s *Service) Close() {
+	s.dao.Close()
+	s.StopConsume()
+}
+
+// StopConsume .
+func (s *Service) StopConsume() error {
+	if s.consumed {
+		s.consumed = false
+	}
+	if s.consumer != nil {
+		s.consumer.closed = true
+	}
+	return nil
+}
+
+// PauseConsume .
+func (s *Service) PauseConsume(t int64) error {
+	if !s.consumed {
+		return _notConsumedErr
+	}
+	if s.consumer.closed {
+		return _closedNowErr
+	}
+	if s.consumer.paused {
+		return _pausedNowErr
+	}
+	s.consumer.paused = true
+	s.consumer.duration = time.Second * time.Duration(t)
+	return nil
+}
+
+func (s *Service) loadalertsettingproc() {
+	for {
+		s.loadalertsettings()
+		time.Sleep(time.Minute)
+	}
+}
+
+func (s *Service) loadNAProducts() {
+	for {
+		if s.c.Products != "" {
+			var m = make(map[string]bool)
+			ps := strings.Split(s.c.Products, ",")
+			for _, p := range ps {
+				m[p] = true
+			}
+			s.mapMutex.Lock()
+			s.naProducts = m
+			s.mapMutex.Unlock()
+		}
+		time.Sleep(time.Minute)
+	}
+}

app/interface/openplatform/monitor-end/service/service_test.go

@@ -0,0 +1,85 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/model/kafka"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/cache/redis"
+	"go-common/library/container/pool"
+	"go-common/library/database/sql"
+	"go-common/library/log/infoc"
+	"go-common/library/time"
+)
+
+var (
+	ctx = context.Background()
+	svr *Service
+)
+
+type TestData map[string]string
+
+type TestCase struct {
+	tag      string
+	testData string
+	expected int
+}
+
+/*[mysql]
+	dsn = "root:123456@tcp(172.16.33.203:3306)/public_monitor?timeout=500s&readTimeout=500s&writeTimeout=500s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "1000s"
+    execTimeout = "200s"#
+    tranTimeout = "2000s"
+
+[redis]
+    name = "article"
+    proto = "tcp"
+    addr = "172.16.33.203:6379"
+    idle = 10
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"*/
+func init() {
+	c := conf.Conf
+	if c.Monitor == nil {
+		c = &conf.Config{
+			Monitor: &monitor.MonitorConfig{Proto: "tcp", Addr: "127.0.0.1:9988"},
+			Kafka: &kafka.Config{
+				Addr:  []string{"1.1.1.1"},
+				Topic: "test_topic",
+			},
+			NeedConsume: false,
+			Redis: &redis.Config{
+				Name:  "article",
+				Proto: "tcp",
+				Addr:  "172.16.33.203:6379",
+				Config: &pool.Config{
+					Idle:   2,
+					Active: 5,
+				},
+				DialTimeout:  time.Duration(int64(1000000000)),
+				ReadTimeout:  time.Duration(int64(1000000000)),
+				WriteTimeout: time.Duration(int64(1000000000)),
+			},
+			MySQL: &sql.Config{
+				DSN:          "root:123456@tcp(172.16.33.203:3306)/public_monitor?timeout=500s&readTimeout=500s&writeTimeout=500s&parseTime=true&loc=Local&charset=utf8,utf8mb4",
+				QueryTimeout: time.Duration(int64(10000000000)),
+				ExecTimeout:  time.Duration(int64(10000000000)),
+				TranTimeout:  time.Duration(int64(20000000000)),
+			},
+			Prom:         &conf.Prom{Limit: 520},
+			CollectInfoc: &infoc.Config{},
+		}
+	}
+	svr = New(c)
+
+	if err := svr.dao.Ping(context.Background()); err != nil {
+		panic(err)
+	}
+}