Create & Init Project...

app/interface/openplatform/monitor-end/model/monitor/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "handler.go",
+        "monitor.go",
+    ],
+    importpath = "go-common/app/interface/openplatform/monitor-end/model/monitor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "handler_test.go",
+        "monitor_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/interface/openplatform/monitor-end/model/monitor/handler.go

@@ -0,0 +1,246 @@
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+	xtime "go-common/library/time"
+
+	"github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+const (
+	// app_id
+	_appID = "app_id"
+	// time format
+	_timeFormat = "2006-01-02T15:04:05.999999"
+	//  log level name: INFO, WARN...
+	_level = "level"
+	// log time.
+	_time = "time"
+	// uniq ID from trace.
+	_tid = "traceid"
+	// default chan size
+	_defaultChan = 2048
+	// default agent timeout
+	_agentTimeout = xtime.Duration(20 * time.Millisecond)
+	// default merge wait time
+	_mergeWait = 1 * time.Second
+	// max buffer size
+	_maxBuffer = 10 * 1024 * 1024 // 10mb
+)
+
+var (
+	_defaultMonitorConfig = &MonitorConfig{
+		Proto: "unixgram",
+		Addr:  "/var/run/lancer/collector.sock",
+	}
+	_defaultTaskIDs = map[string]string{
+		env.DeployEnvFat1: "000069",
+		env.DeployEnvUat:  "000069",
+		env.DeployEnvPre:  "000161",
+		env.DeployEnvProd: "000161",
+	}
+	// log separator
+	_logSeparator = []byte("\u0001")
+)
+
+// MonitorConfig agent config.
+type MonitorConfig struct {
+	TaskID  string
+	Buffer  int
+	Proto   string         `dsn:"network"`
+	Addr    string         `dsn:"address"`
+	Chan    int            `dsn:"query.chan"`
+	Timeout xtime.Duration `dsn:"query.timeout"`
+}
+
+// MonitorHandler .
+type MonitorHandler struct {
+	c      *MonitorConfig
+	msgs   chan map[string]interface{}
+	waiter sync.WaitGroup
+	pool   sync.Pool
+}
+
+// NewMonitor a MonitorHandler.
+func NewMonitor(c *MonitorConfig) (a *MonitorHandler) {
+	if c == nil {
+		c = _defaultMonitorConfig
+	}
+	if c.Buffer == 0 {
+		c.Buffer = 1
+	}
+	if len(c.TaskID) == 0 {
+		c.TaskID = _defaultTaskIDs[env.DeployEnv]
+	}
+	c.Timeout = _agentTimeout
+	a = &MonitorHandler{c: c}
+	a.pool.New = func() interface{} {
+		return make(map[string]interface{}, 20)
+	}
+	a.msgs = make(chan map[string]interface{}, _defaultChan)
+	a.waiter.Add(1)
+	go a.writeproc()
+	return
+}
+
+// Log log to udp statsd daemon.
+func (h *MonitorHandler) Log(ctx context.Context, lv log.Level, appID string, args ...log.D) {
+	if args == nil {
+		return
+	}
+	d := h.data()
+	for _, arg := range args {
+		d[arg.Key] = arg.Value
+	}
+	if t, ok := trace.FromContext(ctx); ok {
+		d[_tid] = fmt.Sprintf("%s", t)
+	}
+	d[_appID] = env.AppID + "." + appID
+	d[_level] = lv.String()
+	d[_time] = time.Now().Format(_timeFormat)
+	select {
+	case h.msgs <- d:
+	default:
+	}
+}
+
+// writeproc write data into connection.
+func (h *MonitorHandler) writeproc() {
+	var (
+		buf   bytes.Buffer
+		conn  net.Conn
+		err   error
+		count int
+		quit  bool
+	)
+	defer h.waiter.Done()
+	taskID := []byte(h.c.TaskID)
+	tick := time.NewTicker(_mergeWait)
+	enc := json.NewEncoder(&buf)
+	for {
+		select {
+		case d := <-h.msgs:
+			if d == nil {
+				quit = true
+				goto DUMP
+			}
+			if buf.Len() >= _maxBuffer {
+				buf.Reset() // avoid oom
+			}
+			now := time.Now()
+			buf.Write(taskID)
+			buf.Write([]byte(fmt.Sprintf("%d", now.UnixNano()/1e6)))
+			enc.Encode(d)
+			h.free(d)
+			if count++; count < h.c.Buffer {
+				buf.Write(_logSeparator)
+				continue
+			}
+		case <-tick.C:
+		}
+		if conn == nil || err != nil {
+			if conn, err = net.DialTimeout(h.c.Proto, h.c.Addr, time.Duration(h.c.Timeout)); err != nil {
+				log.Error("net.DialTimeout(%s:%s) error(%v)\n", h.c.Proto, h.c.Addr, err)
+				continue
+			}
+		}
+	DUMP:
+		if conn != nil && buf.Len() > 0 {
+			count = 0
+			if _, err = conn.Write(buf.Bytes()); err != nil {
+				log.Error("conn.Write(%d bytes) error(%v)\n", buf.Len(), err)
+				conn.Close()
+			} else {
+				// only succeed reset buffer, let conn reconnect.
+				log.Info("conn Write(%d bytes) data(%v)\n", buf.Len(), string(buf.Bytes()))
+				buf.Reset()
+			}
+		}
+		if quit {
+			if conn != nil && err == nil {
+				conn.Close()
+			}
+			return
+		}
+	}
+}
+
+func (h *MonitorHandler) data() map[string]interface{} {
+	return h.pool.Get().(map[string]interface{})
+}
+
+func (h *MonitorHandler) free(d map[string]interface{}) {
+	for k := range d {
+		delete(d, k)
+	}
+	h.pool.Put(d)
+}
+
+// Close close the connection.
+func (h *MonitorHandler) Close() (err error) {
+	h.msgs <- nil
+	h.waiter.Wait()
+	return nil
+}
+
+// SetFormat .
+func (h *MonitorHandler) SetFormat(string) {
+	// discard setformat
+}
+
+// Info .
+func (h *MonitorHandler) Info(ctx context.Context, appID string, args ...log.D) {
+	h.Log(ctx, log.Level(1), appID, args...)
+}
+
+// Warn .
+func (h *MonitorHandler) Warn(ctx context.Context, appID string, args ...log.D) {
+	h.Log(ctx, log.Level(2), appID, args...)
+}
+
+// Error .
+func (h *MonitorHandler) Error(ctx context.Context, appID string, args ...log.D) {
+	h.Log(ctx, log.Level(3), appID, args...)
+}
+
+// CalCode handle codes.
+func (a *Log) CalCode() {
+	if a.HTTPCode == "" {
+		return
+	}
+	a.Codes = a.HTTPCode
+	var (
+		codes Codes
+		err   error
+	)
+	if err = json.Unmarshal([]byte(a.HTTPCode), &codes); err != nil {
+		log.Warn("s.CalCode error(%+v), codes(%s)", err, a.HTTPCode)
+		return
+	}
+	a.HTTPCode = fmt.Sprintf("%v", codes.HTTPCode)
+	a.BusinessCode = fmt.Sprintf("%v", codes.HTTPBusinessCode)
+	a.InnerCode = fmt.Sprintf("%v", codes.HTTPInnerCode)
+	// 电商inner_code 覆盖 business_code
+	if a.InnerCode != "-1" {
+		// 电商code 1 转成 0
+		if a.InnerCode == "1" {
+			a.BusinessCode = "0"
+		} else {
+			a.BusinessCode = a.InnerCode
+		}
+	}
+	if a.BusinessCode == "-1" {
+		a.BusinessCode = "0"
+	}
+}

app/interface/openplatform/monitor-end/model/monitor/handler_test.go

@@ -0,0 +1,40 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+
+	"go-common/library/log"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	m *MonitorHandler
+	c = context.Background()
+)
+
+// TestNewMonitor .
+func TestNewMonitor(t *testing.T) {
+	Convey("new monitor", t, func() {
+		m = NewMonitor(nil)
+		So(m, ShouldNotBeNil)
+	})
+}
+
+// TestLog .
+func TestLog(t *testing.T) {
+	var err error
+	Convey("info", t, func() {
+		m.Info(c, "test", []log.D{}...)
+		So(err, ShouldBeNil)
+	})
+	Convey("warn", t, func() {
+		m.Warn(c, "test", []log.D{}...)
+		So(err, ShouldBeNil)
+	})
+	Convey("error", t, func() {
+		m.Error(c, "test", []log.D{}...)
+		So(err, ShouldBeNil)
+	})
+}

app/interface/openplatform/monitor-end/model/monitor/monitor.go

@@ -0,0 +1,139 @@
+package monitor
+
+import (
+	"bytes"
+	"errors"
+	"net/url"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"go-common/library/log"
+)
+
+const (
+	_regex     = `.*\d{2,}`
+	_fieldNums = 33
+)
+
+// ErrIncompleteLog .
+var ErrIncompleteLog = errors.New("Incomplete log")
+
+// Log .
+type Log struct {
+	// from app log
+	RequestURI   string           `form:"request_uri" json:"request_uri"`
+	TimeIso      string           `form:"time_iso" json:"time_iso"`
+	IP           string           `form:"ip" json:"ip"`
+	Version      string           `form:"version" json:"version"`
+	Buvid        string           `form:"buvid" json:"buvid"`
+	Fts          string           `form:"fts" json:"fts"`
+	Proid        string           `form:"proid" json:"proid"`
+	Chid         string           `form:"chid" json:"chid"`
+	Pid          string           `form:"pid" json:"pid"`
+	Brand        string           `form:"brand" json:"brand"`
+	Deviceid     string           `form:"deviceid" json:"deviceid"`
+	Model        string           `form:"model" json:"model"`
+	Osver        string           `form:"osver" json:"osver"`
+	Ctime        string           `form:"ctime" json:"ctime"`
+	Mid          string           `form:"mid" json:"mid"`
+	Ver          string           `form:"ver" json:"ver"`
+	Net          string           `form:"net" json:"net"`
+	Oid          string           `form:"oid" json:"oid"`
+	Product      string           `form:"product" json:"product"`
+	Createtime   string           `form:"createtime" json:"createtime"`
+	Event        string           `form:"event" json:"event"`
+	SubEvent     string           `form:"sub_event" json:"sub_event"`
+	LogType      string           `form:"log_type" json:"log_type"`
+	Duration     string           `form:"duration" json:"duration"`
+	Message      string           `form:"message" json:"message"`
+	Result       string           `form:"result" json:"result"`
+	ExtJSON      string           `form:"ext_json" json:"ext_json"`
+	Traceid      string           `form:"traceid" json:"traceid"`
+	Desc         string           `form:"desc" json:"desc"`
+	Network      string           `form:"network" json:"network"`
+	TraceidEnd   string           `form:"traceid_end" json:"traceid_end"`
+	HTTPCode     string           `form:"http_code" json:"http_code"`
+	SubProduct   string           `form:"sub_product" json:"sub_product"`
+	Codes        string           `json:"codes"`
+	BusinessCode string           `json:"business_code"`
+	InnerCode    string           `json:"inner_code"`
+	TraceidSvr   string           `json:"traceid_svr"`
+	Type         string           `json:"type"`
+	Query        string           `json:"query"`
+	UserAgent    string           `json:"user_agent"`
+	Details      map[string]int64 `json:"details"`
+}
+
+// Codes .
+type Codes struct {
+	HTTPCode         interface{} `json:"http_code"`
+	HTTPInnerCode    interface{} `json:"http_inner_code"`
+	HTTPBusinessCode interface{} `json:"http_business_code"`
+}
+
+// LogData .
+func (a *Log) LogData() (appID string, logType string, kv []log.D, err error) {
+	// if a.Product == "" {
+	// 	a.Product = "nameless"
+	// }
+	//TODO 去除非法appid
+	var (
+		ok    bool
+		regex = "^[A-Za-z_]{1,}$"
+	)
+	if ok, err = regexp.Match(regex, []byte(a.Product)); err != nil || !ok {
+		a.Product = "nameless"
+	}
+	appID = a.Product
+	logType = a.LogType
+	kv = a.kv()
+	return
+}
+
+func (a *Log) kv() (kv []log.D) {
+	t := reflect.TypeOf(*a)
+	v := reflect.ValueOf(*a)
+	for i := 0; i < t.NumField()-1; i++ {
+		if v.Field(i).String() == "" {
+			continue
+		}
+		tag := t.Field(i).Tag.Get("json")
+		// s, _ := url.QueryUnescape(v.Field(i).Interface().(string))
+		kv = append(kv, log.KV(tag, v.Field(i).Interface()))
+	}
+	if a.Details != nil {
+		kv = append(kv, log.KV("detail", a.Details))
+	}
+	return
+}
+
+// LogFromBytes .
+func LogFromBytes(msg []byte) *Log {
+	var (
+		e error
+		d string
+	)
+	a := &Log{}
+	data := bytes.Split(msg, []byte("|"))
+	v := reflect.ValueOf(a).Elem()
+	l := _fieldNums
+	if len(data) < l {
+		l = len(data)
+	}
+	for i := 0; i < l; i++ {
+		if d, e = url.QueryUnescape(string(data[i])); e != nil {
+			tmp := string(data[i])
+			tmp = tmp[:strings.LastIndex(tmp, "%")]
+			if d, e = url.QueryUnescape(tmp); e != nil {
+				d = string(data[i])
+				log.Warn("s.LogFromBytes url decode error(%+v), data(%s)", e, string(data[i]))
+			}
+		}
+		if len(d) > 512 {
+			d = d[:512]
+		}
+		v.Field(i).SetString(d)
+	}
+	return a
+}

app/interface/openplatform/monitor-end/model/monitor/monitor_test.go

@@ -0,0 +1,42 @@
+package monitor
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestCase .
+type TestCase struct {
+	tag      string
+	testData *Log
+	expected int
+}
+
+// TestLogData .
+func TestLogData(t *testing.T) {
+	var (
+		tcs = []TestCase{
+			TestCase{
+				tag:      "empty log",
+				testData: &Log{},
+				expected: 1,
+			},
+			TestCase{
+				tag:      "normal data",
+				testData: &Log{Product: "test", LogType: "1", Event: "test", SubEvent: "test"},
+				expected: 0,
+			},
+		}
+	)
+	for _, tc := range tcs {
+		Convey(tc.tag, t, func() {
+			_, _, _, err := tc.testData.LogData()
+			if tc.expected == 0 {
+				So(err, ShouldBeNil)
+			} else {
+				So(err, ShouldNotBeNil)
+			}
+		})
+	}
+}